1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 31 /* Local includes */ 32 #include "i40e.h" 33 #include "i40e_diag.h" 34 #include <net/udp_tunnel.h> 35 36 const char i40e_driver_name[] = "i40e"; 37 static const char i40e_driver_string[] = 38 "Intel(R) Ethernet Connection XL710 Network Driver"; 39 40 #define DRV_KERN "-k" 41 42 #define DRV_VERSION_MAJOR 1 43 #define DRV_VERSION_MINOR 6 44 #define DRV_VERSION_BUILD 21 45 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 46 __stringify(DRV_VERSION_MINOR) "." \ 47 __stringify(DRV_VERSION_BUILD) DRV_KERN 48 const char i40e_driver_version_str[] = DRV_VERSION; 49 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 50 51 /* a bit of forward declarations */ 52 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 53 static void i40e_handle_reset_warning(struct i40e_pf *pf); 54 static int i40e_add_vsi(struct i40e_vsi *vsi); 55 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 56 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 57 static int i40e_setup_misc_vector(struct i40e_pf *pf); 58 static void i40e_determine_queue_usage(struct i40e_pf *pf); 59 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 60 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 61 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 62 63 /* i40e_pci_tbl - PCI Device ID Table 64 * 65 * Last entry must be all 0s 66 * 67 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 68 * Class, Class Mask, private data (not used) } 69 */ 70 static const struct pci_device_id i40e_pci_tbl[] = { 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 89 /* required last entry */ 90 {0, } 91 }; 92 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 93 94 #define I40E_MAX_VF_COUNT 128 95 static int debug = -1; 96 module_param(debug, uint, 0); 97 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 98 99 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 100 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 101 MODULE_LICENSE("GPL"); 102 MODULE_VERSION(DRV_VERSION); 103 104 static struct workqueue_struct *i40e_wq; 105 106 /** 107 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 108 * @hw: pointer to the HW structure 109 * @mem: ptr to mem struct to fill out 110 * @size: size of memory requested 111 * @alignment: what to align the allocation to 112 **/ 113 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 114 u64 size, u32 alignment) 115 { 116 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 117 118 mem->size = ALIGN(size, alignment); 119 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 120 &mem->pa, GFP_KERNEL); 121 if (!mem->va) 122 return -ENOMEM; 123 124 return 0; 125 } 126 127 /** 128 * i40e_free_dma_mem_d - OS specific memory free for shared code 129 * @hw: pointer to the HW structure 130 * @mem: ptr to mem struct to free 131 **/ 132 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 133 { 134 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 135 136 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 137 mem->va = NULL; 138 mem->pa = 0; 139 mem->size = 0; 140 141 return 0; 142 } 143 144 /** 145 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 146 * @hw: pointer to the HW structure 147 * @mem: ptr to mem struct to fill out 148 * @size: size of memory requested 149 **/ 150 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 151 u32 size) 152 { 153 mem->size = size; 154 mem->va = kzalloc(size, GFP_KERNEL); 155 156 if (!mem->va) 157 return -ENOMEM; 158 159 return 0; 160 } 161 162 /** 163 * i40e_free_virt_mem_d - OS specific memory free for shared code 164 * @hw: pointer to the HW structure 165 * @mem: ptr to mem struct to free 166 **/ 167 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 168 { 169 /* it's ok to kfree a NULL pointer */ 170 kfree(mem->va); 171 mem->va = NULL; 172 mem->size = 0; 173 174 return 0; 175 } 176 177 /** 178 * i40e_get_lump - find a lump of free generic resource 179 * @pf: board private structure 180 * @pile: the pile of resource to search 181 * @needed: the number of items needed 182 * @id: an owner id to stick on the items assigned 183 * 184 * Returns the base item index of the lump, or negative for error 185 * 186 * The search_hint trick and lack of advanced fit-finding only work 187 * because we're highly likely to have all the same size lump requests. 188 * Linear search time and any fragmentation should be minimal. 189 **/ 190 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 191 u16 needed, u16 id) 192 { 193 int ret = -ENOMEM; 194 int i, j; 195 196 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 197 dev_info(&pf->pdev->dev, 198 "param err: pile=%p needed=%d id=0x%04x\n", 199 pile, needed, id); 200 return -EINVAL; 201 } 202 203 /* start the linear search with an imperfect hint */ 204 i = pile->search_hint; 205 while (i < pile->num_entries) { 206 /* skip already allocated entries */ 207 if (pile->list[i] & I40E_PILE_VALID_BIT) { 208 i++; 209 continue; 210 } 211 212 /* do we have enough in this lump? */ 213 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 214 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 215 break; 216 } 217 218 if (j == needed) { 219 /* there was enough, so assign it to the requestor */ 220 for (j = 0; j < needed; j++) 221 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 222 ret = i; 223 pile->search_hint = i + j; 224 break; 225 } 226 227 /* not enough, so skip over it and continue looking */ 228 i += j; 229 } 230 231 return ret; 232 } 233 234 /** 235 * i40e_put_lump - return a lump of generic resource 236 * @pile: the pile of resource to search 237 * @index: the base item index 238 * @id: the owner id of the items assigned 239 * 240 * Returns the count of items in the lump 241 **/ 242 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 243 { 244 int valid_id = (id | I40E_PILE_VALID_BIT); 245 int count = 0; 246 int i; 247 248 if (!pile || index >= pile->num_entries) 249 return -EINVAL; 250 251 for (i = index; 252 i < pile->num_entries && pile->list[i] == valid_id; 253 i++) { 254 pile->list[i] = 0; 255 count++; 256 } 257 258 if (count && index < pile->search_hint) 259 pile->search_hint = index; 260 261 return count; 262 } 263 264 /** 265 * i40e_find_vsi_from_id - searches for the vsi with the given id 266 * @pf - the pf structure to search for the vsi 267 * @id - id of the vsi it is searching for 268 **/ 269 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 270 { 271 int i; 272 273 for (i = 0; i < pf->num_alloc_vsi; i++) 274 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 275 return pf->vsi[i]; 276 277 return NULL; 278 } 279 280 /** 281 * i40e_service_event_schedule - Schedule the service task to wake up 282 * @pf: board private structure 283 * 284 * If not already scheduled, this puts the task into the work queue 285 **/ 286 void i40e_service_event_schedule(struct i40e_pf *pf) 287 { 288 if (!test_bit(__I40E_DOWN, &pf->state) && 289 !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) && 290 !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state)) 291 queue_work(i40e_wq, &pf->service_task); 292 } 293 294 /** 295 * i40e_tx_timeout - Respond to a Tx Hang 296 * @netdev: network interface device structure 297 * 298 * If any port has noticed a Tx timeout, it is likely that the whole 299 * device is munged, not just the one netdev port, so go for the full 300 * reset. 301 **/ 302 #ifdef I40E_FCOE 303 void i40e_tx_timeout(struct net_device *netdev) 304 #else 305 static void i40e_tx_timeout(struct net_device *netdev) 306 #endif 307 { 308 struct i40e_netdev_priv *np = netdev_priv(netdev); 309 struct i40e_vsi *vsi = np->vsi; 310 struct i40e_pf *pf = vsi->back; 311 struct i40e_ring *tx_ring = NULL; 312 unsigned int i, hung_queue = 0; 313 u32 head, val; 314 315 pf->tx_timeout_count++; 316 317 /* find the stopped queue the same way the stack does */ 318 for (i = 0; i < netdev->num_tx_queues; i++) { 319 struct netdev_queue *q; 320 unsigned long trans_start; 321 322 q = netdev_get_tx_queue(netdev, i); 323 trans_start = q->trans_start; 324 if (netif_xmit_stopped(q) && 325 time_after(jiffies, 326 (trans_start + netdev->watchdog_timeo))) { 327 hung_queue = i; 328 break; 329 } 330 } 331 332 if (i == netdev->num_tx_queues) { 333 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 334 } else { 335 /* now that we have an index, find the tx_ring struct */ 336 for (i = 0; i < vsi->num_queue_pairs; i++) { 337 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 338 if (hung_queue == 339 vsi->tx_rings[i]->queue_index) { 340 tx_ring = vsi->tx_rings[i]; 341 break; 342 } 343 } 344 } 345 } 346 347 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 348 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 349 else if (time_before(jiffies, 350 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 351 return; /* don't do any new action before the next timeout */ 352 353 if (tx_ring) { 354 head = i40e_get_head(tx_ring); 355 /* Read interrupt register */ 356 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 357 val = rd32(&pf->hw, 358 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 359 tx_ring->vsi->base_vector - 1)); 360 else 361 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 362 363 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 364 vsi->seid, hung_queue, tx_ring->next_to_clean, 365 head, tx_ring->next_to_use, 366 readl(tx_ring->tail), val); 367 } 368 369 pf->tx_timeout_last_recovery = jiffies; 370 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 371 pf->tx_timeout_recovery_level, hung_queue); 372 373 switch (pf->tx_timeout_recovery_level) { 374 case 1: 375 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 376 break; 377 case 2: 378 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 379 break; 380 case 3: 381 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 382 break; 383 default: 384 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 385 break; 386 } 387 388 i40e_service_event_schedule(pf); 389 pf->tx_timeout_recovery_level++; 390 } 391 392 /** 393 * i40e_get_vsi_stats_struct - Get System Network Statistics 394 * @vsi: the VSI we care about 395 * 396 * Returns the address of the device statistics structure. 397 * The statistics are actually updated from the service task. 398 **/ 399 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 400 { 401 return &vsi->net_stats; 402 } 403 404 /** 405 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 406 * @netdev: network interface device structure 407 * 408 * Returns the address of the device statistics structure. 409 * The statistics are actually updated from the service task. 410 **/ 411 #ifdef I40E_FCOE 412 struct rtnl_link_stats64 *i40e_get_netdev_stats_struct( 413 struct net_device *netdev, 414 struct rtnl_link_stats64 *stats) 415 #else 416 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct( 417 struct net_device *netdev, 418 struct rtnl_link_stats64 *stats) 419 #endif 420 { 421 struct i40e_netdev_priv *np = netdev_priv(netdev); 422 struct i40e_ring *tx_ring, *rx_ring; 423 struct i40e_vsi *vsi = np->vsi; 424 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 425 int i; 426 427 if (test_bit(__I40E_DOWN, &vsi->state)) 428 return stats; 429 430 if (!vsi->tx_rings) 431 return stats; 432 433 rcu_read_lock(); 434 for (i = 0; i < vsi->num_queue_pairs; i++) { 435 u64 bytes, packets; 436 unsigned int start; 437 438 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 439 if (!tx_ring) 440 continue; 441 442 do { 443 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 444 packets = tx_ring->stats.packets; 445 bytes = tx_ring->stats.bytes; 446 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 447 448 stats->tx_packets += packets; 449 stats->tx_bytes += bytes; 450 rx_ring = &tx_ring[1]; 451 452 do { 453 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 454 packets = rx_ring->stats.packets; 455 bytes = rx_ring->stats.bytes; 456 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 457 458 stats->rx_packets += packets; 459 stats->rx_bytes += bytes; 460 } 461 rcu_read_unlock(); 462 463 /* following stats updated by i40e_watchdog_subtask() */ 464 stats->multicast = vsi_stats->multicast; 465 stats->tx_errors = vsi_stats->tx_errors; 466 stats->tx_dropped = vsi_stats->tx_dropped; 467 stats->rx_errors = vsi_stats->rx_errors; 468 stats->rx_dropped = vsi_stats->rx_dropped; 469 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 470 stats->rx_length_errors = vsi_stats->rx_length_errors; 471 472 return stats; 473 } 474 475 /** 476 * i40e_vsi_reset_stats - Resets all stats of the given vsi 477 * @vsi: the VSI to have its stats reset 478 **/ 479 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 480 { 481 struct rtnl_link_stats64 *ns; 482 int i; 483 484 if (!vsi) 485 return; 486 487 ns = i40e_get_vsi_stats_struct(vsi); 488 memset(ns, 0, sizeof(*ns)); 489 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 490 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 491 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 492 if (vsi->rx_rings && vsi->rx_rings[0]) { 493 for (i = 0; i < vsi->num_queue_pairs; i++) { 494 memset(&vsi->rx_rings[i]->stats, 0, 495 sizeof(vsi->rx_rings[i]->stats)); 496 memset(&vsi->rx_rings[i]->rx_stats, 0, 497 sizeof(vsi->rx_rings[i]->rx_stats)); 498 memset(&vsi->tx_rings[i]->stats, 0, 499 sizeof(vsi->tx_rings[i]->stats)); 500 memset(&vsi->tx_rings[i]->tx_stats, 0, 501 sizeof(vsi->tx_rings[i]->tx_stats)); 502 } 503 } 504 vsi->stat_offsets_loaded = false; 505 } 506 507 /** 508 * i40e_pf_reset_stats - Reset all of the stats for the given PF 509 * @pf: the PF to be reset 510 **/ 511 void i40e_pf_reset_stats(struct i40e_pf *pf) 512 { 513 int i; 514 515 memset(&pf->stats, 0, sizeof(pf->stats)); 516 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 517 pf->stat_offsets_loaded = false; 518 519 for (i = 0; i < I40E_MAX_VEB; i++) { 520 if (pf->veb[i]) { 521 memset(&pf->veb[i]->stats, 0, 522 sizeof(pf->veb[i]->stats)); 523 memset(&pf->veb[i]->stats_offsets, 0, 524 sizeof(pf->veb[i]->stats_offsets)); 525 pf->veb[i]->stat_offsets_loaded = false; 526 } 527 } 528 pf->hw_csum_rx_error = 0; 529 } 530 531 /** 532 * i40e_stat_update48 - read and update a 48 bit stat from the chip 533 * @hw: ptr to the hardware info 534 * @hireg: the high 32 bit reg to read 535 * @loreg: the low 32 bit reg to read 536 * @offset_loaded: has the initial offset been loaded yet 537 * @offset: ptr to current offset value 538 * @stat: ptr to the stat 539 * 540 * Since the device stats are not reset at PFReset, they likely will not 541 * be zeroed when the driver starts. We'll save the first values read 542 * and use them as offsets to be subtracted from the raw values in order 543 * to report stats that count from zero. In the process, we also manage 544 * the potential roll-over. 545 **/ 546 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 547 bool offset_loaded, u64 *offset, u64 *stat) 548 { 549 u64 new_data; 550 551 if (hw->device_id == I40E_DEV_ID_QEMU) { 552 new_data = rd32(hw, loreg); 553 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 554 } else { 555 new_data = rd64(hw, loreg); 556 } 557 if (!offset_loaded) 558 *offset = new_data; 559 if (likely(new_data >= *offset)) 560 *stat = new_data - *offset; 561 else 562 *stat = (new_data + BIT_ULL(48)) - *offset; 563 *stat &= 0xFFFFFFFFFFFFULL; 564 } 565 566 /** 567 * i40e_stat_update32 - read and update a 32 bit stat from the chip 568 * @hw: ptr to the hardware info 569 * @reg: the hw reg to read 570 * @offset_loaded: has the initial offset been loaded yet 571 * @offset: ptr to current offset value 572 * @stat: ptr to the stat 573 **/ 574 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 575 bool offset_loaded, u64 *offset, u64 *stat) 576 { 577 u32 new_data; 578 579 new_data = rd32(hw, reg); 580 if (!offset_loaded) 581 *offset = new_data; 582 if (likely(new_data >= *offset)) 583 *stat = (u32)(new_data - *offset); 584 else 585 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 586 } 587 588 /** 589 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 590 * @vsi: the VSI to be updated 591 **/ 592 void i40e_update_eth_stats(struct i40e_vsi *vsi) 593 { 594 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 595 struct i40e_pf *pf = vsi->back; 596 struct i40e_hw *hw = &pf->hw; 597 struct i40e_eth_stats *oes; 598 struct i40e_eth_stats *es; /* device's eth stats */ 599 600 es = &vsi->eth_stats; 601 oes = &vsi->eth_stats_offsets; 602 603 /* Gather up the stats that the hw collects */ 604 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 605 vsi->stat_offsets_loaded, 606 &oes->tx_errors, &es->tx_errors); 607 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 608 vsi->stat_offsets_loaded, 609 &oes->rx_discards, &es->rx_discards); 610 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 611 vsi->stat_offsets_loaded, 612 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 613 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 614 vsi->stat_offsets_loaded, 615 &oes->tx_errors, &es->tx_errors); 616 617 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 618 I40E_GLV_GORCL(stat_idx), 619 vsi->stat_offsets_loaded, 620 &oes->rx_bytes, &es->rx_bytes); 621 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 622 I40E_GLV_UPRCL(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->rx_unicast, &es->rx_unicast); 625 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 626 I40E_GLV_MPRCL(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->rx_multicast, &es->rx_multicast); 629 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 630 I40E_GLV_BPRCL(stat_idx), 631 vsi->stat_offsets_loaded, 632 &oes->rx_broadcast, &es->rx_broadcast); 633 634 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 635 I40E_GLV_GOTCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->tx_bytes, &es->tx_bytes); 638 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 639 I40E_GLV_UPTCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->tx_unicast, &es->tx_unicast); 642 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 643 I40E_GLV_MPTCL(stat_idx), 644 vsi->stat_offsets_loaded, 645 &oes->tx_multicast, &es->tx_multicast); 646 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 647 I40E_GLV_BPTCL(stat_idx), 648 vsi->stat_offsets_loaded, 649 &oes->tx_broadcast, &es->tx_broadcast); 650 vsi->stat_offsets_loaded = true; 651 } 652 653 /** 654 * i40e_update_veb_stats - Update Switch component statistics 655 * @veb: the VEB being updated 656 **/ 657 static void i40e_update_veb_stats(struct i40e_veb *veb) 658 { 659 struct i40e_pf *pf = veb->pf; 660 struct i40e_hw *hw = &pf->hw; 661 struct i40e_eth_stats *oes; 662 struct i40e_eth_stats *es; /* device's eth stats */ 663 struct i40e_veb_tc_stats *veb_oes; 664 struct i40e_veb_tc_stats *veb_es; 665 int i, idx = 0; 666 667 idx = veb->stats_idx; 668 es = &veb->stats; 669 oes = &veb->stats_offsets; 670 veb_es = &veb->tc_stats; 671 veb_oes = &veb->tc_stats_offsets; 672 673 /* Gather up the stats that the hw collects */ 674 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 675 veb->stat_offsets_loaded, 676 &oes->tx_discards, &es->tx_discards); 677 if (hw->revision_id > 0) 678 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 679 veb->stat_offsets_loaded, 680 &oes->rx_unknown_protocol, 681 &es->rx_unknown_protocol); 682 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 683 veb->stat_offsets_loaded, 684 &oes->rx_bytes, &es->rx_bytes); 685 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 686 veb->stat_offsets_loaded, 687 &oes->rx_unicast, &es->rx_unicast); 688 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 689 veb->stat_offsets_loaded, 690 &oes->rx_multicast, &es->rx_multicast); 691 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 692 veb->stat_offsets_loaded, 693 &oes->rx_broadcast, &es->rx_broadcast); 694 695 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 696 veb->stat_offsets_loaded, 697 &oes->tx_bytes, &es->tx_bytes); 698 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 699 veb->stat_offsets_loaded, 700 &oes->tx_unicast, &es->tx_unicast); 701 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 702 veb->stat_offsets_loaded, 703 &oes->tx_multicast, &es->tx_multicast); 704 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 705 veb->stat_offsets_loaded, 706 &oes->tx_broadcast, &es->tx_broadcast); 707 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 708 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 709 I40E_GLVEBTC_RPCL(i, idx), 710 veb->stat_offsets_loaded, 711 &veb_oes->tc_rx_packets[i], 712 &veb_es->tc_rx_packets[i]); 713 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 714 I40E_GLVEBTC_RBCL(i, idx), 715 veb->stat_offsets_loaded, 716 &veb_oes->tc_rx_bytes[i], 717 &veb_es->tc_rx_bytes[i]); 718 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 719 I40E_GLVEBTC_TPCL(i, idx), 720 veb->stat_offsets_loaded, 721 &veb_oes->tc_tx_packets[i], 722 &veb_es->tc_tx_packets[i]); 723 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 724 I40E_GLVEBTC_TBCL(i, idx), 725 veb->stat_offsets_loaded, 726 &veb_oes->tc_tx_bytes[i], 727 &veb_es->tc_tx_bytes[i]); 728 } 729 veb->stat_offsets_loaded = true; 730 } 731 732 #ifdef I40E_FCOE 733 /** 734 * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters. 735 * @vsi: the VSI that is capable of doing FCoE 736 **/ 737 static void i40e_update_fcoe_stats(struct i40e_vsi *vsi) 738 { 739 struct i40e_pf *pf = vsi->back; 740 struct i40e_hw *hw = &pf->hw; 741 struct i40e_fcoe_stats *ofs; 742 struct i40e_fcoe_stats *fs; /* device's eth stats */ 743 int idx; 744 745 if (vsi->type != I40E_VSI_FCOE) 746 return; 747 748 idx = hw->pf_id + I40E_FCOE_PF_STAT_OFFSET; 749 fs = &vsi->fcoe_stats; 750 ofs = &vsi->fcoe_stats_offsets; 751 752 i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx), 753 vsi->fcoe_stat_offsets_loaded, 754 &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets); 755 i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx), 756 vsi->fcoe_stat_offsets_loaded, 757 &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords); 758 i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx), 759 vsi->fcoe_stat_offsets_loaded, 760 &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped); 761 i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx), 762 vsi->fcoe_stat_offsets_loaded, 763 &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets); 764 i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx), 765 vsi->fcoe_stat_offsets_loaded, 766 &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords); 767 i40e_stat_update32(hw, I40E_GL_FCOECRC(idx), 768 vsi->fcoe_stat_offsets_loaded, 769 &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc); 770 i40e_stat_update32(hw, I40E_GL_FCOELAST(idx), 771 vsi->fcoe_stat_offsets_loaded, 772 &ofs->fcoe_last_error, &fs->fcoe_last_error); 773 i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx), 774 vsi->fcoe_stat_offsets_loaded, 775 &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count); 776 777 vsi->fcoe_stat_offsets_loaded = true; 778 } 779 780 #endif 781 /** 782 * i40e_update_vsi_stats - Update the vsi statistics counters. 783 * @vsi: the VSI to be updated 784 * 785 * There are a few instances where we store the same stat in a 786 * couple of different structs. This is partly because we have 787 * the netdev stats that need to be filled out, which is slightly 788 * different from the "eth_stats" defined by the chip and used in 789 * VF communications. We sort it out here. 790 **/ 791 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 792 { 793 struct i40e_pf *pf = vsi->back; 794 struct rtnl_link_stats64 *ons; 795 struct rtnl_link_stats64 *ns; /* netdev stats */ 796 struct i40e_eth_stats *oes; 797 struct i40e_eth_stats *es; /* device's eth stats */ 798 u32 tx_restart, tx_busy; 799 u64 tx_lost_interrupt; 800 struct i40e_ring *p; 801 u32 rx_page, rx_buf; 802 u64 bytes, packets; 803 unsigned int start; 804 u64 tx_linearize; 805 u64 tx_force_wb; 806 u64 rx_p, rx_b; 807 u64 tx_p, tx_b; 808 u16 q; 809 810 if (test_bit(__I40E_DOWN, &vsi->state) || 811 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 812 return; 813 814 ns = i40e_get_vsi_stats_struct(vsi); 815 ons = &vsi->net_stats_offsets; 816 es = &vsi->eth_stats; 817 oes = &vsi->eth_stats_offsets; 818 819 /* Gather up the netdev and vsi stats that the driver collects 820 * on the fly during packet processing 821 */ 822 rx_b = rx_p = 0; 823 tx_b = tx_p = 0; 824 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 825 tx_lost_interrupt = 0; 826 rx_page = 0; 827 rx_buf = 0; 828 rcu_read_lock(); 829 for (q = 0; q < vsi->num_queue_pairs; q++) { 830 /* locate Tx ring */ 831 p = ACCESS_ONCE(vsi->tx_rings[q]); 832 833 do { 834 start = u64_stats_fetch_begin_irq(&p->syncp); 835 packets = p->stats.packets; 836 bytes = p->stats.bytes; 837 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 838 tx_b += bytes; 839 tx_p += packets; 840 tx_restart += p->tx_stats.restart_queue; 841 tx_busy += p->tx_stats.tx_busy; 842 tx_linearize += p->tx_stats.tx_linearize; 843 tx_force_wb += p->tx_stats.tx_force_wb; 844 tx_lost_interrupt += p->tx_stats.tx_lost_interrupt; 845 846 /* Rx queue is part of the same block as Tx queue */ 847 p = &p[1]; 848 do { 849 start = u64_stats_fetch_begin_irq(&p->syncp); 850 packets = p->stats.packets; 851 bytes = p->stats.bytes; 852 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 853 rx_b += bytes; 854 rx_p += packets; 855 rx_buf += p->rx_stats.alloc_buff_failed; 856 rx_page += p->rx_stats.alloc_page_failed; 857 } 858 rcu_read_unlock(); 859 vsi->tx_restart = tx_restart; 860 vsi->tx_busy = tx_busy; 861 vsi->tx_linearize = tx_linearize; 862 vsi->tx_force_wb = tx_force_wb; 863 vsi->tx_lost_interrupt = tx_lost_interrupt; 864 vsi->rx_page_failed = rx_page; 865 vsi->rx_buf_failed = rx_buf; 866 867 ns->rx_packets = rx_p; 868 ns->rx_bytes = rx_b; 869 ns->tx_packets = tx_p; 870 ns->tx_bytes = tx_b; 871 872 /* update netdev stats from eth stats */ 873 i40e_update_eth_stats(vsi); 874 ons->tx_errors = oes->tx_errors; 875 ns->tx_errors = es->tx_errors; 876 ons->multicast = oes->rx_multicast; 877 ns->multicast = es->rx_multicast; 878 ons->rx_dropped = oes->rx_discards; 879 ns->rx_dropped = es->rx_discards; 880 ons->tx_dropped = oes->tx_discards; 881 ns->tx_dropped = es->tx_discards; 882 883 /* pull in a couple PF stats if this is the main vsi */ 884 if (vsi == pf->vsi[pf->lan_vsi]) { 885 ns->rx_crc_errors = pf->stats.crc_errors; 886 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 887 ns->rx_length_errors = pf->stats.rx_length_errors; 888 } 889 } 890 891 /** 892 * i40e_update_pf_stats - Update the PF statistics counters. 893 * @pf: the PF to be updated 894 **/ 895 static void i40e_update_pf_stats(struct i40e_pf *pf) 896 { 897 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 898 struct i40e_hw_port_stats *nsd = &pf->stats; 899 struct i40e_hw *hw = &pf->hw; 900 u32 val; 901 int i; 902 903 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 904 I40E_GLPRT_GORCL(hw->port), 905 pf->stat_offsets_loaded, 906 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 907 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 908 I40E_GLPRT_GOTCL(hw->port), 909 pf->stat_offsets_loaded, 910 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 911 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->eth.rx_discards, 914 &nsd->eth.rx_discards); 915 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 916 I40E_GLPRT_UPRCL(hw->port), 917 pf->stat_offsets_loaded, 918 &osd->eth.rx_unicast, 919 &nsd->eth.rx_unicast); 920 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 921 I40E_GLPRT_MPRCL(hw->port), 922 pf->stat_offsets_loaded, 923 &osd->eth.rx_multicast, 924 &nsd->eth.rx_multicast); 925 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 926 I40E_GLPRT_BPRCL(hw->port), 927 pf->stat_offsets_loaded, 928 &osd->eth.rx_broadcast, 929 &nsd->eth.rx_broadcast); 930 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 931 I40E_GLPRT_UPTCL(hw->port), 932 pf->stat_offsets_loaded, 933 &osd->eth.tx_unicast, 934 &nsd->eth.tx_unicast); 935 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 936 I40E_GLPRT_MPTCL(hw->port), 937 pf->stat_offsets_loaded, 938 &osd->eth.tx_multicast, 939 &nsd->eth.tx_multicast); 940 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 941 I40E_GLPRT_BPTCL(hw->port), 942 pf->stat_offsets_loaded, 943 &osd->eth.tx_broadcast, 944 &nsd->eth.tx_broadcast); 945 946 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 947 pf->stat_offsets_loaded, 948 &osd->tx_dropped_link_down, 949 &nsd->tx_dropped_link_down); 950 951 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 952 pf->stat_offsets_loaded, 953 &osd->crc_errors, &nsd->crc_errors); 954 955 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 956 pf->stat_offsets_loaded, 957 &osd->illegal_bytes, &nsd->illegal_bytes); 958 959 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 960 pf->stat_offsets_loaded, 961 &osd->mac_local_faults, 962 &nsd->mac_local_faults); 963 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 964 pf->stat_offsets_loaded, 965 &osd->mac_remote_faults, 966 &nsd->mac_remote_faults); 967 968 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 969 pf->stat_offsets_loaded, 970 &osd->rx_length_errors, 971 &nsd->rx_length_errors); 972 973 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 974 pf->stat_offsets_loaded, 975 &osd->link_xon_rx, &nsd->link_xon_rx); 976 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 977 pf->stat_offsets_loaded, 978 &osd->link_xon_tx, &nsd->link_xon_tx); 979 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->link_xoff_rx, &nsd->link_xoff_rx); 982 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 983 pf->stat_offsets_loaded, 984 &osd->link_xoff_tx, &nsd->link_xoff_tx); 985 986 for (i = 0; i < 8; i++) { 987 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 988 pf->stat_offsets_loaded, 989 &osd->priority_xoff_rx[i], 990 &nsd->priority_xoff_rx[i]); 991 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 992 pf->stat_offsets_loaded, 993 &osd->priority_xon_rx[i], 994 &nsd->priority_xon_rx[i]); 995 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 996 pf->stat_offsets_loaded, 997 &osd->priority_xon_tx[i], 998 &nsd->priority_xon_tx[i]); 999 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 1000 pf->stat_offsets_loaded, 1001 &osd->priority_xoff_tx[i], 1002 &nsd->priority_xoff_tx[i]); 1003 i40e_stat_update32(hw, 1004 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1005 pf->stat_offsets_loaded, 1006 &osd->priority_xon_2_xoff[i], 1007 &nsd->priority_xon_2_xoff[i]); 1008 } 1009 1010 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1011 I40E_GLPRT_PRC64L(hw->port), 1012 pf->stat_offsets_loaded, 1013 &osd->rx_size_64, &nsd->rx_size_64); 1014 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1015 I40E_GLPRT_PRC127L(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->rx_size_127, &nsd->rx_size_127); 1018 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1019 I40E_GLPRT_PRC255L(hw->port), 1020 pf->stat_offsets_loaded, 1021 &osd->rx_size_255, &nsd->rx_size_255); 1022 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1023 I40E_GLPRT_PRC511L(hw->port), 1024 pf->stat_offsets_loaded, 1025 &osd->rx_size_511, &nsd->rx_size_511); 1026 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1027 I40E_GLPRT_PRC1023L(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->rx_size_1023, &nsd->rx_size_1023); 1030 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1031 I40E_GLPRT_PRC1522L(hw->port), 1032 pf->stat_offsets_loaded, 1033 &osd->rx_size_1522, &nsd->rx_size_1522); 1034 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1035 I40E_GLPRT_PRC9522L(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->rx_size_big, &nsd->rx_size_big); 1038 1039 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1040 I40E_GLPRT_PTC64L(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->tx_size_64, &nsd->tx_size_64); 1043 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1044 I40E_GLPRT_PTC127L(hw->port), 1045 pf->stat_offsets_loaded, 1046 &osd->tx_size_127, &nsd->tx_size_127); 1047 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1048 I40E_GLPRT_PTC255L(hw->port), 1049 pf->stat_offsets_loaded, 1050 &osd->tx_size_255, &nsd->tx_size_255); 1051 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1052 I40E_GLPRT_PTC511L(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->tx_size_511, &nsd->tx_size_511); 1055 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1056 I40E_GLPRT_PTC1023L(hw->port), 1057 pf->stat_offsets_loaded, 1058 &osd->tx_size_1023, &nsd->tx_size_1023); 1059 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1060 I40E_GLPRT_PTC1522L(hw->port), 1061 pf->stat_offsets_loaded, 1062 &osd->tx_size_1522, &nsd->tx_size_1522); 1063 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1064 I40E_GLPRT_PTC9522L(hw->port), 1065 pf->stat_offsets_loaded, 1066 &osd->tx_size_big, &nsd->tx_size_big); 1067 1068 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1069 pf->stat_offsets_loaded, 1070 &osd->rx_undersize, &nsd->rx_undersize); 1071 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1072 pf->stat_offsets_loaded, 1073 &osd->rx_fragments, &nsd->rx_fragments); 1074 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1075 pf->stat_offsets_loaded, 1076 &osd->rx_oversize, &nsd->rx_oversize); 1077 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1078 pf->stat_offsets_loaded, 1079 &osd->rx_jabber, &nsd->rx_jabber); 1080 1081 /* FDIR stats */ 1082 i40e_stat_update32(hw, 1083 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)), 1084 pf->stat_offsets_loaded, 1085 &osd->fd_atr_match, &nsd->fd_atr_match); 1086 i40e_stat_update32(hw, 1087 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)), 1088 pf->stat_offsets_loaded, 1089 &osd->fd_sb_match, &nsd->fd_sb_match); 1090 i40e_stat_update32(hw, 1091 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)), 1092 pf->stat_offsets_loaded, 1093 &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match); 1094 1095 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1096 nsd->tx_lpi_status = 1097 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1098 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1099 nsd->rx_lpi_status = 1100 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1101 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1102 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1103 pf->stat_offsets_loaded, 1104 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1105 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1106 pf->stat_offsets_loaded, 1107 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1108 1109 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1110 !(pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) 1111 nsd->fd_sb_status = true; 1112 else 1113 nsd->fd_sb_status = false; 1114 1115 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1116 !(pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) 1117 nsd->fd_atr_status = true; 1118 else 1119 nsd->fd_atr_status = false; 1120 1121 pf->stat_offsets_loaded = true; 1122 } 1123 1124 /** 1125 * i40e_update_stats - Update the various statistics counters. 1126 * @vsi: the VSI to be updated 1127 * 1128 * Update the various stats for this VSI and its related entities. 1129 **/ 1130 void i40e_update_stats(struct i40e_vsi *vsi) 1131 { 1132 struct i40e_pf *pf = vsi->back; 1133 1134 if (vsi == pf->vsi[pf->lan_vsi]) 1135 i40e_update_pf_stats(pf); 1136 1137 i40e_update_vsi_stats(vsi); 1138 #ifdef I40E_FCOE 1139 i40e_update_fcoe_stats(vsi); 1140 #endif 1141 } 1142 1143 /** 1144 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1145 * @vsi: the VSI to be searched 1146 * @macaddr: the MAC address 1147 * @vlan: the vlan 1148 * 1149 * Returns ptr to the filter object or NULL 1150 **/ 1151 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1152 const u8 *macaddr, s16 vlan) 1153 { 1154 struct i40e_mac_filter *f; 1155 u64 key; 1156 1157 if (!vsi || !macaddr) 1158 return NULL; 1159 1160 key = i40e_addr_to_hkey(macaddr); 1161 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1162 if ((ether_addr_equal(macaddr, f->macaddr)) && 1163 (vlan == f->vlan)) 1164 return f; 1165 } 1166 return NULL; 1167 } 1168 1169 /** 1170 * i40e_find_mac - Find a mac addr in the macvlan filters list 1171 * @vsi: the VSI to be searched 1172 * @macaddr: the MAC address we are searching for 1173 * 1174 * Returns the first filter with the provided MAC address or NULL if 1175 * MAC address was not found 1176 **/ 1177 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1178 { 1179 struct i40e_mac_filter *f; 1180 u64 key; 1181 1182 if (!vsi || !macaddr) 1183 return NULL; 1184 1185 key = i40e_addr_to_hkey(macaddr); 1186 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1187 if ((ether_addr_equal(macaddr, f->macaddr))) 1188 return f; 1189 } 1190 return NULL; 1191 } 1192 1193 /** 1194 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1195 * @vsi: the VSI to be searched 1196 * 1197 * Returns true if VSI is in vlan mode or false otherwise 1198 **/ 1199 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1200 { 1201 /* If we have a PVID, always operate in VLAN mode */ 1202 if (vsi->info.pvid) 1203 return true; 1204 1205 /* We need to operate in VLAN mode whenever we have any filters with 1206 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1207 * time, incurring search cost repeatedly. However, we can notice two 1208 * things: 1209 * 1210 * 1) the only place where we can gain a VLAN filter is in 1211 * i40e_add_filter. 1212 * 1213 * 2) the only place where filters are actually removed is in 1214 * i40e_vsi_sync_filters_subtask. 1215 * 1216 * Thus, we can simply use a boolean value, has_vlan_filters which we 1217 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1218 * we have to perform the full search after deleting filters in 1219 * i40e_vsi_sync_filters_subtask, but we already have to search 1220 * filters here and can perform the check at the same time. This 1221 * results in avoiding embedding a loop for VLAN mode inside another 1222 * loop over all the filters, and should maintain correctness as noted 1223 * above. 1224 */ 1225 return vsi->has_vlan_filter; 1226 } 1227 1228 /** 1229 * i40e_add_filter - Add a mac/vlan filter to the VSI 1230 * @vsi: the VSI to be searched 1231 * @macaddr: the MAC address 1232 * @vlan: the vlan 1233 * 1234 * Returns ptr to the filter object or NULL when no memory available. 1235 * 1236 * NOTE: This function is expected to be called with mac_filter_hash_lock 1237 * being held. 1238 **/ 1239 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1240 const u8 *macaddr, s16 vlan) 1241 { 1242 struct i40e_mac_filter *f; 1243 u64 key; 1244 1245 if (!vsi || !macaddr) 1246 return NULL; 1247 1248 /* Do not allow broadcast filter to be added since broadcast filter 1249 * is added as part of add VSI for any newly created VSI except 1250 * FDIR VSI 1251 */ 1252 if (is_broadcast_ether_addr(macaddr)) 1253 return NULL; 1254 1255 f = i40e_find_filter(vsi, macaddr, vlan); 1256 if (!f) { 1257 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1258 if (!f) 1259 return NULL; 1260 1261 /* Update the boolean indicating if we need to function in 1262 * VLAN mode. 1263 */ 1264 if (vlan >= 0) 1265 vsi->has_vlan_filter = true; 1266 1267 ether_addr_copy(f->macaddr, macaddr); 1268 f->vlan = vlan; 1269 /* If we're in overflow promisc mode, set the state directly 1270 * to failed, so we don't bother to try sending the filter 1271 * to the hardware. 1272 */ 1273 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state)) 1274 f->state = I40E_FILTER_FAILED; 1275 else 1276 f->state = I40E_FILTER_NEW; 1277 INIT_HLIST_NODE(&f->hlist); 1278 1279 key = i40e_addr_to_hkey(macaddr); 1280 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1281 1282 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1283 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1284 } 1285 1286 /* If we're asked to add a filter that has been marked for removal, it 1287 * is safe to simply restore it to active state. __i40e_del_filter 1288 * will have simply deleted any filters which were previously marked 1289 * NEW or FAILED, so if it is currently marked REMOVE it must have 1290 * previously been ACTIVE. Since we haven't yet run the sync filters 1291 * task, just restore this filter to the ACTIVE state so that the 1292 * sync task leaves it in place 1293 */ 1294 if (f->state == I40E_FILTER_REMOVE) 1295 f->state = I40E_FILTER_ACTIVE; 1296 1297 return f; 1298 } 1299 1300 /** 1301 * __i40e_del_filter - Remove a specific filter from the VSI 1302 * @vsi: VSI to remove from 1303 * @f: the filter to remove from the list 1304 * 1305 * This function should be called instead of i40e_del_filter only if you know 1306 * the exact filter you will remove already, such as via i40e_find_filter or 1307 * i40e_find_mac. 1308 * 1309 * NOTE: This function is expected to be called with mac_filter_hash_lock 1310 * being held. 1311 * ANOTHER NOTE: This function MUST be called from within the context of 1312 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1313 * instead of list_for_each_entry(). 1314 **/ 1315 static void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1316 { 1317 if (!f) 1318 return; 1319 1320 if ((f->state == I40E_FILTER_FAILED) || 1321 (f->state == I40E_FILTER_NEW)) { 1322 /* this one never got added by the FW. Just remove it, 1323 * no need to sync anything. 1324 */ 1325 hash_del(&f->hlist); 1326 kfree(f); 1327 } else { 1328 f->state = I40E_FILTER_REMOVE; 1329 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1330 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1331 } 1332 } 1333 1334 /** 1335 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1336 * @vsi: the VSI to be searched 1337 * @macaddr: the MAC address 1338 * @vlan: the VLAN 1339 * 1340 * NOTE: This function is expected to be called with mac_filter_hash_lock 1341 * being held. 1342 * ANOTHER NOTE: This function MUST be called from within the context of 1343 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1344 * instead of list_for_each_entry(). 1345 **/ 1346 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1347 { 1348 struct i40e_mac_filter *f; 1349 1350 if (!vsi || !macaddr) 1351 return; 1352 1353 f = i40e_find_filter(vsi, macaddr, vlan); 1354 __i40e_del_filter(vsi, f); 1355 } 1356 1357 /** 1358 * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans 1359 * @vsi: the VSI to be searched 1360 * @macaddr: the mac address to be filtered 1361 * 1362 * Goes through all the macvlan filters and adds a macvlan filter for each 1363 * unique vlan that already exists. If a PVID has been assigned, instead only 1364 * add the macaddr to that VLAN. 1365 * 1366 * Returns last filter added on success, else NULL 1367 **/ 1368 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, 1369 const u8 *macaddr) 1370 { 1371 struct i40e_mac_filter *f, *add = NULL; 1372 struct hlist_node *h; 1373 int bkt; 1374 1375 if (vsi->info.pvid) 1376 return i40e_add_filter(vsi, macaddr, 1377 le16_to_cpu(vsi->info.pvid)); 1378 1379 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1380 if (f->state == I40E_FILTER_REMOVE) 1381 continue; 1382 add = i40e_add_filter(vsi, macaddr, f->vlan); 1383 if (!add) 1384 return NULL; 1385 } 1386 1387 return add; 1388 } 1389 1390 /** 1391 * i40e_del_mac_all_vlan - Remove a MAC filter from all VLANS 1392 * @vsi: the VSI to be searched 1393 * @macaddr: the mac address to be removed 1394 * 1395 * Removes a given MAC address from a VSI, regardless of VLAN 1396 * 1397 * Returns 0 for success, or error 1398 **/ 1399 int i40e_del_mac_all_vlan(struct i40e_vsi *vsi, const u8 *macaddr) 1400 { 1401 struct i40e_mac_filter *f; 1402 struct hlist_node *h; 1403 bool found = false; 1404 int bkt; 1405 1406 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1407 "Missing mac_filter_hash_lock\n"); 1408 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1409 if (ether_addr_equal(macaddr, f->macaddr)) { 1410 __i40e_del_filter(vsi, f); 1411 found = true; 1412 } 1413 } 1414 1415 if (found) 1416 return 0; 1417 else 1418 return -ENOENT; 1419 } 1420 1421 /** 1422 * i40e_set_mac - NDO callback to set mac address 1423 * @netdev: network interface device structure 1424 * @p: pointer to an address structure 1425 * 1426 * Returns 0 on success, negative on failure 1427 **/ 1428 #ifdef I40E_FCOE 1429 int i40e_set_mac(struct net_device *netdev, void *p) 1430 #else 1431 static int i40e_set_mac(struct net_device *netdev, void *p) 1432 #endif 1433 { 1434 struct i40e_netdev_priv *np = netdev_priv(netdev); 1435 struct i40e_vsi *vsi = np->vsi; 1436 struct i40e_pf *pf = vsi->back; 1437 struct i40e_hw *hw = &pf->hw; 1438 struct sockaddr *addr = p; 1439 1440 if (!is_valid_ether_addr(addr->sa_data)) 1441 return -EADDRNOTAVAIL; 1442 1443 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1444 netdev_info(netdev, "already using mac address %pM\n", 1445 addr->sa_data); 1446 return 0; 1447 } 1448 1449 if (test_bit(__I40E_DOWN, &vsi->back->state) || 1450 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 1451 return -EADDRNOTAVAIL; 1452 1453 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1454 netdev_info(netdev, "returning to hw mac address %pM\n", 1455 hw->mac.addr); 1456 else 1457 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1458 1459 spin_lock_bh(&vsi->mac_filter_hash_lock); 1460 i40e_del_mac_all_vlan(vsi, netdev->dev_addr); 1461 i40e_put_mac_in_vlan(vsi, addr->sa_data); 1462 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1463 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1464 if (vsi->type == I40E_VSI_MAIN) { 1465 i40e_status ret; 1466 1467 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1468 I40E_AQC_WRITE_TYPE_LAA_WOL, 1469 addr->sa_data, NULL); 1470 if (ret) 1471 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1472 i40e_stat_str(hw, ret), 1473 i40e_aq_str(hw, hw->aq.asq_last_status)); 1474 } 1475 1476 /* schedule our worker thread which will take care of 1477 * applying the new filter changes 1478 */ 1479 i40e_service_event_schedule(vsi->back); 1480 return 0; 1481 } 1482 1483 /** 1484 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1485 * @vsi: the VSI being setup 1486 * @ctxt: VSI context structure 1487 * @enabled_tc: Enabled TCs bitmap 1488 * @is_add: True if called before Add VSI 1489 * 1490 * Setup VSI queue mapping for enabled traffic classes. 1491 **/ 1492 #ifdef I40E_FCOE 1493 void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1494 struct i40e_vsi_context *ctxt, 1495 u8 enabled_tc, 1496 bool is_add) 1497 #else 1498 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1499 struct i40e_vsi_context *ctxt, 1500 u8 enabled_tc, 1501 bool is_add) 1502 #endif 1503 { 1504 struct i40e_pf *pf = vsi->back; 1505 u16 sections = 0; 1506 u8 netdev_tc = 0; 1507 u16 numtc = 0; 1508 u16 qcount; 1509 u8 offset; 1510 u16 qmap; 1511 int i; 1512 u16 num_tc_qps = 0; 1513 1514 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1515 offset = 0; 1516 1517 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1518 /* Find numtc from enabled TC bitmap */ 1519 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1520 if (enabled_tc & BIT(i)) /* TC is enabled */ 1521 numtc++; 1522 } 1523 if (!numtc) { 1524 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1525 numtc = 1; 1526 } 1527 } else { 1528 /* At least TC0 is enabled in case of non-DCB case */ 1529 numtc = 1; 1530 } 1531 1532 vsi->tc_config.numtc = numtc; 1533 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1534 /* Number of queues per enabled TC */ 1535 qcount = vsi->alloc_queue_pairs; 1536 1537 num_tc_qps = qcount / numtc; 1538 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1539 1540 /* Setup queue offset/count for all TCs for given VSI */ 1541 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1542 /* See if the given TC is enabled for the given VSI */ 1543 if (vsi->tc_config.enabled_tc & BIT(i)) { 1544 /* TC is enabled */ 1545 int pow, num_qps; 1546 1547 switch (vsi->type) { 1548 case I40E_VSI_MAIN: 1549 qcount = min_t(int, pf->alloc_rss_size, 1550 num_tc_qps); 1551 break; 1552 #ifdef I40E_FCOE 1553 case I40E_VSI_FCOE: 1554 qcount = num_tc_qps; 1555 break; 1556 #endif 1557 case I40E_VSI_FDIR: 1558 case I40E_VSI_SRIOV: 1559 case I40E_VSI_VMDQ2: 1560 default: 1561 qcount = num_tc_qps; 1562 WARN_ON(i != 0); 1563 break; 1564 } 1565 vsi->tc_config.tc_info[i].qoffset = offset; 1566 vsi->tc_config.tc_info[i].qcount = qcount; 1567 1568 /* find the next higher power-of-2 of num queue pairs */ 1569 num_qps = qcount; 1570 pow = 0; 1571 while (num_qps && (BIT_ULL(pow) < qcount)) { 1572 pow++; 1573 num_qps >>= 1; 1574 } 1575 1576 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1577 qmap = 1578 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1579 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1580 1581 offset += qcount; 1582 } else { 1583 /* TC is not enabled so set the offset to 1584 * default queue and allocate one queue 1585 * for the given TC. 1586 */ 1587 vsi->tc_config.tc_info[i].qoffset = 0; 1588 vsi->tc_config.tc_info[i].qcount = 1; 1589 vsi->tc_config.tc_info[i].netdev_tc = 0; 1590 1591 qmap = 0; 1592 } 1593 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1594 } 1595 1596 /* Set actual Tx/Rx queue pairs */ 1597 vsi->num_queue_pairs = offset; 1598 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1599 if (vsi->req_queue_pairs > 0) 1600 vsi->num_queue_pairs = vsi->req_queue_pairs; 1601 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1602 vsi->num_queue_pairs = pf->num_lan_msix; 1603 } 1604 1605 /* Scheduler section valid can only be set for ADD VSI */ 1606 if (is_add) { 1607 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1608 1609 ctxt->info.up_enable_bits = enabled_tc; 1610 } 1611 if (vsi->type == I40E_VSI_SRIOV) { 1612 ctxt->info.mapping_flags |= 1613 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1614 for (i = 0; i < vsi->num_queue_pairs; i++) 1615 ctxt->info.queue_mapping[i] = 1616 cpu_to_le16(vsi->base_queue + i); 1617 } else { 1618 ctxt->info.mapping_flags |= 1619 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1620 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1621 } 1622 ctxt->info.valid_sections |= cpu_to_le16(sections); 1623 } 1624 1625 /** 1626 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1627 * @netdev: the netdevice 1628 * @addr: address to add 1629 * 1630 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1631 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1632 */ 1633 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1634 { 1635 struct i40e_netdev_priv *np = netdev_priv(netdev); 1636 struct i40e_vsi *vsi = np->vsi; 1637 struct i40e_mac_filter *f; 1638 1639 if (i40e_is_vsi_in_vlan(vsi)) 1640 f = i40e_put_mac_in_vlan(vsi, addr); 1641 else 1642 f = i40e_add_filter(vsi, addr, I40E_VLAN_ANY); 1643 1644 if (f) 1645 return 0; 1646 else 1647 return -ENOMEM; 1648 } 1649 1650 /** 1651 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1652 * @netdev: the netdevice 1653 * @addr: address to add 1654 * 1655 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1656 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1657 */ 1658 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1659 { 1660 struct i40e_netdev_priv *np = netdev_priv(netdev); 1661 struct i40e_vsi *vsi = np->vsi; 1662 1663 if (i40e_is_vsi_in_vlan(vsi)) 1664 i40e_del_mac_all_vlan(vsi, addr); 1665 else 1666 i40e_del_filter(vsi, addr, I40E_VLAN_ANY); 1667 1668 return 0; 1669 } 1670 1671 /** 1672 * i40e_set_rx_mode - NDO callback to set the netdev filters 1673 * @netdev: network interface device structure 1674 **/ 1675 #ifdef I40E_FCOE 1676 void i40e_set_rx_mode(struct net_device *netdev) 1677 #else 1678 static void i40e_set_rx_mode(struct net_device *netdev) 1679 #endif 1680 { 1681 struct i40e_netdev_priv *np = netdev_priv(netdev); 1682 struct i40e_vsi *vsi = np->vsi; 1683 1684 spin_lock_bh(&vsi->mac_filter_hash_lock); 1685 1686 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1687 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1688 1689 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1690 1691 /* check for other flag changes */ 1692 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1693 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1694 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1695 } 1696 1697 /* schedule our worker thread which will take care of 1698 * applying the new filter changes 1699 */ 1700 i40e_service_event_schedule(vsi->back); 1701 } 1702 1703 /** 1704 * i40e_undo_filter_entries - Undo the changes made to MAC filter entries 1705 * @vsi: Pointer to VSI struct 1706 * @from: Pointer to list which contains MAC filter entries - changes to 1707 * those entries needs to be undone. 1708 * 1709 * MAC filter entries from list were slated to be sent to firmware, either for 1710 * addition or deletion. 1711 **/ 1712 static void i40e_undo_filter_entries(struct i40e_vsi *vsi, 1713 struct hlist_head *from) 1714 { 1715 struct i40e_mac_filter *f; 1716 struct hlist_node *h; 1717 1718 hlist_for_each_entry_safe(f, h, from, hlist) { 1719 u64 key = i40e_addr_to_hkey(f->macaddr); 1720 1721 /* Move the element back into MAC filter list*/ 1722 hlist_del(&f->hlist); 1723 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1724 } 1725 } 1726 1727 /** 1728 * i40e_update_filter_state - Update filter state based on return data 1729 * from firmware 1730 * @count: Number of filters added 1731 * @add_list: return data from fw 1732 * @head: pointer to first filter in current batch 1733 * @aq_err: status from fw 1734 * 1735 * MAC filter entries from list were slated to be added to device. Returns 1736 * number of successful filters. Note that 0 does NOT mean success! 1737 **/ 1738 static int 1739 i40e_update_filter_state(int count, 1740 struct i40e_aqc_add_macvlan_element_data *add_list, 1741 struct i40e_mac_filter *add_head, int aq_err) 1742 { 1743 int retval = 0; 1744 int i; 1745 1746 1747 if (!aq_err) { 1748 retval = count; 1749 /* Everything's good, mark all filters active. */ 1750 for (i = 0; i < count ; i++) { 1751 add_head->state = I40E_FILTER_ACTIVE; 1752 add_head = hlist_entry(add_head->hlist.next, 1753 typeof(struct i40e_mac_filter), 1754 hlist); 1755 } 1756 } else if (aq_err == I40E_AQ_RC_ENOSPC) { 1757 /* Device ran out of filter space. Check the return value 1758 * for each filter to see which ones are active. 1759 */ 1760 for (i = 0; i < count ; i++) { 1761 if (add_list[i].match_method == 1762 I40E_AQC_MM_ERR_NO_RES) { 1763 add_head->state = I40E_FILTER_FAILED; 1764 } else { 1765 add_head->state = I40E_FILTER_ACTIVE; 1766 retval++; 1767 } 1768 add_head = hlist_entry(add_head->hlist.next, 1769 typeof(struct i40e_mac_filter), 1770 hlist); 1771 } 1772 } else { 1773 /* Some other horrible thing happened, fail all filters */ 1774 retval = 0; 1775 for (i = 0; i < count ; i++) { 1776 add_head->state = I40E_FILTER_FAILED; 1777 add_head = hlist_entry(add_head->hlist.next, 1778 typeof(struct i40e_mac_filter), 1779 hlist); 1780 } 1781 } 1782 return retval; 1783 } 1784 1785 /** 1786 * i40e_aqc_del_filters - Request firmware to delete a set of filters 1787 * @vsi: ptr to the VSI 1788 * @vsi_name: name to display in messages 1789 * @list: the list of filters to send to firmware 1790 * @num_del: the number of filters to delete 1791 * @retval: Set to -EIO on failure to delete 1792 * 1793 * Send a request to firmware via AdminQ to delete a set of filters. Uses 1794 * *retval instead of a return value so that success does not force ret_val to 1795 * be set to 0. This ensures that a sequence of calls to this function 1796 * preserve the previous value of *retval on successful delete. 1797 */ 1798 static 1799 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 1800 struct i40e_aqc_remove_macvlan_element_data *list, 1801 int num_del, int *retval) 1802 { 1803 struct i40e_hw *hw = &vsi->back->hw; 1804 i40e_status aq_ret; 1805 int aq_err; 1806 1807 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 1808 aq_err = hw->aq.asq_last_status; 1809 1810 /* Explicitly ignore and do not report when firmware returns ENOENT */ 1811 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 1812 *retval = -EIO; 1813 dev_info(&vsi->back->pdev->dev, 1814 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 1815 vsi_name, i40e_stat_str(hw, aq_ret), 1816 i40e_aq_str(hw, aq_err)); 1817 } 1818 } 1819 1820 /** 1821 * i40e_aqc_add_filters - Request firmware to add a set of filters 1822 * @vsi: ptr to the VSI 1823 * @vsi_name: name to display in messages 1824 * @list: the list of filters to send to firmware 1825 * @add_head: Position in the add hlist 1826 * @num_add: the number of filters to add 1827 * @promisc_change: set to true on exit if promiscuous mode was forced on 1828 * 1829 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 1830 * promisc_changed to true if the firmware has run out of space for more 1831 * filters. 1832 */ 1833 static 1834 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 1835 struct i40e_aqc_add_macvlan_element_data *list, 1836 struct i40e_mac_filter *add_head, 1837 int num_add, bool *promisc_changed) 1838 { 1839 struct i40e_hw *hw = &vsi->back->hw; 1840 i40e_status aq_ret; 1841 int aq_err, fcnt; 1842 1843 aq_ret = i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 1844 aq_err = hw->aq.asq_last_status; 1845 fcnt = i40e_update_filter_state(num_add, list, add_head, aq_ret); 1846 vsi->active_filters += fcnt; 1847 1848 if (fcnt != num_add) { 1849 *promisc_changed = true; 1850 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 1851 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 1852 dev_warn(&vsi->back->pdev->dev, 1853 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 1854 i40e_aq_str(hw, aq_err), 1855 vsi_name); 1856 } 1857 } 1858 1859 /** 1860 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 1861 * @vsi: ptr to the VSI 1862 * 1863 * Push any outstanding VSI filter changes through the AdminQ. 1864 * 1865 * Returns 0 or error value 1866 **/ 1867 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 1868 { 1869 struct hlist_head tmp_add_list, tmp_del_list; 1870 struct i40e_mac_filter *f, *add_head = NULL; 1871 struct i40e_hw *hw = &vsi->back->hw; 1872 unsigned int vlan_any_filters = 0; 1873 unsigned int non_vlan_filters = 0; 1874 unsigned int vlan_filters = 0; 1875 bool promisc_changed = false; 1876 char vsi_name[16] = "PF"; 1877 int filter_list_len = 0; 1878 i40e_status aq_ret = 0; 1879 u32 changed_flags = 0; 1880 struct hlist_node *h; 1881 struct i40e_pf *pf; 1882 int num_add = 0; 1883 int num_del = 0; 1884 int retval = 0; 1885 u16 cmd_flags; 1886 int list_size; 1887 int bkt; 1888 1889 /* empty array typed pointers, kcalloc later */ 1890 struct i40e_aqc_add_macvlan_element_data *add_list; 1891 struct i40e_aqc_remove_macvlan_element_data *del_list; 1892 1893 while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state)) 1894 usleep_range(1000, 2000); 1895 pf = vsi->back; 1896 1897 if (vsi->netdev) { 1898 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 1899 vsi->current_netdev_flags = vsi->netdev->flags; 1900 } 1901 1902 INIT_HLIST_HEAD(&tmp_add_list); 1903 INIT_HLIST_HEAD(&tmp_del_list); 1904 1905 if (vsi->type == I40E_VSI_SRIOV) 1906 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 1907 else if (vsi->type != I40E_VSI_MAIN) 1908 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 1909 1910 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 1911 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 1912 1913 spin_lock_bh(&vsi->mac_filter_hash_lock); 1914 /* Create a list of filters to delete. */ 1915 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1916 if (f->state == I40E_FILTER_REMOVE) { 1917 /* Move the element into temporary del_list */ 1918 hash_del(&f->hlist); 1919 hlist_add_head(&f->hlist, &tmp_del_list); 1920 vsi->active_filters--; 1921 1922 /* Avoid counting removed filters */ 1923 continue; 1924 } 1925 if (f->state == I40E_FILTER_NEW) { 1926 hash_del(&f->hlist); 1927 hlist_add_head(&f->hlist, &tmp_add_list); 1928 } 1929 1930 /* Count the number of each type of filter we have 1931 * remaining, ignoring any filters we're about to 1932 * delete. 1933 */ 1934 if (f->vlan > 0) 1935 vlan_filters++; 1936 else if (!f->vlan) 1937 non_vlan_filters++; 1938 else 1939 vlan_any_filters++; 1940 } 1941 1942 /* We should never have VLAN=-1 filters at the same time as we 1943 * have either VLAN=0 or VLAN>0 filters, so warn about this 1944 * case here to help catch any issues. 1945 */ 1946 WARN_ON(vlan_any_filters && (vlan_filters + non_vlan_filters)); 1947 1948 /* If we only have VLAN=0 filters remaining, and don't have 1949 * any other VLAN filters, we need to convert these VLAN=0 1950 * filters into VLAN=-1 (I40E_VLAN_ANY) so that we operate 1951 * correctly in non-VLAN mode and receive all traffic tagged 1952 * or untagged. 1953 */ 1954 if (non_vlan_filters && !vlan_filters) { 1955 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, 1956 hlist) { 1957 /* Only replace VLAN=0 filters */ 1958 if (f->vlan) 1959 continue; 1960 1961 /* Allocate a replacement element */ 1962 add_head = kzalloc(sizeof(*add_head), 1963 GFP_KERNEL); 1964 if (!add_head) 1965 goto err_no_memory_locked; 1966 1967 /* Copy the filter, with new state and VLAN */ 1968 *add_head = *f; 1969 add_head->state = I40E_FILTER_NEW; 1970 add_head->vlan = I40E_VLAN_ANY; 1971 1972 /* Move the replacement to the add list */ 1973 INIT_HLIST_NODE(&add_head->hlist); 1974 hlist_add_head(&add_head->hlist, 1975 &tmp_add_list); 1976 1977 /* Move the original to the delete list */ 1978 f->state = I40E_FILTER_REMOVE; 1979 hash_del(&f->hlist); 1980 hlist_add_head(&f->hlist, &tmp_del_list); 1981 vsi->active_filters--; 1982 } 1983 1984 /* Also update any filters on the tmp_add list */ 1985 hlist_for_each_entry(f, &tmp_add_list, hlist) { 1986 if (!f->vlan) 1987 f->vlan = I40E_VLAN_ANY; 1988 } 1989 add_head = NULL; 1990 } 1991 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1992 } 1993 1994 /* Now process 'del_list' outside the lock */ 1995 if (!hlist_empty(&tmp_del_list)) { 1996 filter_list_len = hw->aq.asq_buf_size / 1997 sizeof(struct i40e_aqc_remove_macvlan_element_data); 1998 list_size = filter_list_len * 1999 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2000 del_list = kzalloc(list_size, GFP_ATOMIC); 2001 if (!del_list) 2002 goto err_no_memory; 2003 2004 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2005 cmd_flags = 0; 2006 2007 /* add to delete list */ 2008 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2009 if (f->vlan == I40E_VLAN_ANY) { 2010 del_list[num_del].vlan_tag = 0; 2011 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2012 } else { 2013 del_list[num_del].vlan_tag = 2014 cpu_to_le16((u16)(f->vlan)); 2015 } 2016 2017 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2018 del_list[num_del].flags = cmd_flags; 2019 num_del++; 2020 2021 /* flush a full buffer */ 2022 if (num_del == filter_list_len) { 2023 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2024 num_del, &retval); 2025 memset(del_list, 0, list_size); 2026 num_del = 0; 2027 } 2028 /* Release memory for MAC filter entries which were 2029 * synced up with HW. 2030 */ 2031 hlist_del(&f->hlist); 2032 kfree(f); 2033 } 2034 2035 if (num_del) { 2036 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2037 num_del, &retval); 2038 } 2039 2040 kfree(del_list); 2041 del_list = NULL; 2042 } 2043 2044 /* After finishing notifying firmware of the deleted filters, update 2045 * the cached value of vsi->has_vlan_filter. Note that we are safe to 2046 * use just !!vlan_filters here because if we only have VLAN=0 (that 2047 * is, non_vlan_filters) these will all be converted to VLAN=-1 in the 2048 * logic above already so this value would still be correct. 2049 */ 2050 vsi->has_vlan_filter = !!vlan_filters; 2051 2052 if (!hlist_empty(&tmp_add_list)) { 2053 /* Do all the adds now. */ 2054 filter_list_len = hw->aq.asq_buf_size / 2055 sizeof(struct i40e_aqc_add_macvlan_element_data); 2056 list_size = filter_list_len * 2057 sizeof(struct i40e_aqc_add_macvlan_element_data); 2058 add_list = kzalloc(list_size, GFP_ATOMIC); 2059 if (!add_list) 2060 goto err_no_memory; 2061 2062 num_add = 0; 2063 hlist_for_each_entry(f, &tmp_add_list, hlist) { 2064 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2065 &vsi->state)) { 2066 f->state = I40E_FILTER_FAILED; 2067 continue; 2068 } 2069 /* add to add array */ 2070 if (num_add == 0) 2071 add_head = f; 2072 cmd_flags = 0; 2073 ether_addr_copy(add_list[num_add].mac_addr, f->macaddr); 2074 if (f->vlan == I40E_VLAN_ANY) { 2075 add_list[num_add].vlan_tag = 0; 2076 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2077 } else { 2078 add_list[num_add].vlan_tag = 2079 cpu_to_le16((u16)(f->vlan)); 2080 } 2081 add_list[num_add].queue_number = 0; 2082 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2083 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2084 num_add++; 2085 2086 /* flush a full buffer */ 2087 if (num_add == filter_list_len) { 2088 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2089 add_head, num_add, 2090 &promisc_changed); 2091 memset(add_list, 0, list_size); 2092 num_add = 0; 2093 } 2094 } 2095 if (num_add) { 2096 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2097 num_add, &promisc_changed); 2098 } 2099 /* Now move all of the filters from the temp add list back to 2100 * the VSI's list. 2101 */ 2102 spin_lock_bh(&vsi->mac_filter_hash_lock); 2103 hlist_for_each_entry_safe(f, h, &tmp_add_list, hlist) { 2104 u64 key = i40e_addr_to_hkey(f->macaddr); 2105 2106 hlist_del(&f->hlist); 2107 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2108 } 2109 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2110 kfree(add_list); 2111 add_list = NULL; 2112 } 2113 2114 /* Check to see if we can drop out of overflow promiscuous mode. */ 2115 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state) && 2116 (vsi->active_filters < vsi->promisc_threshold)) { 2117 int failed_count = 0; 2118 /* See if we have any failed filters. We can't drop out of 2119 * promiscuous until these have all been deleted. 2120 */ 2121 spin_lock_bh(&vsi->mac_filter_hash_lock); 2122 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2123 if (f->state == I40E_FILTER_FAILED) 2124 failed_count++; 2125 } 2126 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2127 if (!failed_count) { 2128 dev_info(&pf->pdev->dev, 2129 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2130 vsi_name); 2131 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2132 promisc_changed = true; 2133 vsi->promisc_threshold = 0; 2134 } 2135 } 2136 2137 /* if the VF is not trusted do not do promisc */ 2138 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2139 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2140 goto out; 2141 } 2142 2143 /* check for changes in promiscuous modes */ 2144 if (changed_flags & IFF_ALLMULTI) { 2145 bool cur_multipromisc; 2146 2147 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2148 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2149 vsi->seid, 2150 cur_multipromisc, 2151 NULL); 2152 if (aq_ret) { 2153 retval = i40e_aq_rc_to_posix(aq_ret, 2154 hw->aq.asq_last_status); 2155 dev_info(&pf->pdev->dev, 2156 "set multi promisc failed on %s, err %s aq_err %s\n", 2157 vsi_name, 2158 i40e_stat_str(hw, aq_ret), 2159 i40e_aq_str(hw, hw->aq.asq_last_status)); 2160 } 2161 } 2162 if ((changed_flags & IFF_PROMISC) || 2163 (promisc_changed && 2164 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))) { 2165 bool cur_promisc; 2166 2167 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2168 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2169 &vsi->state)); 2170 if ((vsi->type == I40E_VSI_MAIN) && 2171 (pf->lan_veb != I40E_NO_VEB) && 2172 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2173 /* set defport ON for Main VSI instead of true promisc 2174 * this way we will get all unicast/multicast and VLAN 2175 * promisc behavior but will not get VF or VMDq traffic 2176 * replicated on the Main VSI. 2177 */ 2178 if (pf->cur_promisc != cur_promisc) { 2179 pf->cur_promisc = cur_promisc; 2180 if (cur_promisc) 2181 aq_ret = 2182 i40e_aq_set_default_vsi(hw, 2183 vsi->seid, 2184 NULL); 2185 else 2186 aq_ret = 2187 i40e_aq_clear_default_vsi(hw, 2188 vsi->seid, 2189 NULL); 2190 if (aq_ret) { 2191 retval = i40e_aq_rc_to_posix(aq_ret, 2192 hw->aq.asq_last_status); 2193 dev_info(&pf->pdev->dev, 2194 "Set default VSI failed on %s, err %s, aq_err %s\n", 2195 vsi_name, 2196 i40e_stat_str(hw, aq_ret), 2197 i40e_aq_str(hw, 2198 hw->aq.asq_last_status)); 2199 } 2200 } 2201 } else { 2202 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2203 hw, 2204 vsi->seid, 2205 cur_promisc, NULL, 2206 true); 2207 if (aq_ret) { 2208 retval = 2209 i40e_aq_rc_to_posix(aq_ret, 2210 hw->aq.asq_last_status); 2211 dev_info(&pf->pdev->dev, 2212 "set unicast promisc failed on %s, err %s, aq_err %s\n", 2213 vsi_name, 2214 i40e_stat_str(hw, aq_ret), 2215 i40e_aq_str(hw, 2216 hw->aq.asq_last_status)); 2217 } 2218 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2219 hw, 2220 vsi->seid, 2221 cur_promisc, NULL); 2222 if (aq_ret) { 2223 retval = 2224 i40e_aq_rc_to_posix(aq_ret, 2225 hw->aq.asq_last_status); 2226 dev_info(&pf->pdev->dev, 2227 "set multicast promisc failed on %s, err %s, aq_err %s\n", 2228 vsi_name, 2229 i40e_stat_str(hw, aq_ret), 2230 i40e_aq_str(hw, 2231 hw->aq.asq_last_status)); 2232 } 2233 } 2234 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 2235 vsi->seid, 2236 cur_promisc, NULL); 2237 if (aq_ret) { 2238 retval = i40e_aq_rc_to_posix(aq_ret, 2239 pf->hw.aq.asq_last_status); 2240 dev_info(&pf->pdev->dev, 2241 "set brdcast promisc failed, err %s, aq_err %s\n", 2242 i40e_stat_str(hw, aq_ret), 2243 i40e_aq_str(hw, 2244 hw->aq.asq_last_status)); 2245 } 2246 } 2247 out: 2248 /* if something went wrong then set the changed flag so we try again */ 2249 if (retval) 2250 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2251 2252 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2253 return retval; 2254 2255 err_no_memory: 2256 /* Restore elements on the temporary add and delete lists */ 2257 spin_lock_bh(&vsi->mac_filter_hash_lock); 2258 err_no_memory_locked: 2259 i40e_undo_filter_entries(vsi, &tmp_del_list); 2260 i40e_undo_filter_entries(vsi, &tmp_add_list); 2261 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2262 2263 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2264 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2265 return -ENOMEM; 2266 } 2267 2268 /** 2269 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2270 * @pf: board private structure 2271 **/ 2272 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2273 { 2274 int v; 2275 2276 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2277 return; 2278 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2279 2280 for (v = 0; v < pf->num_alloc_vsi; v++) { 2281 if (pf->vsi[v] && 2282 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2283 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2284 2285 if (ret) { 2286 /* come back and try again later */ 2287 pf->flags |= I40E_FLAG_FILTER_SYNC; 2288 break; 2289 } 2290 } 2291 } 2292 } 2293 2294 /** 2295 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2296 * @netdev: network interface device structure 2297 * @new_mtu: new value for maximum frame size 2298 * 2299 * Returns 0 on success, negative on failure 2300 **/ 2301 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2302 { 2303 struct i40e_netdev_priv *np = netdev_priv(netdev); 2304 struct i40e_vsi *vsi = np->vsi; 2305 2306 netdev_info(netdev, "changing MTU from %d to %d\n", 2307 netdev->mtu, new_mtu); 2308 netdev->mtu = new_mtu; 2309 if (netif_running(netdev)) 2310 i40e_vsi_reinit_locked(vsi); 2311 i40e_notify_client_of_l2_param_changes(vsi); 2312 return 0; 2313 } 2314 2315 /** 2316 * i40e_ioctl - Access the hwtstamp interface 2317 * @netdev: network interface device structure 2318 * @ifr: interface request data 2319 * @cmd: ioctl command 2320 **/ 2321 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2322 { 2323 struct i40e_netdev_priv *np = netdev_priv(netdev); 2324 struct i40e_pf *pf = np->vsi->back; 2325 2326 switch (cmd) { 2327 case SIOCGHWTSTAMP: 2328 return i40e_ptp_get_ts_config(pf, ifr); 2329 case SIOCSHWTSTAMP: 2330 return i40e_ptp_set_ts_config(pf, ifr); 2331 default: 2332 return -EOPNOTSUPP; 2333 } 2334 } 2335 2336 /** 2337 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2338 * @vsi: the vsi being adjusted 2339 **/ 2340 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2341 { 2342 struct i40e_vsi_context ctxt; 2343 i40e_status ret; 2344 2345 if ((vsi->info.valid_sections & 2346 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2347 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2348 return; /* already enabled */ 2349 2350 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2351 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2352 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2353 2354 ctxt.seid = vsi->seid; 2355 ctxt.info = vsi->info; 2356 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2357 if (ret) { 2358 dev_info(&vsi->back->pdev->dev, 2359 "update vlan stripping failed, err %s aq_err %s\n", 2360 i40e_stat_str(&vsi->back->hw, ret), 2361 i40e_aq_str(&vsi->back->hw, 2362 vsi->back->hw.aq.asq_last_status)); 2363 } 2364 } 2365 2366 /** 2367 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2368 * @vsi: the vsi being adjusted 2369 **/ 2370 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2371 { 2372 struct i40e_vsi_context ctxt; 2373 i40e_status ret; 2374 2375 if ((vsi->info.valid_sections & 2376 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2377 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2378 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2379 return; /* already disabled */ 2380 2381 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2382 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2383 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2384 2385 ctxt.seid = vsi->seid; 2386 ctxt.info = vsi->info; 2387 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2388 if (ret) { 2389 dev_info(&vsi->back->pdev->dev, 2390 "update vlan stripping failed, err %s aq_err %s\n", 2391 i40e_stat_str(&vsi->back->hw, ret), 2392 i40e_aq_str(&vsi->back->hw, 2393 vsi->back->hw.aq.asq_last_status)); 2394 } 2395 } 2396 2397 /** 2398 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2399 * @netdev: network interface to be adjusted 2400 * @features: netdev features to test if VLAN offload is enabled or not 2401 **/ 2402 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2403 { 2404 struct i40e_netdev_priv *np = netdev_priv(netdev); 2405 struct i40e_vsi *vsi = np->vsi; 2406 2407 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2408 i40e_vlan_stripping_enable(vsi); 2409 else 2410 i40e_vlan_stripping_disable(vsi); 2411 } 2412 2413 /** 2414 * i40e_vsi_add_vlan - Add vsi membership for given vlan 2415 * @vsi: the vsi being configured 2416 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2417 **/ 2418 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid) 2419 { 2420 struct i40e_mac_filter *f, *add_f, *del_f; 2421 struct hlist_node *h; 2422 int bkt; 2423 2424 /* Locked once because all functions invoked below iterates list*/ 2425 spin_lock_bh(&vsi->mac_filter_hash_lock); 2426 2427 if (vsi->netdev) { 2428 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid); 2429 if (!add_f) { 2430 dev_info(&vsi->back->pdev->dev, 2431 "Could not add vlan filter %d for %pM\n", 2432 vid, vsi->netdev->dev_addr); 2433 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2434 return -ENOMEM; 2435 } 2436 } 2437 2438 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2439 if (f->state == I40E_FILTER_REMOVE) 2440 continue; 2441 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2442 if (!add_f) { 2443 dev_info(&vsi->back->pdev->dev, 2444 "Could not add vlan filter %d for %pM\n", 2445 vid, f->macaddr); 2446 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2447 return -ENOMEM; 2448 } 2449 } 2450 2451 /* Now if we add a vlan tag, make sure to check if it is the first 2452 * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag" 2453 * with 0, so we now accept untagged and specified tagged traffic 2454 * (and not all tags along with untagged) 2455 */ 2456 if (vid > 0 && vsi->netdev) { 2457 del_f = i40e_find_filter(vsi, vsi->netdev->dev_addr, 2458 I40E_VLAN_ANY); 2459 if (del_f) { 2460 __i40e_del_filter(vsi, del_f); 2461 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0); 2462 if (!add_f) { 2463 dev_info(&vsi->back->pdev->dev, 2464 "Could not add filter 0 for %pM\n", 2465 vsi->netdev->dev_addr); 2466 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2467 return -ENOMEM; 2468 } 2469 } 2470 } 2471 2472 /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */ 2473 if (vid > 0 && !vsi->info.pvid) { 2474 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2475 if (f->state == I40E_FILTER_REMOVE) 2476 continue; 2477 del_f = i40e_find_filter(vsi, f->macaddr, 2478 I40E_VLAN_ANY); 2479 if (!del_f) 2480 continue; 2481 __i40e_del_filter(vsi, del_f); 2482 add_f = i40e_add_filter(vsi, f->macaddr, 0); 2483 if (!add_f) { 2484 dev_info(&vsi->back->pdev->dev, 2485 "Could not add filter 0 for %pM\n", 2486 f->macaddr); 2487 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2488 return -ENOMEM; 2489 } 2490 } 2491 } 2492 2493 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2494 2495 /* schedule our worker thread which will take care of 2496 * applying the new filter changes 2497 */ 2498 i40e_service_event_schedule(vsi->back); 2499 return 0; 2500 } 2501 2502 /** 2503 * i40e_vsi_kill_vlan - Remove vsi membership for given vlan 2504 * @vsi: the vsi being configured 2505 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2506 **/ 2507 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid) 2508 { 2509 struct net_device *netdev = vsi->netdev; 2510 struct i40e_mac_filter *f; 2511 struct hlist_node *h; 2512 int bkt; 2513 2514 /* Locked once because all functions invoked below iterates list */ 2515 spin_lock_bh(&vsi->mac_filter_hash_lock); 2516 2517 if (vsi->netdev) 2518 i40e_del_filter(vsi, netdev->dev_addr, vid); 2519 2520 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2521 if (f->vlan == vid) 2522 __i40e_del_filter(vsi, f); 2523 } 2524 2525 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2526 2527 /* schedule our worker thread which will take care of 2528 * applying the new filter changes 2529 */ 2530 i40e_service_event_schedule(vsi->back); 2531 } 2532 2533 /** 2534 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2535 * @netdev: network interface to be adjusted 2536 * @vid: vlan id to be added 2537 * 2538 * net_device_ops implementation for adding vlan ids 2539 **/ 2540 #ifdef I40E_FCOE 2541 int i40e_vlan_rx_add_vid(struct net_device *netdev, 2542 __always_unused __be16 proto, u16 vid) 2543 #else 2544 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2545 __always_unused __be16 proto, u16 vid) 2546 #endif 2547 { 2548 struct i40e_netdev_priv *np = netdev_priv(netdev); 2549 struct i40e_vsi *vsi = np->vsi; 2550 int ret = 0; 2551 2552 if (vid > 4095) 2553 return -EINVAL; 2554 2555 /* If the network stack called us with vid = 0 then 2556 * it is asking to receive priority tagged packets with 2557 * vlan id 0. Our HW receives them by default when configured 2558 * to receive untagged packets so there is no need to add an 2559 * extra filter for vlan 0 tagged packets. 2560 */ 2561 if (vid) 2562 ret = i40e_vsi_add_vlan(vsi, vid); 2563 2564 if (!ret && (vid < VLAN_N_VID)) 2565 set_bit(vid, vsi->active_vlans); 2566 2567 return ret; 2568 } 2569 2570 /** 2571 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2572 * @netdev: network interface to be adjusted 2573 * @vid: vlan id to be removed 2574 * 2575 * net_device_ops implementation for removing vlan ids 2576 **/ 2577 #ifdef I40E_FCOE 2578 int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2579 __always_unused __be16 proto, u16 vid) 2580 #else 2581 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2582 __always_unused __be16 proto, u16 vid) 2583 #endif 2584 { 2585 struct i40e_netdev_priv *np = netdev_priv(netdev); 2586 struct i40e_vsi *vsi = np->vsi; 2587 2588 /* return code is ignored as there is nothing a user 2589 * can do about failure to remove and a log message was 2590 * already printed from the other function 2591 */ 2592 i40e_vsi_kill_vlan(vsi, vid); 2593 2594 clear_bit(vid, vsi->active_vlans); 2595 2596 return 0; 2597 } 2598 2599 /** 2600 * i40e_macaddr_init - explicitly write the mac address filters 2601 * 2602 * @vsi: pointer to the vsi 2603 * @macaddr: the MAC address 2604 * 2605 * This is needed when the macaddr has been obtained by other 2606 * means than the default, e.g., from Open Firmware or IDPROM. 2607 * Returns 0 on success, negative on failure 2608 **/ 2609 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr) 2610 { 2611 int ret; 2612 struct i40e_aqc_add_macvlan_element_data element; 2613 2614 ret = i40e_aq_mac_address_write(&vsi->back->hw, 2615 I40E_AQC_WRITE_TYPE_LAA_WOL, 2616 macaddr, NULL); 2617 if (ret) { 2618 dev_info(&vsi->back->pdev->dev, 2619 "Addr change for VSI failed: %d\n", ret); 2620 return -EADDRNOTAVAIL; 2621 } 2622 2623 memset(&element, 0, sizeof(element)); 2624 ether_addr_copy(element.mac_addr, macaddr); 2625 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH); 2626 ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL); 2627 if (ret) { 2628 dev_info(&vsi->back->pdev->dev, 2629 "add filter failed err %s aq_err %s\n", 2630 i40e_stat_str(&vsi->back->hw, ret), 2631 i40e_aq_str(&vsi->back->hw, 2632 vsi->back->hw.aq.asq_last_status)); 2633 } 2634 return ret; 2635 } 2636 2637 /** 2638 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2639 * @vsi: the vsi being brought back up 2640 **/ 2641 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2642 { 2643 u16 vid; 2644 2645 if (!vsi->netdev) 2646 return; 2647 2648 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2649 2650 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2651 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2652 vid); 2653 } 2654 2655 /** 2656 * i40e_vsi_add_pvid - Add pvid for the VSI 2657 * @vsi: the vsi being adjusted 2658 * @vid: the vlan id to set as a PVID 2659 **/ 2660 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2661 { 2662 struct i40e_vsi_context ctxt; 2663 i40e_status ret; 2664 2665 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2666 vsi->info.pvid = cpu_to_le16(vid); 2667 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2668 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2669 I40E_AQ_VSI_PVLAN_EMOD_STR; 2670 2671 ctxt.seid = vsi->seid; 2672 ctxt.info = vsi->info; 2673 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2674 if (ret) { 2675 dev_info(&vsi->back->pdev->dev, 2676 "add pvid failed, err %s aq_err %s\n", 2677 i40e_stat_str(&vsi->back->hw, ret), 2678 i40e_aq_str(&vsi->back->hw, 2679 vsi->back->hw.aq.asq_last_status)); 2680 return -ENOENT; 2681 } 2682 2683 return 0; 2684 } 2685 2686 /** 2687 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2688 * @vsi: the vsi being adjusted 2689 * 2690 * Just use the vlan_rx_register() service to put it back to normal 2691 **/ 2692 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2693 { 2694 i40e_vlan_stripping_disable(vsi); 2695 2696 vsi->info.pvid = 0; 2697 } 2698 2699 /** 2700 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2701 * @vsi: ptr to the VSI 2702 * 2703 * If this function returns with an error, then it's possible one or 2704 * more of the rings is populated (while the rest are not). It is the 2705 * callers duty to clean those orphaned rings. 2706 * 2707 * Return 0 on success, negative on failure 2708 **/ 2709 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2710 { 2711 int i, err = 0; 2712 2713 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2714 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2715 2716 return err; 2717 } 2718 2719 /** 2720 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2721 * @vsi: ptr to the VSI 2722 * 2723 * Free VSI's transmit software resources 2724 **/ 2725 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2726 { 2727 int i; 2728 2729 if (!vsi->tx_rings) 2730 return; 2731 2732 for (i = 0; i < vsi->num_queue_pairs; i++) 2733 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2734 i40e_free_tx_resources(vsi->tx_rings[i]); 2735 } 2736 2737 /** 2738 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2739 * @vsi: ptr to the VSI 2740 * 2741 * If this function returns with an error, then it's possible one or 2742 * more of the rings is populated (while the rest are not). It is the 2743 * callers duty to clean those orphaned rings. 2744 * 2745 * Return 0 on success, negative on failure 2746 **/ 2747 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2748 { 2749 int i, err = 0; 2750 2751 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2752 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2753 #ifdef I40E_FCOE 2754 i40e_fcoe_setup_ddp_resources(vsi); 2755 #endif 2756 return err; 2757 } 2758 2759 /** 2760 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2761 * @vsi: ptr to the VSI 2762 * 2763 * Free all receive software resources 2764 **/ 2765 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2766 { 2767 int i; 2768 2769 if (!vsi->rx_rings) 2770 return; 2771 2772 for (i = 0; i < vsi->num_queue_pairs; i++) 2773 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2774 i40e_free_rx_resources(vsi->rx_rings[i]); 2775 #ifdef I40E_FCOE 2776 i40e_fcoe_free_ddp_resources(vsi); 2777 #endif 2778 } 2779 2780 /** 2781 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 2782 * @ring: The Tx ring to configure 2783 * 2784 * This enables/disables XPS for a given Tx descriptor ring 2785 * based on the TCs enabled for the VSI that ring belongs to. 2786 **/ 2787 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 2788 { 2789 struct i40e_vsi *vsi = ring->vsi; 2790 cpumask_var_t mask; 2791 2792 if (!ring->q_vector || !ring->netdev) 2793 return; 2794 2795 /* Single TC mode enable XPS */ 2796 if (vsi->tc_config.numtc <= 1) { 2797 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) 2798 netif_set_xps_queue(ring->netdev, 2799 &ring->q_vector->affinity_mask, 2800 ring->queue_index); 2801 } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) { 2802 /* Disable XPS to allow selection based on TC */ 2803 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits); 2804 netif_set_xps_queue(ring->netdev, mask, ring->queue_index); 2805 free_cpumask_var(mask); 2806 } 2807 2808 /* schedule our worker thread which will take care of 2809 * applying the new filter changes 2810 */ 2811 i40e_service_event_schedule(vsi->back); 2812 } 2813 2814 /** 2815 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2816 * @ring: The Tx ring to configure 2817 * 2818 * Configure the Tx descriptor ring in the HMC context. 2819 **/ 2820 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2821 { 2822 struct i40e_vsi *vsi = ring->vsi; 2823 u16 pf_q = vsi->base_queue + ring->queue_index; 2824 struct i40e_hw *hw = &vsi->back->hw; 2825 struct i40e_hmc_obj_txq tx_ctx; 2826 i40e_status err = 0; 2827 u32 qtx_ctl = 0; 2828 2829 /* some ATR related tx ring init */ 2830 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2831 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2832 ring->atr_count = 0; 2833 } else { 2834 ring->atr_sample_rate = 0; 2835 } 2836 2837 /* configure XPS */ 2838 i40e_config_xps_tx_ring(ring); 2839 2840 /* clear the context structure first */ 2841 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2842 2843 tx_ctx.new_context = 1; 2844 tx_ctx.base = (ring->dma / 128); 2845 tx_ctx.qlen = ring->count; 2846 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2847 I40E_FLAG_FD_ATR_ENABLED)); 2848 #ifdef I40E_FCOE 2849 tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE); 2850 #endif 2851 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2852 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2853 if (vsi->type != I40E_VSI_FDIR) 2854 tx_ctx.head_wb_ena = 1; 2855 tx_ctx.head_wb_addr = ring->dma + 2856 (ring->count * sizeof(struct i40e_tx_desc)); 2857 2858 /* As part of VSI creation/update, FW allocates certain 2859 * Tx arbitration queue sets for each TC enabled for 2860 * the VSI. The FW returns the handles to these queue 2861 * sets as part of the response buffer to Add VSI, 2862 * Update VSI, etc. AQ commands. It is expected that 2863 * these queue set handles be associated with the Tx 2864 * queues by the driver as part of the TX queue context 2865 * initialization. This has to be done regardless of 2866 * DCB as by default everything is mapped to TC0. 2867 */ 2868 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2869 tx_ctx.rdylist_act = 0; 2870 2871 /* clear the context in the HMC */ 2872 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2873 if (err) { 2874 dev_info(&vsi->back->pdev->dev, 2875 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2876 ring->queue_index, pf_q, err); 2877 return -ENOMEM; 2878 } 2879 2880 /* set the context in the HMC */ 2881 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2882 if (err) { 2883 dev_info(&vsi->back->pdev->dev, 2884 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2885 ring->queue_index, pf_q, err); 2886 return -ENOMEM; 2887 } 2888 2889 /* Now associate this queue with this PCI function */ 2890 if (vsi->type == I40E_VSI_VMDQ2) { 2891 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2892 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 2893 I40E_QTX_CTL_VFVM_INDX_MASK; 2894 } else { 2895 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2896 } 2897 2898 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2899 I40E_QTX_CTL_PF_INDX_MASK); 2900 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2901 i40e_flush(hw); 2902 2903 /* cache tail off for easier writes later */ 2904 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2905 2906 return 0; 2907 } 2908 2909 /** 2910 * i40e_configure_rx_ring - Configure a receive ring context 2911 * @ring: The Rx ring to configure 2912 * 2913 * Configure the Rx descriptor ring in the HMC context. 2914 **/ 2915 static int i40e_configure_rx_ring(struct i40e_ring *ring) 2916 { 2917 struct i40e_vsi *vsi = ring->vsi; 2918 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 2919 u16 pf_q = vsi->base_queue + ring->queue_index; 2920 struct i40e_hw *hw = &vsi->back->hw; 2921 struct i40e_hmc_obj_rxq rx_ctx; 2922 i40e_status err = 0; 2923 2924 ring->state = 0; 2925 2926 /* clear the context structure first */ 2927 memset(&rx_ctx, 0, sizeof(rx_ctx)); 2928 2929 ring->rx_buf_len = vsi->rx_buf_len; 2930 2931 rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT; 2932 2933 rx_ctx.base = (ring->dma / 128); 2934 rx_ctx.qlen = ring->count; 2935 2936 /* use 32 byte descriptors */ 2937 rx_ctx.dsize = 1; 2938 2939 /* descriptor type is always zero 2940 * rx_ctx.dtype = 0; 2941 */ 2942 rx_ctx.hsplit_0 = 0; 2943 2944 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 2945 if (hw->revision_id == 0) 2946 rx_ctx.lrxqthresh = 0; 2947 else 2948 rx_ctx.lrxqthresh = 2; 2949 rx_ctx.crcstrip = 1; 2950 rx_ctx.l2tsel = 1; 2951 /* this controls whether VLAN is stripped from inner headers */ 2952 rx_ctx.showiv = 0; 2953 #ifdef I40E_FCOE 2954 rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE); 2955 #endif 2956 /* set the prefena field to 1 because the manual says to */ 2957 rx_ctx.prefena = 1; 2958 2959 /* clear the context in the HMC */ 2960 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 2961 if (err) { 2962 dev_info(&vsi->back->pdev->dev, 2963 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 2964 ring->queue_index, pf_q, err); 2965 return -ENOMEM; 2966 } 2967 2968 /* set the context in the HMC */ 2969 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 2970 if (err) { 2971 dev_info(&vsi->back->pdev->dev, 2972 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 2973 ring->queue_index, pf_q, err); 2974 return -ENOMEM; 2975 } 2976 2977 /* cache tail for quicker writes, and clear the reg before use */ 2978 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 2979 writel(0, ring->tail); 2980 2981 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 2982 2983 return 0; 2984 } 2985 2986 /** 2987 * i40e_vsi_configure_tx - Configure the VSI for Tx 2988 * @vsi: VSI structure describing this set of rings and resources 2989 * 2990 * Configure the Tx VSI for operation. 2991 **/ 2992 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 2993 { 2994 int err = 0; 2995 u16 i; 2996 2997 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 2998 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 2999 3000 return err; 3001 } 3002 3003 /** 3004 * i40e_vsi_configure_rx - Configure the VSI for Rx 3005 * @vsi: the VSI being configured 3006 * 3007 * Configure the Rx VSI for operation. 3008 **/ 3009 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3010 { 3011 int err = 0; 3012 u16 i; 3013 3014 if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN)) 3015 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN 3016 + ETH_FCS_LEN + VLAN_HLEN; 3017 else 3018 vsi->max_frame = I40E_RXBUFFER_2048; 3019 3020 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3021 3022 #ifdef I40E_FCOE 3023 /* setup rx buffer for FCoE */ 3024 if ((vsi->type == I40E_VSI_FCOE) && 3025 (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) { 3026 vsi->rx_buf_len = I40E_RXBUFFER_3072; 3027 vsi->max_frame = I40E_RXBUFFER_3072; 3028 } 3029 3030 #endif /* I40E_FCOE */ 3031 /* round up for the chip's needs */ 3032 vsi->rx_buf_len = ALIGN(vsi->rx_buf_len, 3033 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3034 3035 /* set up individual rings */ 3036 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3037 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3038 3039 return err; 3040 } 3041 3042 /** 3043 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3044 * @vsi: ptr to the VSI 3045 **/ 3046 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3047 { 3048 struct i40e_ring *tx_ring, *rx_ring; 3049 u16 qoffset, qcount; 3050 int i, n; 3051 3052 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3053 /* Reset the TC information */ 3054 for (i = 0; i < vsi->num_queue_pairs; i++) { 3055 rx_ring = vsi->rx_rings[i]; 3056 tx_ring = vsi->tx_rings[i]; 3057 rx_ring->dcb_tc = 0; 3058 tx_ring->dcb_tc = 0; 3059 } 3060 } 3061 3062 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3063 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3064 continue; 3065 3066 qoffset = vsi->tc_config.tc_info[n].qoffset; 3067 qcount = vsi->tc_config.tc_info[n].qcount; 3068 for (i = qoffset; i < (qoffset + qcount); i++) { 3069 rx_ring = vsi->rx_rings[i]; 3070 tx_ring = vsi->tx_rings[i]; 3071 rx_ring->dcb_tc = n; 3072 tx_ring->dcb_tc = n; 3073 } 3074 } 3075 } 3076 3077 /** 3078 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3079 * @vsi: ptr to the VSI 3080 **/ 3081 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3082 { 3083 struct i40e_pf *pf = vsi->back; 3084 int err; 3085 3086 if (vsi->netdev) 3087 i40e_set_rx_mode(vsi->netdev); 3088 3089 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 3090 err = i40e_macaddr_init(vsi, pf->hw.mac.addr); 3091 if (err) { 3092 dev_warn(&pf->pdev->dev, 3093 "could not set up macaddr; err %d\n", err); 3094 } 3095 } 3096 } 3097 3098 /** 3099 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3100 * @vsi: Pointer to the targeted VSI 3101 * 3102 * This function replays the hlist on the hw where all the SB Flow Director 3103 * filters were saved. 3104 **/ 3105 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3106 { 3107 struct i40e_fdir_filter *filter; 3108 struct i40e_pf *pf = vsi->back; 3109 struct hlist_node *node; 3110 3111 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3112 return; 3113 3114 hlist_for_each_entry_safe(filter, node, 3115 &pf->fdir_filter_list, fdir_node) { 3116 i40e_add_del_fdir(vsi, filter, true); 3117 } 3118 } 3119 3120 /** 3121 * i40e_vsi_configure - Set up the VSI for action 3122 * @vsi: the VSI being configured 3123 **/ 3124 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3125 { 3126 int err; 3127 3128 i40e_set_vsi_rx_mode(vsi); 3129 i40e_restore_vlan(vsi); 3130 i40e_vsi_config_dcb_rings(vsi); 3131 err = i40e_vsi_configure_tx(vsi); 3132 if (!err) 3133 err = i40e_vsi_configure_rx(vsi); 3134 3135 return err; 3136 } 3137 3138 /** 3139 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3140 * @vsi: the VSI being configured 3141 **/ 3142 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3143 { 3144 struct i40e_pf *pf = vsi->back; 3145 struct i40e_hw *hw = &pf->hw; 3146 u16 vector; 3147 int i, q; 3148 u32 qp; 3149 3150 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3151 * and PFINT_LNKLSTn registers, e.g.: 3152 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3153 */ 3154 qp = vsi->base_queue; 3155 vector = vsi->base_vector; 3156 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3157 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3158 3159 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3160 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3161 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3162 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3163 q_vector->rx.itr); 3164 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3165 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3166 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3167 q_vector->tx.itr); 3168 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3169 INTRL_USEC_TO_REG(vsi->int_rate_limit)); 3170 3171 /* Linked list for the queuepairs assigned to this vector */ 3172 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3173 for (q = 0; q < q_vector->num_ringpairs; q++) { 3174 u32 val; 3175 3176 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3177 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3178 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3179 (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3180 (I40E_QUEUE_TYPE_TX 3181 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3182 3183 wr32(hw, I40E_QINT_RQCTL(qp), val); 3184 3185 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3186 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3187 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3188 ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| 3189 (I40E_QUEUE_TYPE_RX 3190 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3191 3192 /* Terminate the linked list */ 3193 if (q == (q_vector->num_ringpairs - 1)) 3194 val |= (I40E_QUEUE_END_OF_LIST 3195 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3196 3197 wr32(hw, I40E_QINT_TQCTL(qp), val); 3198 qp++; 3199 } 3200 } 3201 3202 i40e_flush(hw); 3203 } 3204 3205 /** 3206 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3207 * @hw: ptr to the hardware info 3208 **/ 3209 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3210 { 3211 struct i40e_hw *hw = &pf->hw; 3212 u32 val; 3213 3214 /* clear things first */ 3215 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3216 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3217 3218 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3219 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3220 I40E_PFINT_ICR0_ENA_GRST_MASK | 3221 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3222 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3223 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3224 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3225 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3226 3227 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3228 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3229 3230 if (pf->flags & I40E_FLAG_PTP) 3231 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3232 3233 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3234 3235 /* SW_ITR_IDX = 0, but don't change INTENA */ 3236 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3237 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3238 3239 /* OTHER_ITR_IDX = 0 */ 3240 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3241 } 3242 3243 /** 3244 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3245 * @vsi: the VSI being configured 3246 **/ 3247 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3248 { 3249 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3250 struct i40e_pf *pf = vsi->back; 3251 struct i40e_hw *hw = &pf->hw; 3252 u32 val; 3253 3254 /* set the ITR configuration */ 3255 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3256 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3257 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3258 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3259 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3260 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3261 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3262 3263 i40e_enable_misc_int_causes(pf); 3264 3265 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3266 wr32(hw, I40E_PFINT_LNKLST0, 0); 3267 3268 /* Associate the queue pair to the vector and enable the queue int */ 3269 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3270 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3271 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3272 3273 wr32(hw, I40E_QINT_RQCTL(0), val); 3274 3275 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3276 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3277 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3278 3279 wr32(hw, I40E_QINT_TQCTL(0), val); 3280 i40e_flush(hw); 3281 } 3282 3283 /** 3284 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3285 * @pf: board private structure 3286 **/ 3287 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3288 { 3289 struct i40e_hw *hw = &pf->hw; 3290 3291 wr32(hw, I40E_PFINT_DYN_CTL0, 3292 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3293 i40e_flush(hw); 3294 } 3295 3296 /** 3297 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3298 * @pf: board private structure 3299 * @clearpba: true when all pending interrupt events should be cleared 3300 **/ 3301 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba) 3302 { 3303 struct i40e_hw *hw = &pf->hw; 3304 u32 val; 3305 3306 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3307 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) | 3308 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3309 3310 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3311 i40e_flush(hw); 3312 } 3313 3314 /** 3315 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3316 * @irq: interrupt number 3317 * @data: pointer to a q_vector 3318 **/ 3319 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3320 { 3321 struct i40e_q_vector *q_vector = data; 3322 3323 if (!q_vector->tx.ring && !q_vector->rx.ring) 3324 return IRQ_HANDLED; 3325 3326 napi_schedule_irqoff(&q_vector->napi); 3327 3328 return IRQ_HANDLED; 3329 } 3330 3331 /** 3332 * i40e_irq_affinity_notify - Callback for affinity changes 3333 * @notify: context as to what irq was changed 3334 * @mask: the new affinity mask 3335 * 3336 * This is a callback function used by the irq_set_affinity_notifier function 3337 * so that we may register to receive changes to the irq affinity masks. 3338 **/ 3339 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3340 const cpumask_t *mask) 3341 { 3342 struct i40e_q_vector *q_vector = 3343 container_of(notify, struct i40e_q_vector, affinity_notify); 3344 3345 q_vector->affinity_mask = *mask; 3346 } 3347 3348 /** 3349 * i40e_irq_affinity_release - Callback for affinity notifier release 3350 * @ref: internal core kernel usage 3351 * 3352 * This is a callback function used by the irq_set_affinity_notifier function 3353 * to inform the current notification subscriber that they will no longer 3354 * receive notifications. 3355 **/ 3356 static void i40e_irq_affinity_release(struct kref *ref) {} 3357 3358 /** 3359 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3360 * @vsi: the VSI being configured 3361 * @basename: name for the vector 3362 * 3363 * Allocates MSI-X vectors and requests interrupts from the kernel. 3364 **/ 3365 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3366 { 3367 int q_vectors = vsi->num_q_vectors; 3368 struct i40e_pf *pf = vsi->back; 3369 int base = vsi->base_vector; 3370 int rx_int_idx = 0; 3371 int tx_int_idx = 0; 3372 int vector, err; 3373 int irq_num; 3374 3375 for (vector = 0; vector < q_vectors; vector++) { 3376 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3377 3378 irq_num = pf->msix_entries[base + vector].vector; 3379 3380 if (q_vector->tx.ring && q_vector->rx.ring) { 3381 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3382 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3383 tx_int_idx++; 3384 } else if (q_vector->rx.ring) { 3385 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3386 "%s-%s-%d", basename, "rx", rx_int_idx++); 3387 } else if (q_vector->tx.ring) { 3388 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3389 "%s-%s-%d", basename, "tx", tx_int_idx++); 3390 } else { 3391 /* skip this unused q_vector */ 3392 continue; 3393 } 3394 err = request_irq(irq_num, 3395 vsi->irq_handler, 3396 0, 3397 q_vector->name, 3398 q_vector); 3399 if (err) { 3400 dev_info(&pf->pdev->dev, 3401 "MSIX request_irq failed, error: %d\n", err); 3402 goto free_queue_irqs; 3403 } 3404 3405 /* register for affinity change notifications */ 3406 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3407 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3408 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3409 /* assign the mask for this irq */ 3410 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 3411 } 3412 3413 vsi->irqs_ready = true; 3414 return 0; 3415 3416 free_queue_irqs: 3417 while (vector) { 3418 vector--; 3419 irq_num = pf->msix_entries[base + vector].vector; 3420 irq_set_affinity_notifier(irq_num, NULL); 3421 irq_set_affinity_hint(irq_num, NULL); 3422 free_irq(irq_num, &vsi->q_vectors[vector]); 3423 } 3424 return err; 3425 } 3426 3427 /** 3428 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3429 * @vsi: the VSI being un-configured 3430 **/ 3431 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3432 { 3433 struct i40e_pf *pf = vsi->back; 3434 struct i40e_hw *hw = &pf->hw; 3435 int base = vsi->base_vector; 3436 int i; 3437 3438 for (i = 0; i < vsi->num_queue_pairs; i++) { 3439 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); 3440 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); 3441 } 3442 3443 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3444 for (i = vsi->base_vector; 3445 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3446 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3447 3448 i40e_flush(hw); 3449 for (i = 0; i < vsi->num_q_vectors; i++) 3450 synchronize_irq(pf->msix_entries[i + base].vector); 3451 } else { 3452 /* Legacy and MSI mode - this stops all interrupt handling */ 3453 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3454 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3455 i40e_flush(hw); 3456 synchronize_irq(pf->pdev->irq); 3457 } 3458 } 3459 3460 /** 3461 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3462 * @vsi: the VSI being configured 3463 **/ 3464 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3465 { 3466 struct i40e_pf *pf = vsi->back; 3467 int i; 3468 3469 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3470 for (i = 0; i < vsi->num_q_vectors; i++) 3471 i40e_irq_dynamic_enable(vsi, i); 3472 } else { 3473 i40e_irq_dynamic_enable_icr0(pf, true); 3474 } 3475 3476 i40e_flush(&pf->hw); 3477 return 0; 3478 } 3479 3480 /** 3481 * i40e_stop_misc_vector - Stop the vector that handles non-queue events 3482 * @pf: board private structure 3483 **/ 3484 static void i40e_stop_misc_vector(struct i40e_pf *pf) 3485 { 3486 /* Disable ICR 0 */ 3487 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3488 i40e_flush(&pf->hw); 3489 } 3490 3491 /** 3492 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3493 * @irq: interrupt number 3494 * @data: pointer to a q_vector 3495 * 3496 * This is the handler used for all MSI/Legacy interrupts, and deals 3497 * with both queue and non-queue interrupts. This is also used in 3498 * MSIX mode to handle the non-queue interrupts. 3499 **/ 3500 static irqreturn_t i40e_intr(int irq, void *data) 3501 { 3502 struct i40e_pf *pf = (struct i40e_pf *)data; 3503 struct i40e_hw *hw = &pf->hw; 3504 irqreturn_t ret = IRQ_NONE; 3505 u32 icr0, icr0_remaining; 3506 u32 val, ena_mask; 3507 3508 icr0 = rd32(hw, I40E_PFINT_ICR0); 3509 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3510 3511 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3512 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3513 goto enable_intr; 3514 3515 /* if interrupt but no bits showing, must be SWINT */ 3516 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3517 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3518 pf->sw_int_count++; 3519 3520 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3521 (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3522 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3523 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3524 dev_info(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3525 } 3526 3527 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3528 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3529 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3530 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3531 3532 /* We do not have a way to disarm Queue causes while leaving 3533 * interrupt enabled for all other causes, ideally 3534 * interrupt should be disabled while we are in NAPI but 3535 * this is not a performance path and napi_schedule() 3536 * can deal with rescheduling. 3537 */ 3538 if (!test_bit(__I40E_DOWN, &pf->state)) 3539 napi_schedule_irqoff(&q_vector->napi); 3540 } 3541 3542 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3543 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3544 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 3545 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3546 } 3547 3548 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3549 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3550 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 3551 } 3552 3553 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3554 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3555 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state); 3556 } 3557 3558 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3559 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 3560 set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 3561 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3562 val = rd32(hw, I40E_GLGEN_RSTAT); 3563 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3564 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3565 if (val == I40E_RESET_CORER) { 3566 pf->corer_count++; 3567 } else if (val == I40E_RESET_GLOBR) { 3568 pf->globr_count++; 3569 } else if (val == I40E_RESET_EMPR) { 3570 pf->empr_count++; 3571 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state); 3572 } 3573 } 3574 3575 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3576 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3577 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3578 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3579 rd32(hw, I40E_PFHMC_ERRORINFO), 3580 rd32(hw, I40E_PFHMC_ERRORDATA)); 3581 } 3582 3583 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3584 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3585 3586 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3587 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3588 i40e_ptp_tx_hwtstamp(pf); 3589 } 3590 } 3591 3592 /* If a critical error is pending we have no choice but to reset the 3593 * device. 3594 * Report and mask out any remaining unexpected interrupts. 3595 */ 3596 icr0_remaining = icr0 & ena_mask; 3597 if (icr0_remaining) { 3598 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3599 icr0_remaining); 3600 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3601 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3602 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3603 dev_info(&pf->pdev->dev, "device will be reset\n"); 3604 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 3605 i40e_service_event_schedule(pf); 3606 } 3607 ena_mask &= ~icr0_remaining; 3608 } 3609 ret = IRQ_HANDLED; 3610 3611 enable_intr: 3612 /* re-enable interrupt causes */ 3613 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3614 if (!test_bit(__I40E_DOWN, &pf->state)) { 3615 i40e_service_event_schedule(pf); 3616 i40e_irq_dynamic_enable_icr0(pf, false); 3617 } 3618 3619 return ret; 3620 } 3621 3622 /** 3623 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3624 * @tx_ring: tx ring to clean 3625 * @budget: how many cleans we're allowed 3626 * 3627 * Returns true if there's any budget left (e.g. the clean is finished) 3628 **/ 3629 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3630 { 3631 struct i40e_vsi *vsi = tx_ring->vsi; 3632 u16 i = tx_ring->next_to_clean; 3633 struct i40e_tx_buffer *tx_buf; 3634 struct i40e_tx_desc *tx_desc; 3635 3636 tx_buf = &tx_ring->tx_bi[i]; 3637 tx_desc = I40E_TX_DESC(tx_ring, i); 3638 i -= tx_ring->count; 3639 3640 do { 3641 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3642 3643 /* if next_to_watch is not set then there is no work pending */ 3644 if (!eop_desc) 3645 break; 3646 3647 /* prevent any other reads prior to eop_desc */ 3648 read_barrier_depends(); 3649 3650 /* if the descriptor isn't done, no work yet to do */ 3651 if (!(eop_desc->cmd_type_offset_bsz & 3652 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3653 break; 3654 3655 /* clear next_to_watch to prevent false hangs */ 3656 tx_buf->next_to_watch = NULL; 3657 3658 tx_desc->buffer_addr = 0; 3659 tx_desc->cmd_type_offset_bsz = 0; 3660 /* move past filter desc */ 3661 tx_buf++; 3662 tx_desc++; 3663 i++; 3664 if (unlikely(!i)) { 3665 i -= tx_ring->count; 3666 tx_buf = tx_ring->tx_bi; 3667 tx_desc = I40E_TX_DESC(tx_ring, 0); 3668 } 3669 /* unmap skb header data */ 3670 dma_unmap_single(tx_ring->dev, 3671 dma_unmap_addr(tx_buf, dma), 3672 dma_unmap_len(tx_buf, len), 3673 DMA_TO_DEVICE); 3674 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3675 kfree(tx_buf->raw_buf); 3676 3677 tx_buf->raw_buf = NULL; 3678 tx_buf->tx_flags = 0; 3679 tx_buf->next_to_watch = NULL; 3680 dma_unmap_len_set(tx_buf, len, 0); 3681 tx_desc->buffer_addr = 0; 3682 tx_desc->cmd_type_offset_bsz = 0; 3683 3684 /* move us past the eop_desc for start of next FD desc */ 3685 tx_buf++; 3686 tx_desc++; 3687 i++; 3688 if (unlikely(!i)) { 3689 i -= tx_ring->count; 3690 tx_buf = tx_ring->tx_bi; 3691 tx_desc = I40E_TX_DESC(tx_ring, 0); 3692 } 3693 3694 /* update budget accounting */ 3695 budget--; 3696 } while (likely(budget)); 3697 3698 i += tx_ring->count; 3699 tx_ring->next_to_clean = i; 3700 3701 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 3702 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 3703 3704 return budget > 0; 3705 } 3706 3707 /** 3708 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3709 * @irq: interrupt number 3710 * @data: pointer to a q_vector 3711 **/ 3712 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3713 { 3714 struct i40e_q_vector *q_vector = data; 3715 struct i40e_vsi *vsi; 3716 3717 if (!q_vector->tx.ring) 3718 return IRQ_HANDLED; 3719 3720 vsi = q_vector->tx.ring->vsi; 3721 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3722 3723 return IRQ_HANDLED; 3724 } 3725 3726 /** 3727 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3728 * @vsi: the VSI being configured 3729 * @v_idx: vector index 3730 * @qp_idx: queue pair index 3731 **/ 3732 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3733 { 3734 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3735 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3736 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3737 3738 tx_ring->q_vector = q_vector; 3739 tx_ring->next = q_vector->tx.ring; 3740 q_vector->tx.ring = tx_ring; 3741 q_vector->tx.count++; 3742 3743 rx_ring->q_vector = q_vector; 3744 rx_ring->next = q_vector->rx.ring; 3745 q_vector->rx.ring = rx_ring; 3746 q_vector->rx.count++; 3747 } 3748 3749 /** 3750 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3751 * @vsi: the VSI being configured 3752 * 3753 * This function maps descriptor rings to the queue-specific vectors 3754 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3755 * one vector per queue pair, but on a constrained vector budget, we 3756 * group the queue pairs as "efficiently" as possible. 3757 **/ 3758 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3759 { 3760 int qp_remaining = vsi->num_queue_pairs; 3761 int q_vectors = vsi->num_q_vectors; 3762 int num_ringpairs; 3763 int v_start = 0; 3764 int qp_idx = 0; 3765 3766 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3767 * group them so there are multiple queues per vector. 3768 * It is also important to go through all the vectors available to be 3769 * sure that if we don't use all the vectors, that the remaining vectors 3770 * are cleared. This is especially important when decreasing the 3771 * number of queues in use. 3772 */ 3773 for (; v_start < q_vectors; v_start++) { 3774 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3775 3776 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3777 3778 q_vector->num_ringpairs = num_ringpairs; 3779 3780 q_vector->rx.count = 0; 3781 q_vector->tx.count = 0; 3782 q_vector->rx.ring = NULL; 3783 q_vector->tx.ring = NULL; 3784 3785 while (num_ringpairs--) { 3786 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 3787 qp_idx++; 3788 qp_remaining--; 3789 } 3790 } 3791 } 3792 3793 /** 3794 * i40e_vsi_request_irq - Request IRQ from the OS 3795 * @vsi: the VSI being configured 3796 * @basename: name for the vector 3797 **/ 3798 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3799 { 3800 struct i40e_pf *pf = vsi->back; 3801 int err; 3802 3803 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3804 err = i40e_vsi_request_irq_msix(vsi, basename); 3805 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3806 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3807 pf->int_name, pf); 3808 else 3809 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3810 pf->int_name, pf); 3811 3812 if (err) 3813 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3814 3815 return err; 3816 } 3817 3818 #ifdef CONFIG_NET_POLL_CONTROLLER 3819 /** 3820 * i40e_netpoll - A Polling 'interrupt' handler 3821 * @netdev: network interface device structure 3822 * 3823 * This is used by netconsole to send skbs without having to re-enable 3824 * interrupts. It's not called while the normal interrupt routine is executing. 3825 **/ 3826 #ifdef I40E_FCOE 3827 void i40e_netpoll(struct net_device *netdev) 3828 #else 3829 static void i40e_netpoll(struct net_device *netdev) 3830 #endif 3831 { 3832 struct i40e_netdev_priv *np = netdev_priv(netdev); 3833 struct i40e_vsi *vsi = np->vsi; 3834 struct i40e_pf *pf = vsi->back; 3835 int i; 3836 3837 /* if interface is down do nothing */ 3838 if (test_bit(__I40E_DOWN, &vsi->state)) 3839 return; 3840 3841 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3842 for (i = 0; i < vsi->num_q_vectors; i++) 3843 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3844 } else { 3845 i40e_intr(pf->pdev->irq, netdev); 3846 } 3847 } 3848 #endif 3849 3850 /** 3851 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 3852 * @pf: the PF being configured 3853 * @pf_q: the PF queue 3854 * @enable: enable or disable state of the queue 3855 * 3856 * This routine will wait for the given Tx queue of the PF to reach the 3857 * enabled or disabled state. 3858 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3859 * multiple retries; else will return 0 in case of success. 3860 **/ 3861 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 3862 { 3863 int i; 3864 u32 tx_reg; 3865 3866 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 3867 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 3868 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3869 break; 3870 3871 usleep_range(10, 20); 3872 } 3873 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 3874 return -ETIMEDOUT; 3875 3876 return 0; 3877 } 3878 3879 /** 3880 * i40e_vsi_control_tx - Start or stop a VSI's rings 3881 * @vsi: the VSI being configured 3882 * @enable: start or stop the rings 3883 **/ 3884 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 3885 { 3886 struct i40e_pf *pf = vsi->back; 3887 struct i40e_hw *hw = &pf->hw; 3888 int i, j, pf_q, ret = 0; 3889 u32 tx_reg; 3890 3891 pf_q = vsi->base_queue; 3892 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3893 3894 /* warn the TX unit of coming changes */ 3895 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 3896 if (!enable) 3897 usleep_range(10, 20); 3898 3899 for (j = 0; j < 50; j++) { 3900 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 3901 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 3902 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 3903 break; 3904 usleep_range(1000, 2000); 3905 } 3906 /* Skip if the queue is already in the requested state */ 3907 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3908 continue; 3909 3910 /* turn on/off the queue */ 3911 if (enable) { 3912 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 3913 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 3914 } else { 3915 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3916 } 3917 3918 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 3919 /* No waiting for the Tx queue to disable */ 3920 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 3921 continue; 3922 3923 /* wait for the change to finish */ 3924 ret = i40e_pf_txq_wait(pf, pf_q, enable); 3925 if (ret) { 3926 dev_info(&pf->pdev->dev, 3927 "VSI seid %d Tx ring %d %sable timeout\n", 3928 vsi->seid, pf_q, (enable ? "en" : "dis")); 3929 break; 3930 } 3931 } 3932 3933 if (hw->revision_id == 0) 3934 mdelay(50); 3935 return ret; 3936 } 3937 3938 /** 3939 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 3940 * @pf: the PF being configured 3941 * @pf_q: the PF queue 3942 * @enable: enable or disable state of the queue 3943 * 3944 * This routine will wait for the given Rx queue of the PF to reach the 3945 * enabled or disabled state. 3946 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3947 * multiple retries; else will return 0 in case of success. 3948 **/ 3949 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 3950 { 3951 int i; 3952 u32 rx_reg; 3953 3954 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 3955 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 3956 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3957 break; 3958 3959 usleep_range(10, 20); 3960 } 3961 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 3962 return -ETIMEDOUT; 3963 3964 return 0; 3965 } 3966 3967 /** 3968 * i40e_vsi_control_rx - Start or stop a VSI's rings 3969 * @vsi: the VSI being configured 3970 * @enable: start or stop the rings 3971 **/ 3972 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 3973 { 3974 struct i40e_pf *pf = vsi->back; 3975 struct i40e_hw *hw = &pf->hw; 3976 int i, j, pf_q, ret = 0; 3977 u32 rx_reg; 3978 3979 pf_q = vsi->base_queue; 3980 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3981 for (j = 0; j < 50; j++) { 3982 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 3983 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 3984 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 3985 break; 3986 usleep_range(1000, 2000); 3987 } 3988 3989 /* Skip if the queue is already in the requested state */ 3990 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3991 continue; 3992 3993 /* turn on/off the queue */ 3994 if (enable) 3995 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 3996 else 3997 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 3998 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 3999 /* No waiting for the Tx queue to disable */ 4000 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 4001 continue; 4002 4003 /* wait for the change to finish */ 4004 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4005 if (ret) { 4006 dev_info(&pf->pdev->dev, 4007 "VSI seid %d Rx ring %d %sable timeout\n", 4008 vsi->seid, pf_q, (enable ? "en" : "dis")); 4009 break; 4010 } 4011 } 4012 4013 return ret; 4014 } 4015 4016 /** 4017 * i40e_vsi_start_rings - Start a VSI's rings 4018 * @vsi: the VSI being configured 4019 **/ 4020 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4021 { 4022 int ret = 0; 4023 4024 /* do rx first for enable and last for disable */ 4025 ret = i40e_vsi_control_rx(vsi, true); 4026 if (ret) 4027 return ret; 4028 ret = i40e_vsi_control_tx(vsi, true); 4029 4030 return ret; 4031 } 4032 4033 /** 4034 * i40e_vsi_stop_rings - Stop a VSI's rings 4035 * @vsi: the VSI being configured 4036 **/ 4037 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4038 { 4039 /* do rx first for enable and last for disable 4040 * Ignore return value, we need to shutdown whatever we can 4041 */ 4042 i40e_vsi_control_tx(vsi, false); 4043 i40e_vsi_control_rx(vsi, false); 4044 } 4045 4046 /** 4047 * i40e_vsi_free_irq - Free the irq association with the OS 4048 * @vsi: the VSI being configured 4049 **/ 4050 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4051 { 4052 struct i40e_pf *pf = vsi->back; 4053 struct i40e_hw *hw = &pf->hw; 4054 int base = vsi->base_vector; 4055 u32 val, qp; 4056 int i; 4057 4058 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4059 if (!vsi->q_vectors) 4060 return; 4061 4062 if (!vsi->irqs_ready) 4063 return; 4064 4065 vsi->irqs_ready = false; 4066 for (i = 0; i < vsi->num_q_vectors; i++) { 4067 int irq_num; 4068 u16 vector; 4069 4070 vector = i + base; 4071 irq_num = pf->msix_entries[vector].vector; 4072 4073 /* free only the irqs that were actually requested */ 4074 if (!vsi->q_vectors[i] || 4075 !vsi->q_vectors[i]->num_ringpairs) 4076 continue; 4077 4078 /* clear the affinity notifier in the IRQ descriptor */ 4079 irq_set_affinity_notifier(irq_num, NULL); 4080 /* clear the affinity_mask in the IRQ descriptor */ 4081 irq_set_affinity_hint(irq_num, NULL); 4082 synchronize_irq(irq_num); 4083 free_irq(irq_num, vsi->q_vectors[i]); 4084 4085 /* Tear down the interrupt queue link list 4086 * 4087 * We know that they come in pairs and always 4088 * the Rx first, then the Tx. To clear the 4089 * link list, stick the EOL value into the 4090 * next_q field of the registers. 4091 */ 4092 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4093 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4094 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4095 val |= I40E_QUEUE_END_OF_LIST 4096 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4097 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4098 4099 while (qp != I40E_QUEUE_END_OF_LIST) { 4100 u32 next; 4101 4102 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4103 4104 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4105 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4106 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4107 I40E_QINT_RQCTL_INTEVENT_MASK); 4108 4109 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4110 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4111 4112 wr32(hw, I40E_QINT_RQCTL(qp), val); 4113 4114 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4115 4116 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4117 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4118 4119 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4120 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4121 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4122 I40E_QINT_TQCTL_INTEVENT_MASK); 4123 4124 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4125 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4126 4127 wr32(hw, I40E_QINT_TQCTL(qp), val); 4128 qp = next; 4129 } 4130 } 4131 } else { 4132 free_irq(pf->pdev->irq, pf); 4133 4134 val = rd32(hw, I40E_PFINT_LNKLST0); 4135 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4136 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4137 val |= I40E_QUEUE_END_OF_LIST 4138 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4139 wr32(hw, I40E_PFINT_LNKLST0, val); 4140 4141 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4142 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4143 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4144 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4145 I40E_QINT_RQCTL_INTEVENT_MASK); 4146 4147 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4148 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4149 4150 wr32(hw, I40E_QINT_RQCTL(qp), val); 4151 4152 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4153 4154 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4155 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4156 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4157 I40E_QINT_TQCTL_INTEVENT_MASK); 4158 4159 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4160 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4161 4162 wr32(hw, I40E_QINT_TQCTL(qp), val); 4163 } 4164 } 4165 4166 /** 4167 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4168 * @vsi: the VSI being configured 4169 * @v_idx: Index of vector to be freed 4170 * 4171 * This function frees the memory allocated to the q_vector. In addition if 4172 * NAPI is enabled it will delete any references to the NAPI struct prior 4173 * to freeing the q_vector. 4174 **/ 4175 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4176 { 4177 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4178 struct i40e_ring *ring; 4179 4180 if (!q_vector) 4181 return; 4182 4183 /* disassociate q_vector from rings */ 4184 i40e_for_each_ring(ring, q_vector->tx) 4185 ring->q_vector = NULL; 4186 4187 i40e_for_each_ring(ring, q_vector->rx) 4188 ring->q_vector = NULL; 4189 4190 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4191 if (vsi->netdev) 4192 netif_napi_del(&q_vector->napi); 4193 4194 vsi->q_vectors[v_idx] = NULL; 4195 4196 kfree_rcu(q_vector, rcu); 4197 } 4198 4199 /** 4200 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4201 * @vsi: the VSI being un-configured 4202 * 4203 * This frees the memory allocated to the q_vectors and 4204 * deletes references to the NAPI struct. 4205 **/ 4206 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4207 { 4208 int v_idx; 4209 4210 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4211 i40e_free_q_vector(vsi, v_idx); 4212 } 4213 4214 /** 4215 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4216 * @pf: board private structure 4217 **/ 4218 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4219 { 4220 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4221 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4222 pci_disable_msix(pf->pdev); 4223 kfree(pf->msix_entries); 4224 pf->msix_entries = NULL; 4225 kfree(pf->irq_pile); 4226 pf->irq_pile = NULL; 4227 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4228 pci_disable_msi(pf->pdev); 4229 } 4230 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4231 } 4232 4233 /** 4234 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4235 * @pf: board private structure 4236 * 4237 * We go through and clear interrupt specific resources and reset the structure 4238 * to pre-load conditions 4239 **/ 4240 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4241 { 4242 int i; 4243 4244 i40e_stop_misc_vector(pf); 4245 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 4246 synchronize_irq(pf->msix_entries[0].vector); 4247 free_irq(pf->msix_entries[0].vector, pf); 4248 } 4249 4250 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4251 I40E_IWARP_IRQ_PILE_ID); 4252 4253 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4254 for (i = 0; i < pf->num_alloc_vsi; i++) 4255 if (pf->vsi[i]) 4256 i40e_vsi_free_q_vectors(pf->vsi[i]); 4257 i40e_reset_interrupt_capability(pf); 4258 } 4259 4260 /** 4261 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4262 * @vsi: the VSI being configured 4263 **/ 4264 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4265 { 4266 int q_idx; 4267 4268 if (!vsi->netdev) 4269 return; 4270 4271 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 4272 napi_enable(&vsi->q_vectors[q_idx]->napi); 4273 } 4274 4275 /** 4276 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4277 * @vsi: the VSI being configured 4278 **/ 4279 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4280 { 4281 int q_idx; 4282 4283 if (!vsi->netdev) 4284 return; 4285 4286 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 4287 napi_disable(&vsi->q_vectors[q_idx]->napi); 4288 } 4289 4290 /** 4291 * i40e_vsi_close - Shut down a VSI 4292 * @vsi: the vsi to be quelled 4293 **/ 4294 static void i40e_vsi_close(struct i40e_vsi *vsi) 4295 { 4296 bool reset = false; 4297 4298 if (!test_and_set_bit(__I40E_DOWN, &vsi->state)) 4299 i40e_down(vsi); 4300 i40e_vsi_free_irq(vsi); 4301 i40e_vsi_free_tx_resources(vsi); 4302 i40e_vsi_free_rx_resources(vsi); 4303 vsi->current_netdev_flags = 0; 4304 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 4305 reset = true; 4306 i40e_notify_client_of_netdev_close(vsi, reset); 4307 } 4308 4309 /** 4310 * i40e_quiesce_vsi - Pause a given VSI 4311 * @vsi: the VSI being paused 4312 **/ 4313 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4314 { 4315 if (test_bit(__I40E_DOWN, &vsi->state)) 4316 return; 4317 4318 /* No need to disable FCoE VSI when Tx suspended */ 4319 if ((test_bit(__I40E_PORT_TX_SUSPENDED, &vsi->back->state)) && 4320 vsi->type == I40E_VSI_FCOE) { 4321 dev_dbg(&vsi->back->pdev->dev, 4322 "VSI seid %d skipping FCoE VSI disable\n", vsi->seid); 4323 return; 4324 } 4325 4326 set_bit(__I40E_NEEDS_RESTART, &vsi->state); 4327 if (vsi->netdev && netif_running(vsi->netdev)) 4328 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4329 else 4330 i40e_vsi_close(vsi); 4331 } 4332 4333 /** 4334 * i40e_unquiesce_vsi - Resume a given VSI 4335 * @vsi: the VSI being resumed 4336 **/ 4337 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4338 { 4339 if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state)) 4340 return; 4341 4342 clear_bit(__I40E_NEEDS_RESTART, &vsi->state); 4343 if (vsi->netdev && netif_running(vsi->netdev)) 4344 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4345 else 4346 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4347 } 4348 4349 /** 4350 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4351 * @pf: the PF 4352 **/ 4353 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4354 { 4355 int v; 4356 4357 for (v = 0; v < pf->num_alloc_vsi; v++) { 4358 if (pf->vsi[v]) 4359 i40e_quiesce_vsi(pf->vsi[v]); 4360 } 4361 } 4362 4363 /** 4364 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4365 * @pf: the PF 4366 **/ 4367 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4368 { 4369 int v; 4370 4371 for (v = 0; v < pf->num_alloc_vsi; v++) { 4372 if (pf->vsi[v]) 4373 i40e_unquiesce_vsi(pf->vsi[v]); 4374 } 4375 } 4376 4377 #ifdef CONFIG_I40E_DCB 4378 /** 4379 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4380 * @vsi: the VSI being configured 4381 * 4382 * This function waits for the given VSI's queues to be disabled. 4383 **/ 4384 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4385 { 4386 struct i40e_pf *pf = vsi->back; 4387 int i, pf_q, ret; 4388 4389 pf_q = vsi->base_queue; 4390 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4391 /* Check and wait for the disable status of the queue */ 4392 ret = i40e_pf_txq_wait(pf, pf_q, false); 4393 if (ret) { 4394 dev_info(&pf->pdev->dev, 4395 "VSI seid %d Tx ring %d disable timeout\n", 4396 vsi->seid, pf_q); 4397 return ret; 4398 } 4399 } 4400 4401 pf_q = vsi->base_queue; 4402 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4403 /* Check and wait for the disable status of the queue */ 4404 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4405 if (ret) { 4406 dev_info(&pf->pdev->dev, 4407 "VSI seid %d Rx ring %d disable timeout\n", 4408 vsi->seid, pf_q); 4409 return ret; 4410 } 4411 } 4412 4413 return 0; 4414 } 4415 4416 /** 4417 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4418 * @pf: the PF 4419 * 4420 * This function waits for the queues to be in disabled state for all the 4421 * VSIs that are managed by this PF. 4422 **/ 4423 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4424 { 4425 int v, ret = 0; 4426 4427 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4428 /* No need to wait for FCoE VSI queues */ 4429 if (pf->vsi[v] && pf->vsi[v]->type != I40E_VSI_FCOE) { 4430 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4431 if (ret) 4432 break; 4433 } 4434 } 4435 4436 return ret; 4437 } 4438 4439 #endif 4440 4441 /** 4442 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4443 * @q_idx: TX queue number 4444 * @vsi: Pointer to VSI struct 4445 * 4446 * This function checks specified queue for given VSI. Detects hung condition. 4447 * Sets hung bit since it is two step process. Before next run of service task 4448 * if napi_poll runs, it reset 'hung' bit for respective q_vector. If not, 4449 * hung condition remain unchanged and during subsequent run, this function 4450 * issues SW interrupt to recover from hung condition. 4451 **/ 4452 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4453 { 4454 struct i40e_ring *tx_ring = NULL; 4455 struct i40e_pf *pf; 4456 u32 head, val, tx_pending_hw; 4457 int i; 4458 4459 pf = vsi->back; 4460 4461 /* now that we have an index, find the tx_ring struct */ 4462 for (i = 0; i < vsi->num_queue_pairs; i++) { 4463 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4464 if (q_idx == vsi->tx_rings[i]->queue_index) { 4465 tx_ring = vsi->tx_rings[i]; 4466 break; 4467 } 4468 } 4469 } 4470 4471 if (!tx_ring) 4472 return; 4473 4474 /* Read interrupt register */ 4475 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4476 val = rd32(&pf->hw, 4477 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4478 tx_ring->vsi->base_vector - 1)); 4479 else 4480 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4481 4482 head = i40e_get_head(tx_ring); 4483 4484 tx_pending_hw = i40e_get_tx_pending(tx_ring, false); 4485 4486 /* HW is done executing descriptors, updated HEAD write back, 4487 * but SW hasn't processed those descriptors. If interrupt is 4488 * not generated from this point ON, it could result into 4489 * dev_watchdog detecting timeout on those netdev_queue, 4490 * hence proactively trigger SW interrupt. 4491 */ 4492 if (tx_pending_hw && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4493 /* NAPI Poll didn't run and clear since it was set */ 4494 if (test_and_clear_bit(I40E_Q_VECTOR_HUNG_DETECT, 4495 &tx_ring->q_vector->hung_detected)) { 4496 netdev_info(vsi->netdev, "VSI_seid %d, Hung TX queue %d, tx_pending_hw: %d, NTC:0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x\n", 4497 vsi->seid, q_idx, tx_pending_hw, 4498 tx_ring->next_to_clean, head, 4499 tx_ring->next_to_use, 4500 readl(tx_ring->tail)); 4501 netdev_info(vsi->netdev, "VSI_seid %d, Issuing force_wb for TX queue %d, Interrupt Reg: 0x%x\n", 4502 vsi->seid, q_idx, val); 4503 i40e_force_wb(vsi, tx_ring->q_vector); 4504 } else { 4505 /* First Chance - detected possible hung */ 4506 set_bit(I40E_Q_VECTOR_HUNG_DETECT, 4507 &tx_ring->q_vector->hung_detected); 4508 } 4509 } 4510 4511 /* This is the case where we have interrupts missing, 4512 * so the tx_pending in HW will most likely be 0, but we 4513 * will have tx_pending in SW since the WB happened but the 4514 * interrupt got lost. 4515 */ 4516 if ((!tx_pending_hw) && i40e_get_tx_pending(tx_ring, true) && 4517 (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4518 if (napi_reschedule(&tx_ring->q_vector->napi)) 4519 tx_ring->tx_stats.tx_lost_interrupt++; 4520 } 4521 } 4522 4523 /** 4524 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4525 * @pf: pointer to PF struct 4526 * 4527 * LAN VSI has netdev and netdev has TX queues. This function is to check 4528 * each of those TX queues if they are hung, trigger recovery by issuing 4529 * SW interrupt. 4530 **/ 4531 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4532 { 4533 struct net_device *netdev; 4534 struct i40e_vsi *vsi; 4535 int i; 4536 4537 /* Only for LAN VSI */ 4538 vsi = pf->vsi[pf->lan_vsi]; 4539 4540 if (!vsi) 4541 return; 4542 4543 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4544 if (test_bit(__I40E_DOWN, &vsi->back->state) || 4545 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 4546 return; 4547 4548 /* Make sure type is MAIN VSI */ 4549 if (vsi->type != I40E_VSI_MAIN) 4550 return; 4551 4552 netdev = vsi->netdev; 4553 if (!netdev) 4554 return; 4555 4556 /* Bail out if netif_carrier is not OK */ 4557 if (!netif_carrier_ok(netdev)) 4558 return; 4559 4560 /* Go thru' TX queues for netdev */ 4561 for (i = 0; i < netdev->num_tx_queues; i++) { 4562 struct netdev_queue *q; 4563 4564 q = netdev_get_tx_queue(netdev, i); 4565 if (q) 4566 i40e_detect_recover_hung_queue(i, vsi); 4567 } 4568 } 4569 4570 /** 4571 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4572 * @pf: pointer to PF 4573 * 4574 * Get TC map for ISCSI PF type that will include iSCSI TC 4575 * and LAN TC. 4576 **/ 4577 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4578 { 4579 struct i40e_dcb_app_priority_table app; 4580 struct i40e_hw *hw = &pf->hw; 4581 u8 enabled_tc = 1; /* TC0 is always enabled */ 4582 u8 tc, i; 4583 /* Get the iSCSI APP TLV */ 4584 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4585 4586 for (i = 0; i < dcbcfg->numapps; i++) { 4587 app = dcbcfg->app[i]; 4588 if (app.selector == I40E_APP_SEL_TCPIP && 4589 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4590 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4591 enabled_tc |= BIT(tc); 4592 break; 4593 } 4594 } 4595 4596 return enabled_tc; 4597 } 4598 4599 /** 4600 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4601 * @dcbcfg: the corresponding DCBx configuration structure 4602 * 4603 * Return the number of TCs from given DCBx configuration 4604 **/ 4605 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4606 { 4607 int i, tc_unused = 0; 4608 u8 num_tc = 0; 4609 u8 ret = 0; 4610 4611 /* Scan the ETS Config Priority Table to find 4612 * traffic class enabled for a given priority 4613 * and create a bitmask of enabled TCs 4614 */ 4615 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4616 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4617 4618 /* Now scan the bitmask to check for 4619 * contiguous TCs starting with TC0 4620 */ 4621 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4622 if (num_tc & BIT(i)) { 4623 if (!tc_unused) { 4624 ret++; 4625 } else { 4626 pr_err("Non-contiguous TC - Disabling DCB\n"); 4627 return 1; 4628 } 4629 } else { 4630 tc_unused = 1; 4631 } 4632 } 4633 4634 /* There is always at least TC0 */ 4635 if (!ret) 4636 ret = 1; 4637 4638 return ret; 4639 } 4640 4641 /** 4642 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4643 * @dcbcfg: the corresponding DCBx configuration structure 4644 * 4645 * Query the current DCB configuration and return the number of 4646 * traffic classes enabled from the given DCBX config 4647 **/ 4648 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4649 { 4650 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4651 u8 enabled_tc = 1; 4652 u8 i; 4653 4654 for (i = 0; i < num_tc; i++) 4655 enabled_tc |= BIT(i); 4656 4657 return enabled_tc; 4658 } 4659 4660 /** 4661 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4662 * @pf: PF being queried 4663 * 4664 * Return number of traffic classes enabled for the given PF 4665 **/ 4666 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4667 { 4668 struct i40e_hw *hw = &pf->hw; 4669 u8 i, enabled_tc = 1; 4670 u8 num_tc = 0; 4671 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4672 4673 /* If DCB is not enabled then always in single TC */ 4674 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4675 return 1; 4676 4677 /* SFP mode will be enabled for all TCs on port */ 4678 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4679 return i40e_dcb_get_num_tc(dcbcfg); 4680 4681 /* MFP mode return count of enabled TCs for this PF */ 4682 if (pf->hw.func_caps.iscsi) 4683 enabled_tc = i40e_get_iscsi_tc_map(pf); 4684 else 4685 return 1; /* Only TC0 */ 4686 4687 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4688 if (enabled_tc & BIT(i)) 4689 num_tc++; 4690 } 4691 return num_tc; 4692 } 4693 4694 /** 4695 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 4696 * @pf: PF being queried 4697 * 4698 * Return a bitmap for enabled traffic classes for this PF. 4699 **/ 4700 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 4701 { 4702 /* If DCB is not enabled for this PF then just return default TC */ 4703 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4704 return I40E_DEFAULT_TRAFFIC_CLASS; 4705 4706 /* SFP mode we want PF to be enabled for all TCs */ 4707 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4708 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 4709 4710 /* MFP enabled and iSCSI PF type */ 4711 if (pf->hw.func_caps.iscsi) 4712 return i40e_get_iscsi_tc_map(pf); 4713 else 4714 return I40E_DEFAULT_TRAFFIC_CLASS; 4715 } 4716 4717 /** 4718 * i40e_vsi_get_bw_info - Query VSI BW Information 4719 * @vsi: the VSI being queried 4720 * 4721 * Returns 0 on success, negative value on failure 4722 **/ 4723 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 4724 { 4725 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 4726 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 4727 struct i40e_pf *pf = vsi->back; 4728 struct i40e_hw *hw = &pf->hw; 4729 i40e_status ret; 4730 u32 tc_bw_max; 4731 int i; 4732 4733 /* Get the VSI level BW configuration */ 4734 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 4735 if (ret) { 4736 dev_info(&pf->pdev->dev, 4737 "couldn't get PF vsi bw config, err %s aq_err %s\n", 4738 i40e_stat_str(&pf->hw, ret), 4739 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4740 return -EINVAL; 4741 } 4742 4743 /* Get the VSI level BW configuration per TC */ 4744 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 4745 NULL); 4746 if (ret) { 4747 dev_info(&pf->pdev->dev, 4748 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 4749 i40e_stat_str(&pf->hw, ret), 4750 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4751 return -EINVAL; 4752 } 4753 4754 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 4755 dev_info(&pf->pdev->dev, 4756 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 4757 bw_config.tc_valid_bits, 4758 bw_ets_config.tc_valid_bits); 4759 /* Still continuing */ 4760 } 4761 4762 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 4763 vsi->bw_max_quanta = bw_config.max_bw; 4764 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 4765 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 4766 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4767 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 4768 vsi->bw_ets_limit_credits[i] = 4769 le16_to_cpu(bw_ets_config.credits[i]); 4770 /* 3 bits out of 4 for each TC */ 4771 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 4772 } 4773 4774 return 0; 4775 } 4776 4777 /** 4778 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 4779 * @vsi: the VSI being configured 4780 * @enabled_tc: TC bitmap 4781 * @bw_credits: BW shared credits per TC 4782 * 4783 * Returns 0 on success, negative value on failure 4784 **/ 4785 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 4786 u8 *bw_share) 4787 { 4788 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 4789 i40e_status ret; 4790 int i; 4791 4792 bw_data.tc_valid_bits = enabled_tc; 4793 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4794 bw_data.tc_bw_credits[i] = bw_share[i]; 4795 4796 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 4797 NULL); 4798 if (ret) { 4799 dev_info(&vsi->back->pdev->dev, 4800 "AQ command Config VSI BW allocation per TC failed = %d\n", 4801 vsi->back->hw.aq.asq_last_status); 4802 return -EINVAL; 4803 } 4804 4805 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4806 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 4807 4808 return 0; 4809 } 4810 4811 /** 4812 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 4813 * @vsi: the VSI being configured 4814 * @enabled_tc: TC map to be enabled 4815 * 4816 **/ 4817 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4818 { 4819 struct net_device *netdev = vsi->netdev; 4820 struct i40e_pf *pf = vsi->back; 4821 struct i40e_hw *hw = &pf->hw; 4822 u8 netdev_tc = 0; 4823 int i; 4824 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4825 4826 if (!netdev) 4827 return; 4828 4829 if (!enabled_tc) { 4830 netdev_reset_tc(netdev); 4831 return; 4832 } 4833 4834 /* Set up actual enabled TCs on the VSI */ 4835 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 4836 return; 4837 4838 /* set per TC queues for the VSI */ 4839 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4840 /* Only set TC queues for enabled tcs 4841 * 4842 * e.g. For a VSI that has TC0 and TC3 enabled the 4843 * enabled_tc bitmap would be 0x00001001; the driver 4844 * will set the numtc for netdev as 2 that will be 4845 * referenced by the netdev layer as TC 0 and 1. 4846 */ 4847 if (vsi->tc_config.enabled_tc & BIT(i)) 4848 netdev_set_tc_queue(netdev, 4849 vsi->tc_config.tc_info[i].netdev_tc, 4850 vsi->tc_config.tc_info[i].qcount, 4851 vsi->tc_config.tc_info[i].qoffset); 4852 } 4853 4854 /* Assign UP2TC map for the VSI */ 4855 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 4856 /* Get the actual TC# for the UP */ 4857 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 4858 /* Get the mapped netdev TC# for the UP */ 4859 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 4860 netdev_set_prio_tc_map(netdev, i, netdev_tc); 4861 } 4862 } 4863 4864 /** 4865 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 4866 * @vsi: the VSI being configured 4867 * @ctxt: the ctxt buffer returned from AQ VSI update param command 4868 **/ 4869 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 4870 struct i40e_vsi_context *ctxt) 4871 { 4872 /* copy just the sections touched not the entire info 4873 * since not all sections are valid as returned by 4874 * update vsi params 4875 */ 4876 vsi->info.mapping_flags = ctxt->info.mapping_flags; 4877 memcpy(&vsi->info.queue_mapping, 4878 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 4879 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 4880 sizeof(vsi->info.tc_mapping)); 4881 } 4882 4883 /** 4884 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 4885 * @vsi: VSI to be configured 4886 * @enabled_tc: TC bitmap 4887 * 4888 * This configures a particular VSI for TCs that are mapped to the 4889 * given TC bitmap. It uses default bandwidth share for TCs across 4890 * VSIs to configure TC for a particular VSI. 4891 * 4892 * NOTE: 4893 * It is expected that the VSI queues have been quisced before calling 4894 * this function. 4895 **/ 4896 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4897 { 4898 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 4899 struct i40e_vsi_context ctxt; 4900 int ret = 0; 4901 int i; 4902 4903 /* Check if enabled_tc is same as existing or new TCs */ 4904 if (vsi->tc_config.enabled_tc == enabled_tc) 4905 return ret; 4906 4907 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 4908 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4909 if (enabled_tc & BIT(i)) 4910 bw_share[i] = 1; 4911 } 4912 4913 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 4914 if (ret) { 4915 dev_info(&vsi->back->pdev->dev, 4916 "Failed configuring TC map %d for VSI %d\n", 4917 enabled_tc, vsi->seid); 4918 goto out; 4919 } 4920 4921 /* Update Queue Pairs Mapping for currently enabled UPs */ 4922 ctxt.seid = vsi->seid; 4923 ctxt.pf_num = vsi->back->hw.pf_id; 4924 ctxt.vf_num = 0; 4925 ctxt.uplink_seid = vsi->uplink_seid; 4926 ctxt.info = vsi->info; 4927 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 4928 4929 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 4930 ctxt.info.valid_sections |= 4931 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 4932 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 4933 } 4934 4935 /* Update the VSI after updating the VSI queue-mapping information */ 4936 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 4937 if (ret) { 4938 dev_info(&vsi->back->pdev->dev, 4939 "Update vsi tc config failed, err %s aq_err %s\n", 4940 i40e_stat_str(&vsi->back->hw, ret), 4941 i40e_aq_str(&vsi->back->hw, 4942 vsi->back->hw.aq.asq_last_status)); 4943 goto out; 4944 } 4945 /* update the local VSI info with updated queue map */ 4946 i40e_vsi_update_queue_map(vsi, &ctxt); 4947 vsi->info.valid_sections = 0; 4948 4949 /* Update current VSI BW information */ 4950 ret = i40e_vsi_get_bw_info(vsi); 4951 if (ret) { 4952 dev_info(&vsi->back->pdev->dev, 4953 "Failed updating vsi bw info, err %s aq_err %s\n", 4954 i40e_stat_str(&vsi->back->hw, ret), 4955 i40e_aq_str(&vsi->back->hw, 4956 vsi->back->hw.aq.asq_last_status)); 4957 goto out; 4958 } 4959 4960 /* Update the netdev TC setup */ 4961 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 4962 out: 4963 return ret; 4964 } 4965 4966 /** 4967 * i40e_veb_config_tc - Configure TCs for given VEB 4968 * @veb: given VEB 4969 * @enabled_tc: TC bitmap 4970 * 4971 * Configures given TC bitmap for VEB (switching) element 4972 **/ 4973 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 4974 { 4975 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 4976 struct i40e_pf *pf = veb->pf; 4977 int ret = 0; 4978 int i; 4979 4980 /* No TCs or already enabled TCs just return */ 4981 if (!enabled_tc || veb->enabled_tc == enabled_tc) 4982 return ret; 4983 4984 bw_data.tc_valid_bits = enabled_tc; 4985 /* bw_data.absolute_credits is not set (relative) */ 4986 4987 /* Enable ETS TCs with equal BW Share for now */ 4988 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4989 if (enabled_tc & BIT(i)) 4990 bw_data.tc_bw_share_credits[i] = 1; 4991 } 4992 4993 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 4994 &bw_data, NULL); 4995 if (ret) { 4996 dev_info(&pf->pdev->dev, 4997 "VEB bw config failed, err %s aq_err %s\n", 4998 i40e_stat_str(&pf->hw, ret), 4999 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5000 goto out; 5001 } 5002 5003 /* Update the BW information */ 5004 ret = i40e_veb_get_bw_info(veb); 5005 if (ret) { 5006 dev_info(&pf->pdev->dev, 5007 "Failed getting veb bw config, err %s aq_err %s\n", 5008 i40e_stat_str(&pf->hw, ret), 5009 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5010 } 5011 5012 out: 5013 return ret; 5014 } 5015 5016 #ifdef CONFIG_I40E_DCB 5017 /** 5018 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 5019 * @pf: PF struct 5020 * 5021 * Reconfigure VEB/VSIs on a given PF; it is assumed that 5022 * the caller would've quiesce all the VSIs before calling 5023 * this function 5024 **/ 5025 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 5026 { 5027 u8 tc_map = 0; 5028 int ret; 5029 u8 v; 5030 5031 /* Enable the TCs available on PF to all VEBs */ 5032 tc_map = i40e_pf_get_tc_map(pf); 5033 for (v = 0; v < I40E_MAX_VEB; v++) { 5034 if (!pf->veb[v]) 5035 continue; 5036 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 5037 if (ret) { 5038 dev_info(&pf->pdev->dev, 5039 "Failed configuring TC for VEB seid=%d\n", 5040 pf->veb[v]->seid); 5041 /* Will try to configure as many components */ 5042 } 5043 } 5044 5045 /* Update each VSI */ 5046 for (v = 0; v < pf->num_alloc_vsi; v++) { 5047 if (!pf->vsi[v]) 5048 continue; 5049 5050 /* - Enable all TCs for the LAN VSI 5051 #ifdef I40E_FCOE 5052 * - For FCoE VSI only enable the TC configured 5053 * as per the APP TLV 5054 #endif 5055 * - For all others keep them at TC0 for now 5056 */ 5057 if (v == pf->lan_vsi) 5058 tc_map = i40e_pf_get_tc_map(pf); 5059 else 5060 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 5061 #ifdef I40E_FCOE 5062 if (pf->vsi[v]->type == I40E_VSI_FCOE) 5063 tc_map = i40e_get_fcoe_tc_map(pf); 5064 #endif /* #ifdef I40E_FCOE */ 5065 5066 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 5067 if (ret) { 5068 dev_info(&pf->pdev->dev, 5069 "Failed configuring TC for VSI seid=%d\n", 5070 pf->vsi[v]->seid); 5071 /* Will try to configure as many components */ 5072 } else { 5073 /* Re-configure VSI vectors based on updated TC map */ 5074 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 5075 if (pf->vsi[v]->netdev) 5076 i40e_dcbnl_set_all(pf->vsi[v]); 5077 } 5078 } 5079 } 5080 5081 /** 5082 * i40e_resume_port_tx - Resume port Tx 5083 * @pf: PF struct 5084 * 5085 * Resume a port's Tx and issue a PF reset in case of failure to 5086 * resume. 5087 **/ 5088 static int i40e_resume_port_tx(struct i40e_pf *pf) 5089 { 5090 struct i40e_hw *hw = &pf->hw; 5091 int ret; 5092 5093 ret = i40e_aq_resume_port_tx(hw, NULL); 5094 if (ret) { 5095 dev_info(&pf->pdev->dev, 5096 "Resume Port Tx failed, err %s aq_err %s\n", 5097 i40e_stat_str(&pf->hw, ret), 5098 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5099 /* Schedule PF reset to recover */ 5100 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5101 i40e_service_event_schedule(pf); 5102 } 5103 5104 return ret; 5105 } 5106 5107 /** 5108 * i40e_init_pf_dcb - Initialize DCB configuration 5109 * @pf: PF being configured 5110 * 5111 * Query the current DCB configuration and cache it 5112 * in the hardware structure 5113 **/ 5114 static int i40e_init_pf_dcb(struct i40e_pf *pf) 5115 { 5116 struct i40e_hw *hw = &pf->hw; 5117 int err = 0; 5118 5119 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 5120 if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT) 5121 goto out; 5122 5123 /* Get the initial DCB configuration */ 5124 err = i40e_init_dcb(hw); 5125 if (!err) { 5126 /* Device/Function is not DCBX capable */ 5127 if ((!hw->func_caps.dcb) || 5128 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 5129 dev_info(&pf->pdev->dev, 5130 "DCBX offload is not supported or is disabled for this PF.\n"); 5131 5132 if (pf->flags & I40E_FLAG_MFP_ENABLED) 5133 goto out; 5134 5135 } else { 5136 /* When status is not DISABLED then DCBX in FW */ 5137 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 5138 DCB_CAP_DCBX_VER_IEEE; 5139 5140 pf->flags |= I40E_FLAG_DCB_CAPABLE; 5141 /* Enable DCB tagging only when more than one TC 5142 * or explicitly disable if only one TC 5143 */ 5144 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5145 pf->flags |= I40E_FLAG_DCB_ENABLED; 5146 else 5147 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5148 dev_dbg(&pf->pdev->dev, 5149 "DCBX offload is supported for this PF.\n"); 5150 } 5151 } else { 5152 dev_info(&pf->pdev->dev, 5153 "Query for DCB configuration failed, err %s aq_err %s\n", 5154 i40e_stat_str(&pf->hw, err), 5155 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5156 } 5157 5158 out: 5159 return err; 5160 } 5161 #endif /* CONFIG_I40E_DCB */ 5162 #define SPEED_SIZE 14 5163 #define FC_SIZE 8 5164 /** 5165 * i40e_print_link_message - print link up or down 5166 * @vsi: the VSI for which link needs a message 5167 */ 5168 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 5169 { 5170 char *speed = "Unknown"; 5171 char *fc = "Unknown"; 5172 5173 if (vsi->current_isup == isup) 5174 return; 5175 vsi->current_isup = isup; 5176 if (!isup) { 5177 netdev_info(vsi->netdev, "NIC Link is Down\n"); 5178 return; 5179 } 5180 5181 /* Warn user if link speed on NPAR enabled partition is not at 5182 * least 10GB 5183 */ 5184 if (vsi->back->hw.func_caps.npar_enable && 5185 (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 5186 vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 5187 netdev_warn(vsi->netdev, 5188 "The partition detected link speed that is less than 10Gbps\n"); 5189 5190 switch (vsi->back->hw.phy.link_info.link_speed) { 5191 case I40E_LINK_SPEED_40GB: 5192 speed = "40 G"; 5193 break; 5194 case I40E_LINK_SPEED_20GB: 5195 speed = "20 G"; 5196 break; 5197 case I40E_LINK_SPEED_10GB: 5198 speed = "10 G"; 5199 break; 5200 case I40E_LINK_SPEED_1GB: 5201 speed = "1000 M"; 5202 break; 5203 case I40E_LINK_SPEED_100MB: 5204 speed = "100 M"; 5205 break; 5206 default: 5207 break; 5208 } 5209 5210 switch (vsi->back->hw.fc.current_mode) { 5211 case I40E_FC_FULL: 5212 fc = "RX/TX"; 5213 break; 5214 case I40E_FC_TX_PAUSE: 5215 fc = "TX"; 5216 break; 5217 case I40E_FC_RX_PAUSE: 5218 fc = "RX"; 5219 break; 5220 default: 5221 fc = "None"; 5222 break; 5223 } 5224 5225 netdev_info(vsi->netdev, "NIC Link is Up %sbps Full Duplex, Flow Control: %s\n", 5226 speed, fc); 5227 } 5228 5229 /** 5230 * i40e_up_complete - Finish the last steps of bringing up a connection 5231 * @vsi: the VSI being configured 5232 **/ 5233 static int i40e_up_complete(struct i40e_vsi *vsi) 5234 { 5235 struct i40e_pf *pf = vsi->back; 5236 int err; 5237 5238 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5239 i40e_vsi_configure_msix(vsi); 5240 else 5241 i40e_configure_msi_and_legacy(vsi); 5242 5243 /* start rings */ 5244 err = i40e_vsi_start_rings(vsi); 5245 if (err) 5246 return err; 5247 5248 clear_bit(__I40E_DOWN, &vsi->state); 5249 i40e_napi_enable_all(vsi); 5250 i40e_vsi_enable_irq(vsi); 5251 5252 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 5253 (vsi->netdev)) { 5254 i40e_print_link_message(vsi, true); 5255 netif_tx_start_all_queues(vsi->netdev); 5256 netif_carrier_on(vsi->netdev); 5257 } else if (vsi->netdev) { 5258 i40e_print_link_message(vsi, false); 5259 /* need to check for qualified module here*/ 5260 if ((pf->hw.phy.link_info.link_info & 5261 I40E_AQ_MEDIA_AVAILABLE) && 5262 (!(pf->hw.phy.link_info.an_info & 5263 I40E_AQ_QUALIFIED_MODULE))) 5264 netdev_err(vsi->netdev, 5265 "the driver failed to link because an unqualified module was detected."); 5266 } 5267 5268 /* replay FDIR SB filters */ 5269 if (vsi->type == I40E_VSI_FDIR) { 5270 /* reset fd counters */ 5271 pf->fd_add_err = pf->fd_atr_cnt = 0; 5272 if (pf->fd_tcp_rule > 0) { 5273 pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED; 5274 if (I40E_DEBUG_FD & pf->hw.debug_mask) 5275 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n"); 5276 pf->fd_tcp_rule = 0; 5277 } 5278 i40e_fdir_filter_restore(vsi); 5279 } 5280 5281 /* On the next run of the service_task, notify any clients of the new 5282 * opened netdev 5283 */ 5284 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 5285 i40e_service_event_schedule(pf); 5286 5287 return 0; 5288 } 5289 5290 /** 5291 * i40e_vsi_reinit_locked - Reset the VSI 5292 * @vsi: the VSI being configured 5293 * 5294 * Rebuild the ring structs after some configuration 5295 * has changed, e.g. MTU size. 5296 **/ 5297 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 5298 { 5299 struct i40e_pf *pf = vsi->back; 5300 5301 WARN_ON(in_interrupt()); 5302 while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state)) 5303 usleep_range(1000, 2000); 5304 i40e_down(vsi); 5305 5306 i40e_up(vsi); 5307 clear_bit(__I40E_CONFIG_BUSY, &pf->state); 5308 } 5309 5310 /** 5311 * i40e_up - Bring the connection back up after being down 5312 * @vsi: the VSI being configured 5313 **/ 5314 int i40e_up(struct i40e_vsi *vsi) 5315 { 5316 int err; 5317 5318 err = i40e_vsi_configure(vsi); 5319 if (!err) 5320 err = i40e_up_complete(vsi); 5321 5322 return err; 5323 } 5324 5325 /** 5326 * i40e_down - Shutdown the connection processing 5327 * @vsi: the VSI being stopped 5328 **/ 5329 void i40e_down(struct i40e_vsi *vsi) 5330 { 5331 int i; 5332 5333 /* It is assumed that the caller of this function 5334 * sets the vsi->state __I40E_DOWN bit. 5335 */ 5336 if (vsi->netdev) { 5337 netif_carrier_off(vsi->netdev); 5338 netif_tx_disable(vsi->netdev); 5339 } 5340 i40e_vsi_disable_irq(vsi); 5341 i40e_vsi_stop_rings(vsi); 5342 i40e_napi_disable_all(vsi); 5343 5344 for (i = 0; i < vsi->num_queue_pairs; i++) { 5345 i40e_clean_tx_ring(vsi->tx_rings[i]); 5346 i40e_clean_rx_ring(vsi->rx_rings[i]); 5347 } 5348 5349 i40e_notify_client_of_netdev_close(vsi, false); 5350 5351 } 5352 5353 /** 5354 * i40e_setup_tc - configure multiple traffic classes 5355 * @netdev: net device to configure 5356 * @tc: number of traffic classes to enable 5357 **/ 5358 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 5359 { 5360 struct i40e_netdev_priv *np = netdev_priv(netdev); 5361 struct i40e_vsi *vsi = np->vsi; 5362 struct i40e_pf *pf = vsi->back; 5363 u8 enabled_tc = 0; 5364 int ret = -EINVAL; 5365 int i; 5366 5367 /* Check if DCB enabled to continue */ 5368 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5369 netdev_info(netdev, "DCB is not enabled for adapter\n"); 5370 goto exit; 5371 } 5372 5373 /* Check if MFP enabled */ 5374 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 5375 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 5376 goto exit; 5377 } 5378 5379 /* Check whether tc count is within enabled limit */ 5380 if (tc > i40e_pf_get_num_tc(pf)) { 5381 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 5382 goto exit; 5383 } 5384 5385 /* Generate TC map for number of tc requested */ 5386 for (i = 0; i < tc; i++) 5387 enabled_tc |= BIT(i); 5388 5389 /* Requesting same TC configuration as already enabled */ 5390 if (enabled_tc == vsi->tc_config.enabled_tc) 5391 return 0; 5392 5393 /* Quiesce VSI queues */ 5394 i40e_quiesce_vsi(vsi); 5395 5396 /* Configure VSI for enabled TCs */ 5397 ret = i40e_vsi_config_tc(vsi, enabled_tc); 5398 if (ret) { 5399 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 5400 vsi->seid); 5401 goto exit; 5402 } 5403 5404 /* Unquiesce VSI */ 5405 i40e_unquiesce_vsi(vsi); 5406 5407 exit: 5408 return ret; 5409 } 5410 5411 #ifdef I40E_FCOE 5412 int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5413 struct tc_to_netdev *tc) 5414 #else 5415 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5416 struct tc_to_netdev *tc) 5417 #endif 5418 { 5419 if (handle != TC_H_ROOT || tc->type != TC_SETUP_MQPRIO) 5420 return -EINVAL; 5421 return i40e_setup_tc(netdev, tc->tc); 5422 } 5423 5424 /** 5425 * i40e_open - Called when a network interface is made active 5426 * @netdev: network interface device structure 5427 * 5428 * The open entry point is called when a network interface is made 5429 * active by the system (IFF_UP). At this point all resources needed 5430 * for transmit and receive operations are allocated, the interrupt 5431 * handler is registered with the OS, the netdev watchdog subtask is 5432 * enabled, and the stack is notified that the interface is ready. 5433 * 5434 * Returns 0 on success, negative value on failure 5435 **/ 5436 int i40e_open(struct net_device *netdev) 5437 { 5438 struct i40e_netdev_priv *np = netdev_priv(netdev); 5439 struct i40e_vsi *vsi = np->vsi; 5440 struct i40e_pf *pf = vsi->back; 5441 int err; 5442 5443 /* disallow open during test or if eeprom is broken */ 5444 if (test_bit(__I40E_TESTING, &pf->state) || 5445 test_bit(__I40E_BAD_EEPROM, &pf->state)) 5446 return -EBUSY; 5447 5448 netif_carrier_off(netdev); 5449 5450 err = i40e_vsi_open(vsi); 5451 if (err) 5452 return err; 5453 5454 /* configure global TSO hardware offload settings */ 5455 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 5456 TCP_FLAG_FIN) >> 16); 5457 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 5458 TCP_FLAG_FIN | 5459 TCP_FLAG_CWR) >> 16); 5460 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 5461 5462 udp_tunnel_get_rx_info(netdev); 5463 5464 return 0; 5465 } 5466 5467 /** 5468 * i40e_vsi_open - 5469 * @vsi: the VSI to open 5470 * 5471 * Finish initialization of the VSI. 5472 * 5473 * Returns 0 on success, negative value on failure 5474 **/ 5475 int i40e_vsi_open(struct i40e_vsi *vsi) 5476 { 5477 struct i40e_pf *pf = vsi->back; 5478 char int_name[I40E_INT_NAME_STR_LEN]; 5479 int err; 5480 5481 /* allocate descriptors */ 5482 err = i40e_vsi_setup_tx_resources(vsi); 5483 if (err) 5484 goto err_setup_tx; 5485 err = i40e_vsi_setup_rx_resources(vsi); 5486 if (err) 5487 goto err_setup_rx; 5488 5489 err = i40e_vsi_configure(vsi); 5490 if (err) 5491 goto err_setup_rx; 5492 5493 if (vsi->netdev) { 5494 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5495 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 5496 err = i40e_vsi_request_irq(vsi, int_name); 5497 if (err) 5498 goto err_setup_rx; 5499 5500 /* Notify the stack of the actual queue counts. */ 5501 err = netif_set_real_num_tx_queues(vsi->netdev, 5502 vsi->num_queue_pairs); 5503 if (err) 5504 goto err_set_queues; 5505 5506 err = netif_set_real_num_rx_queues(vsi->netdev, 5507 vsi->num_queue_pairs); 5508 if (err) 5509 goto err_set_queues; 5510 5511 } else if (vsi->type == I40E_VSI_FDIR) { 5512 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 5513 dev_driver_string(&pf->pdev->dev), 5514 dev_name(&pf->pdev->dev)); 5515 err = i40e_vsi_request_irq(vsi, int_name); 5516 5517 } else { 5518 err = -EINVAL; 5519 goto err_setup_rx; 5520 } 5521 5522 err = i40e_up_complete(vsi); 5523 if (err) 5524 goto err_up_complete; 5525 5526 return 0; 5527 5528 err_up_complete: 5529 i40e_down(vsi); 5530 err_set_queues: 5531 i40e_vsi_free_irq(vsi); 5532 err_setup_rx: 5533 i40e_vsi_free_rx_resources(vsi); 5534 err_setup_tx: 5535 i40e_vsi_free_tx_resources(vsi); 5536 if (vsi == pf->vsi[pf->lan_vsi]) 5537 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 5538 5539 return err; 5540 } 5541 5542 /** 5543 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 5544 * @pf: Pointer to PF 5545 * 5546 * This function destroys the hlist where all the Flow Director 5547 * filters were saved. 5548 **/ 5549 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 5550 { 5551 struct i40e_fdir_filter *filter; 5552 struct hlist_node *node2; 5553 5554 hlist_for_each_entry_safe(filter, node2, 5555 &pf->fdir_filter_list, fdir_node) { 5556 hlist_del(&filter->fdir_node); 5557 kfree(filter); 5558 } 5559 pf->fdir_pf_active_filters = 0; 5560 } 5561 5562 /** 5563 * i40e_close - Disables a network interface 5564 * @netdev: network interface device structure 5565 * 5566 * The close entry point is called when an interface is de-activated 5567 * by the OS. The hardware is still under the driver's control, but 5568 * this netdev interface is disabled. 5569 * 5570 * Returns 0, this is not allowed to fail 5571 **/ 5572 int i40e_close(struct net_device *netdev) 5573 { 5574 struct i40e_netdev_priv *np = netdev_priv(netdev); 5575 struct i40e_vsi *vsi = np->vsi; 5576 5577 i40e_vsi_close(vsi); 5578 5579 return 0; 5580 } 5581 5582 /** 5583 * i40e_do_reset - Start a PF or Core Reset sequence 5584 * @pf: board private structure 5585 * @reset_flags: which reset is requested 5586 * 5587 * The essential difference in resets is that the PF Reset 5588 * doesn't clear the packet buffers, doesn't reset the PE 5589 * firmware, and doesn't bother the other PFs on the chip. 5590 **/ 5591 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags) 5592 { 5593 u32 val; 5594 5595 WARN_ON(in_interrupt()); 5596 5597 5598 /* do the biggest reset indicated */ 5599 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 5600 5601 /* Request a Global Reset 5602 * 5603 * This will start the chip's countdown to the actual full 5604 * chip reset event, and a warning interrupt to be sent 5605 * to all PFs, including the requestor. Our handler 5606 * for the warning interrupt will deal with the shutdown 5607 * and recovery of the switch setup. 5608 */ 5609 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 5610 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5611 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 5612 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5613 5614 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 5615 5616 /* Request a Core Reset 5617 * 5618 * Same as Global Reset, except does *not* include the MAC/PHY 5619 */ 5620 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 5621 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5622 val |= I40E_GLGEN_RTRIG_CORER_MASK; 5623 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5624 i40e_flush(&pf->hw); 5625 5626 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) { 5627 5628 /* Request a PF Reset 5629 * 5630 * Resets only the PF-specific registers 5631 * 5632 * This goes directly to the tear-down and rebuild of 5633 * the switch, since we need to do all the recovery as 5634 * for the Core Reset. 5635 */ 5636 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 5637 i40e_handle_reset_warning(pf); 5638 5639 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 5640 int v; 5641 5642 /* Find the VSI(s) that requested a re-init */ 5643 dev_info(&pf->pdev->dev, 5644 "VSI reinit requested\n"); 5645 for (v = 0; v < pf->num_alloc_vsi; v++) { 5646 struct i40e_vsi *vsi = pf->vsi[v]; 5647 5648 if (vsi != NULL && 5649 test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) { 5650 i40e_vsi_reinit_locked(pf->vsi[v]); 5651 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state); 5652 } 5653 } 5654 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 5655 int v; 5656 5657 /* Find the VSI(s) that needs to be brought down */ 5658 dev_info(&pf->pdev->dev, "VSI down requested\n"); 5659 for (v = 0; v < pf->num_alloc_vsi; v++) { 5660 struct i40e_vsi *vsi = pf->vsi[v]; 5661 5662 if (vsi != NULL && 5663 test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) { 5664 set_bit(__I40E_DOWN, &vsi->state); 5665 i40e_down(vsi); 5666 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state); 5667 } 5668 } 5669 } else { 5670 dev_info(&pf->pdev->dev, 5671 "bad reset request 0x%08x\n", reset_flags); 5672 } 5673 } 5674 5675 #ifdef CONFIG_I40E_DCB 5676 /** 5677 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 5678 * @pf: board private structure 5679 * @old_cfg: current DCB config 5680 * @new_cfg: new DCB config 5681 **/ 5682 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 5683 struct i40e_dcbx_config *old_cfg, 5684 struct i40e_dcbx_config *new_cfg) 5685 { 5686 bool need_reconfig = false; 5687 5688 /* Check if ETS configuration has changed */ 5689 if (memcmp(&new_cfg->etscfg, 5690 &old_cfg->etscfg, 5691 sizeof(new_cfg->etscfg))) { 5692 /* If Priority Table has changed reconfig is needed */ 5693 if (memcmp(&new_cfg->etscfg.prioritytable, 5694 &old_cfg->etscfg.prioritytable, 5695 sizeof(new_cfg->etscfg.prioritytable))) { 5696 need_reconfig = true; 5697 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 5698 } 5699 5700 if (memcmp(&new_cfg->etscfg.tcbwtable, 5701 &old_cfg->etscfg.tcbwtable, 5702 sizeof(new_cfg->etscfg.tcbwtable))) 5703 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 5704 5705 if (memcmp(&new_cfg->etscfg.tsatable, 5706 &old_cfg->etscfg.tsatable, 5707 sizeof(new_cfg->etscfg.tsatable))) 5708 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 5709 } 5710 5711 /* Check if PFC configuration has changed */ 5712 if (memcmp(&new_cfg->pfc, 5713 &old_cfg->pfc, 5714 sizeof(new_cfg->pfc))) { 5715 need_reconfig = true; 5716 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 5717 } 5718 5719 /* Check if APP Table has changed */ 5720 if (memcmp(&new_cfg->app, 5721 &old_cfg->app, 5722 sizeof(new_cfg->app))) { 5723 need_reconfig = true; 5724 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 5725 } 5726 5727 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 5728 return need_reconfig; 5729 } 5730 5731 /** 5732 * i40e_handle_lldp_event - Handle LLDP Change MIB event 5733 * @pf: board private structure 5734 * @e: event info posted on ARQ 5735 **/ 5736 static int i40e_handle_lldp_event(struct i40e_pf *pf, 5737 struct i40e_arq_event_info *e) 5738 { 5739 struct i40e_aqc_lldp_get_mib *mib = 5740 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 5741 struct i40e_hw *hw = &pf->hw; 5742 struct i40e_dcbx_config tmp_dcbx_cfg; 5743 bool need_reconfig = false; 5744 int ret = 0; 5745 u8 type; 5746 5747 /* Not DCB capable or capability disabled */ 5748 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 5749 return ret; 5750 5751 /* Ignore if event is not for Nearest Bridge */ 5752 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 5753 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 5754 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 5755 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 5756 return ret; 5757 5758 /* Check MIB Type and return if event for Remote MIB update */ 5759 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 5760 dev_dbg(&pf->pdev->dev, 5761 "LLDP event mib type %s\n", type ? "remote" : "local"); 5762 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 5763 /* Update the remote cached instance and return */ 5764 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 5765 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 5766 &hw->remote_dcbx_config); 5767 goto exit; 5768 } 5769 5770 /* Store the old configuration */ 5771 tmp_dcbx_cfg = hw->local_dcbx_config; 5772 5773 /* Reset the old DCBx configuration data */ 5774 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 5775 /* Get updated DCBX data from firmware */ 5776 ret = i40e_get_dcb_config(&pf->hw); 5777 if (ret) { 5778 dev_info(&pf->pdev->dev, 5779 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 5780 i40e_stat_str(&pf->hw, ret), 5781 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5782 goto exit; 5783 } 5784 5785 /* No change detected in DCBX configs */ 5786 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 5787 sizeof(tmp_dcbx_cfg))) { 5788 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 5789 goto exit; 5790 } 5791 5792 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 5793 &hw->local_dcbx_config); 5794 5795 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 5796 5797 if (!need_reconfig) 5798 goto exit; 5799 5800 /* Enable DCB tagging only when more than one TC */ 5801 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5802 pf->flags |= I40E_FLAG_DCB_ENABLED; 5803 else 5804 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5805 5806 set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5807 /* Reconfiguration needed quiesce all VSIs */ 5808 i40e_pf_quiesce_all_vsi(pf); 5809 5810 /* Changes in configuration update VEB/VSI */ 5811 i40e_dcb_reconfigure(pf); 5812 5813 ret = i40e_resume_port_tx(pf); 5814 5815 clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5816 /* In case of error no point in resuming VSIs */ 5817 if (ret) 5818 goto exit; 5819 5820 /* Wait for the PF's queues to be disabled */ 5821 ret = i40e_pf_wait_queues_disabled(pf); 5822 if (ret) { 5823 /* Schedule PF reset to recover */ 5824 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5825 i40e_service_event_schedule(pf); 5826 } else { 5827 i40e_pf_unquiesce_all_vsi(pf); 5828 /* Notify the client for the DCB changes */ 5829 i40e_notify_client_of_l2_param_changes(pf->vsi[pf->lan_vsi]); 5830 } 5831 5832 exit: 5833 return ret; 5834 } 5835 #endif /* CONFIG_I40E_DCB */ 5836 5837 /** 5838 * i40e_do_reset_safe - Protected reset path for userland calls. 5839 * @pf: board private structure 5840 * @reset_flags: which reset is requested 5841 * 5842 **/ 5843 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 5844 { 5845 rtnl_lock(); 5846 i40e_do_reset(pf, reset_flags); 5847 rtnl_unlock(); 5848 } 5849 5850 /** 5851 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 5852 * @pf: board private structure 5853 * @e: event info posted on ARQ 5854 * 5855 * Handler for LAN Queue Overflow Event generated by the firmware for PF 5856 * and VF queues 5857 **/ 5858 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 5859 struct i40e_arq_event_info *e) 5860 { 5861 struct i40e_aqc_lan_overflow *data = 5862 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 5863 u32 queue = le32_to_cpu(data->prtdcb_rupto); 5864 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 5865 struct i40e_hw *hw = &pf->hw; 5866 struct i40e_vf *vf; 5867 u16 vf_id; 5868 5869 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 5870 queue, qtx_ctl); 5871 5872 /* Queue belongs to VF, find the VF and issue VF reset */ 5873 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 5874 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 5875 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 5876 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 5877 vf_id -= hw->func_caps.vf_base_id; 5878 vf = &pf->vf[vf_id]; 5879 i40e_vc_notify_vf_reset(vf); 5880 /* Allow VF to process pending reset notification */ 5881 msleep(20); 5882 i40e_reset_vf(vf, false); 5883 } 5884 } 5885 5886 /** 5887 * i40e_service_event_complete - Finish up the service event 5888 * @pf: board private structure 5889 **/ 5890 static void i40e_service_event_complete(struct i40e_pf *pf) 5891 { 5892 WARN_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state)); 5893 5894 /* flush memory to make sure state is correct before next watchog */ 5895 smp_mb__before_atomic(); 5896 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 5897 } 5898 5899 /** 5900 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 5901 * @pf: board private structure 5902 **/ 5903 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 5904 { 5905 u32 val, fcnt_prog; 5906 5907 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 5908 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 5909 return fcnt_prog; 5910 } 5911 5912 /** 5913 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 5914 * @pf: board private structure 5915 **/ 5916 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 5917 { 5918 u32 val, fcnt_prog; 5919 5920 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 5921 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 5922 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 5923 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 5924 return fcnt_prog; 5925 } 5926 5927 /** 5928 * i40e_get_global_fd_count - Get total FD filters programmed on device 5929 * @pf: board private structure 5930 **/ 5931 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 5932 { 5933 u32 val, fcnt_prog; 5934 5935 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 5936 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 5937 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 5938 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 5939 return fcnt_prog; 5940 } 5941 5942 /** 5943 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 5944 * @pf: board private structure 5945 **/ 5946 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 5947 { 5948 struct i40e_fdir_filter *filter; 5949 u32 fcnt_prog, fcnt_avail; 5950 struct hlist_node *node; 5951 5952 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 5953 return; 5954 5955 /* Check if, FD SB or ATR was auto disabled and if there is enough room 5956 * to re-enable 5957 */ 5958 fcnt_prog = i40e_get_global_fd_count(pf); 5959 fcnt_avail = pf->fdir_pf_filter_count; 5960 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 5961 (pf->fd_add_err == 0) || 5962 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 5963 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 5964 (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) { 5965 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED; 5966 if (I40E_DEBUG_FD & pf->hw.debug_mask) 5967 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 5968 } 5969 } 5970 5971 /* Wait for some more space to be available to turn on ATR. We also 5972 * must check that no existing ntuple rules for TCP are in effect 5973 */ 5974 if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) { 5975 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 5976 (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED) && 5977 (pf->fd_tcp_rule == 0)) { 5978 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 5979 if (I40E_DEBUG_FD & pf->hw.debug_mask) 5980 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 5981 } 5982 } 5983 5984 /* if hw had a problem adding a filter, delete it */ 5985 if (pf->fd_inv > 0) { 5986 hlist_for_each_entry_safe(filter, node, 5987 &pf->fdir_filter_list, fdir_node) { 5988 if (filter->fd_id == pf->fd_inv) { 5989 hlist_del(&filter->fdir_node); 5990 kfree(filter); 5991 pf->fdir_pf_active_filters--; 5992 } 5993 } 5994 } 5995 } 5996 5997 #define I40E_MIN_FD_FLUSH_INTERVAL 10 5998 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 5999 /** 6000 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 6001 * @pf: board private structure 6002 **/ 6003 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 6004 { 6005 unsigned long min_flush_time; 6006 int flush_wait_retry = 50; 6007 bool disable_atr = false; 6008 int fd_room; 6009 int reg; 6010 6011 if (!time_after(jiffies, pf->fd_flush_timestamp + 6012 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 6013 return; 6014 6015 /* If the flush is happening too quick and we have mostly SB rules we 6016 * should not re-enable ATR for some time. 6017 */ 6018 min_flush_time = pf->fd_flush_timestamp + 6019 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 6020 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 6021 6022 if (!(time_after(jiffies, min_flush_time)) && 6023 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 6024 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6025 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 6026 disable_atr = true; 6027 } 6028 6029 pf->fd_flush_timestamp = jiffies; 6030 pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED; 6031 /* flush all filters */ 6032 wr32(&pf->hw, I40E_PFQF_CTL_1, 6033 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 6034 i40e_flush(&pf->hw); 6035 pf->fd_flush_cnt++; 6036 pf->fd_add_err = 0; 6037 do { 6038 /* Check FD flush status every 5-6msec */ 6039 usleep_range(5000, 6000); 6040 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 6041 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 6042 break; 6043 } while (flush_wait_retry--); 6044 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 6045 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 6046 } else { 6047 /* replay sideband filters */ 6048 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 6049 if (!disable_atr) 6050 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 6051 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state); 6052 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6053 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 6054 } 6055 } 6056 6057 /** 6058 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 6059 * @pf: board private structure 6060 **/ 6061 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 6062 { 6063 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 6064 } 6065 6066 /* We can see up to 256 filter programming desc in transit if the filters are 6067 * being applied really fast; before we see the first 6068 * filter miss error on Rx queue 0. Accumulating enough error messages before 6069 * reacting will make sure we don't cause flush too often. 6070 */ 6071 #define I40E_MAX_FD_PROGRAM_ERROR 256 6072 6073 /** 6074 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 6075 * @pf: board private structure 6076 **/ 6077 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 6078 { 6079 6080 /* if interface is down do nothing */ 6081 if (test_bit(__I40E_DOWN, &pf->state)) 6082 return; 6083 6084 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 6085 i40e_fdir_flush_and_replay(pf); 6086 6087 i40e_fdir_check_and_reenable(pf); 6088 6089 } 6090 6091 /** 6092 * i40e_vsi_link_event - notify VSI of a link event 6093 * @vsi: vsi to be notified 6094 * @link_up: link up or down 6095 **/ 6096 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 6097 { 6098 if (!vsi || test_bit(__I40E_DOWN, &vsi->state)) 6099 return; 6100 6101 switch (vsi->type) { 6102 case I40E_VSI_MAIN: 6103 #ifdef I40E_FCOE 6104 case I40E_VSI_FCOE: 6105 #endif 6106 if (!vsi->netdev || !vsi->netdev_registered) 6107 break; 6108 6109 if (link_up) { 6110 netif_carrier_on(vsi->netdev); 6111 netif_tx_wake_all_queues(vsi->netdev); 6112 } else { 6113 netif_carrier_off(vsi->netdev); 6114 netif_tx_stop_all_queues(vsi->netdev); 6115 } 6116 break; 6117 6118 case I40E_VSI_SRIOV: 6119 case I40E_VSI_VMDQ2: 6120 case I40E_VSI_CTRL: 6121 case I40E_VSI_IWARP: 6122 case I40E_VSI_MIRROR: 6123 default: 6124 /* there is no notification for other VSIs */ 6125 break; 6126 } 6127 } 6128 6129 /** 6130 * i40e_veb_link_event - notify elements on the veb of a link event 6131 * @veb: veb to be notified 6132 * @link_up: link up or down 6133 **/ 6134 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 6135 { 6136 struct i40e_pf *pf; 6137 int i; 6138 6139 if (!veb || !veb->pf) 6140 return; 6141 pf = veb->pf; 6142 6143 /* depth first... */ 6144 for (i = 0; i < I40E_MAX_VEB; i++) 6145 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 6146 i40e_veb_link_event(pf->veb[i], link_up); 6147 6148 /* ... now the local VSIs */ 6149 for (i = 0; i < pf->num_alloc_vsi; i++) 6150 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 6151 i40e_vsi_link_event(pf->vsi[i], link_up); 6152 } 6153 6154 /** 6155 * i40e_link_event - Update netif_carrier status 6156 * @pf: board private structure 6157 **/ 6158 static void i40e_link_event(struct i40e_pf *pf) 6159 { 6160 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6161 u8 new_link_speed, old_link_speed; 6162 i40e_status status; 6163 bool new_link, old_link; 6164 6165 /* save off old link status information */ 6166 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 6167 6168 /* set this to force the get_link_status call to refresh state */ 6169 pf->hw.phy.get_link_info = true; 6170 6171 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 6172 6173 status = i40e_get_link_status(&pf->hw, &new_link); 6174 if (status) { 6175 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 6176 status); 6177 return; 6178 } 6179 6180 old_link_speed = pf->hw.phy.link_info_old.link_speed; 6181 new_link_speed = pf->hw.phy.link_info.link_speed; 6182 6183 if (new_link == old_link && 6184 new_link_speed == old_link_speed && 6185 (test_bit(__I40E_DOWN, &vsi->state) || 6186 new_link == netif_carrier_ok(vsi->netdev))) 6187 return; 6188 6189 if (!test_bit(__I40E_DOWN, &vsi->state)) 6190 i40e_print_link_message(vsi, new_link); 6191 6192 /* Notify the base of the switch tree connected to 6193 * the link. Floating VEBs are not notified. 6194 */ 6195 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 6196 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 6197 else 6198 i40e_vsi_link_event(vsi, new_link); 6199 6200 if (pf->vf) 6201 i40e_vc_notify_link_state(pf); 6202 6203 if (pf->flags & I40E_FLAG_PTP) 6204 i40e_ptp_set_increment(pf); 6205 } 6206 6207 /** 6208 * i40e_watchdog_subtask - periodic checks not using event driven response 6209 * @pf: board private structure 6210 **/ 6211 static void i40e_watchdog_subtask(struct i40e_pf *pf) 6212 { 6213 int i; 6214 6215 /* if interface is down do nothing */ 6216 if (test_bit(__I40E_DOWN, &pf->state) || 6217 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6218 return; 6219 6220 /* make sure we don't do these things too often */ 6221 if (time_before(jiffies, (pf->service_timer_previous + 6222 pf->service_timer_period))) 6223 return; 6224 pf->service_timer_previous = jiffies; 6225 6226 if (pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) 6227 i40e_link_event(pf); 6228 6229 /* Update the stats for active netdevs so the network stack 6230 * can look at updated numbers whenever it cares to 6231 */ 6232 for (i = 0; i < pf->num_alloc_vsi; i++) 6233 if (pf->vsi[i] && pf->vsi[i]->netdev) 6234 i40e_update_stats(pf->vsi[i]); 6235 6236 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 6237 /* Update the stats for the active switching components */ 6238 for (i = 0; i < I40E_MAX_VEB; i++) 6239 if (pf->veb[i]) 6240 i40e_update_veb_stats(pf->veb[i]); 6241 } 6242 6243 i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]); 6244 } 6245 6246 /** 6247 * i40e_reset_subtask - Set up for resetting the device and driver 6248 * @pf: board private structure 6249 **/ 6250 static void i40e_reset_subtask(struct i40e_pf *pf) 6251 { 6252 u32 reset_flags = 0; 6253 6254 rtnl_lock(); 6255 if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) { 6256 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 6257 clear_bit(__I40E_REINIT_REQUESTED, &pf->state); 6258 } 6259 if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) { 6260 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 6261 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 6262 } 6263 if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) { 6264 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 6265 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 6266 } 6267 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) { 6268 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 6269 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 6270 } 6271 if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) { 6272 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 6273 clear_bit(__I40E_DOWN_REQUESTED, &pf->state); 6274 } 6275 6276 /* If there's a recovery already waiting, it takes 6277 * precedence before starting a new reset sequence. 6278 */ 6279 if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) { 6280 i40e_handle_reset_warning(pf); 6281 goto unlock; 6282 } 6283 6284 /* If we're already down or resetting, just bail */ 6285 if (reset_flags && 6286 !test_bit(__I40E_DOWN, &pf->state) && 6287 !test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6288 i40e_do_reset(pf, reset_flags); 6289 6290 unlock: 6291 rtnl_unlock(); 6292 } 6293 6294 /** 6295 * i40e_handle_link_event - Handle link event 6296 * @pf: board private structure 6297 * @e: event info posted on ARQ 6298 **/ 6299 static void i40e_handle_link_event(struct i40e_pf *pf, 6300 struct i40e_arq_event_info *e) 6301 { 6302 struct i40e_aqc_get_link_status *status = 6303 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 6304 6305 /* Do a new status request to re-enable LSE reporting 6306 * and load new status information into the hw struct 6307 * This completely ignores any state information 6308 * in the ARQ event info, instead choosing to always 6309 * issue the AQ update link status command. 6310 */ 6311 i40e_link_event(pf); 6312 6313 /* check for unqualified module, if link is down */ 6314 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 6315 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 6316 (!(status->link_info & I40E_AQ_LINK_UP))) 6317 dev_err(&pf->pdev->dev, 6318 "The driver failed to link because an unqualified module was detected.\n"); 6319 } 6320 6321 /** 6322 * i40e_clean_adminq_subtask - Clean the AdminQ rings 6323 * @pf: board private structure 6324 **/ 6325 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 6326 { 6327 struct i40e_arq_event_info event; 6328 struct i40e_hw *hw = &pf->hw; 6329 u16 pending, i = 0; 6330 i40e_status ret; 6331 u16 opcode; 6332 u32 oldval; 6333 u32 val; 6334 6335 /* Do not run clean AQ when PF reset fails */ 6336 if (test_bit(__I40E_RESET_FAILED, &pf->state)) 6337 return; 6338 6339 /* check for error indications */ 6340 val = rd32(&pf->hw, pf->hw.aq.arq.len); 6341 oldval = val; 6342 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 6343 if (hw->debug_mask & I40E_DEBUG_AQ) 6344 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 6345 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 6346 } 6347 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 6348 if (hw->debug_mask & I40E_DEBUG_AQ) 6349 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 6350 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 6351 pf->arq_overflows++; 6352 } 6353 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 6354 if (hw->debug_mask & I40E_DEBUG_AQ) 6355 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 6356 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 6357 } 6358 if (oldval != val) 6359 wr32(&pf->hw, pf->hw.aq.arq.len, val); 6360 6361 val = rd32(&pf->hw, pf->hw.aq.asq.len); 6362 oldval = val; 6363 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 6364 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6365 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 6366 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 6367 } 6368 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 6369 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6370 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 6371 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 6372 } 6373 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 6374 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6375 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 6376 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 6377 } 6378 if (oldval != val) 6379 wr32(&pf->hw, pf->hw.aq.asq.len, val); 6380 6381 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 6382 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 6383 if (!event.msg_buf) 6384 return; 6385 6386 do { 6387 ret = i40e_clean_arq_element(hw, &event, &pending); 6388 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 6389 break; 6390 else if (ret) { 6391 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 6392 break; 6393 } 6394 6395 opcode = le16_to_cpu(event.desc.opcode); 6396 switch (opcode) { 6397 6398 case i40e_aqc_opc_get_link_status: 6399 i40e_handle_link_event(pf, &event); 6400 break; 6401 case i40e_aqc_opc_send_msg_to_pf: 6402 ret = i40e_vc_process_vf_msg(pf, 6403 le16_to_cpu(event.desc.retval), 6404 le32_to_cpu(event.desc.cookie_high), 6405 le32_to_cpu(event.desc.cookie_low), 6406 event.msg_buf, 6407 event.msg_len); 6408 break; 6409 case i40e_aqc_opc_lldp_update_mib: 6410 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 6411 #ifdef CONFIG_I40E_DCB 6412 rtnl_lock(); 6413 ret = i40e_handle_lldp_event(pf, &event); 6414 rtnl_unlock(); 6415 #endif /* CONFIG_I40E_DCB */ 6416 break; 6417 case i40e_aqc_opc_event_lan_overflow: 6418 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 6419 i40e_handle_lan_overflow_event(pf, &event); 6420 break; 6421 case i40e_aqc_opc_send_msg_to_peer: 6422 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 6423 break; 6424 case i40e_aqc_opc_nvm_erase: 6425 case i40e_aqc_opc_nvm_update: 6426 case i40e_aqc_opc_oem_post_update: 6427 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 6428 "ARQ NVM operation 0x%04x completed\n", 6429 opcode); 6430 break; 6431 default: 6432 dev_info(&pf->pdev->dev, 6433 "ARQ: Unknown event 0x%04x ignored\n", 6434 opcode); 6435 break; 6436 } 6437 } while (pending && (i++ < pf->adminq_work_limit)); 6438 6439 clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 6440 /* re-enable Admin queue interrupt cause */ 6441 val = rd32(hw, I40E_PFINT_ICR0_ENA); 6442 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6443 wr32(hw, I40E_PFINT_ICR0_ENA, val); 6444 i40e_flush(hw); 6445 6446 kfree(event.msg_buf); 6447 } 6448 6449 /** 6450 * i40e_verify_eeprom - make sure eeprom is good to use 6451 * @pf: board private structure 6452 **/ 6453 static void i40e_verify_eeprom(struct i40e_pf *pf) 6454 { 6455 int err; 6456 6457 err = i40e_diag_eeprom_test(&pf->hw); 6458 if (err) { 6459 /* retry in case of garbage read */ 6460 err = i40e_diag_eeprom_test(&pf->hw); 6461 if (err) { 6462 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 6463 err); 6464 set_bit(__I40E_BAD_EEPROM, &pf->state); 6465 } 6466 } 6467 6468 if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) { 6469 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 6470 clear_bit(__I40E_BAD_EEPROM, &pf->state); 6471 } 6472 } 6473 6474 /** 6475 * i40e_enable_pf_switch_lb 6476 * @pf: pointer to the PF structure 6477 * 6478 * enable switch loop back or die - no point in a return value 6479 **/ 6480 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 6481 { 6482 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6483 struct i40e_vsi_context ctxt; 6484 int ret; 6485 6486 ctxt.seid = pf->main_vsi_seid; 6487 ctxt.pf_num = pf->hw.pf_id; 6488 ctxt.vf_num = 0; 6489 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6490 if (ret) { 6491 dev_info(&pf->pdev->dev, 6492 "couldn't get PF vsi config, err %s aq_err %s\n", 6493 i40e_stat_str(&pf->hw, ret), 6494 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6495 return; 6496 } 6497 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6498 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6499 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6500 6501 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6502 if (ret) { 6503 dev_info(&pf->pdev->dev, 6504 "update vsi switch failed, err %s aq_err %s\n", 6505 i40e_stat_str(&pf->hw, ret), 6506 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6507 } 6508 } 6509 6510 /** 6511 * i40e_disable_pf_switch_lb 6512 * @pf: pointer to the PF structure 6513 * 6514 * disable switch loop back or die - no point in a return value 6515 **/ 6516 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 6517 { 6518 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6519 struct i40e_vsi_context ctxt; 6520 int ret; 6521 6522 ctxt.seid = pf->main_vsi_seid; 6523 ctxt.pf_num = pf->hw.pf_id; 6524 ctxt.vf_num = 0; 6525 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6526 if (ret) { 6527 dev_info(&pf->pdev->dev, 6528 "couldn't get PF vsi config, err %s aq_err %s\n", 6529 i40e_stat_str(&pf->hw, ret), 6530 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6531 return; 6532 } 6533 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6534 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6535 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6536 6537 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6538 if (ret) { 6539 dev_info(&pf->pdev->dev, 6540 "update vsi switch failed, err %s aq_err %s\n", 6541 i40e_stat_str(&pf->hw, ret), 6542 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6543 } 6544 } 6545 6546 /** 6547 * i40e_config_bridge_mode - Configure the HW bridge mode 6548 * @veb: pointer to the bridge instance 6549 * 6550 * Configure the loop back mode for the LAN VSI that is downlink to the 6551 * specified HW bridge instance. It is expected this function is called 6552 * when a new HW bridge is instantiated. 6553 **/ 6554 static void i40e_config_bridge_mode(struct i40e_veb *veb) 6555 { 6556 struct i40e_pf *pf = veb->pf; 6557 6558 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 6559 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 6560 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 6561 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 6562 i40e_disable_pf_switch_lb(pf); 6563 else 6564 i40e_enable_pf_switch_lb(pf); 6565 } 6566 6567 /** 6568 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 6569 * @veb: pointer to the VEB instance 6570 * 6571 * This is a recursive function that first builds the attached VSIs then 6572 * recurses in to build the next layer of VEB. We track the connections 6573 * through our own index numbers because the seid's from the HW could 6574 * change across the reset. 6575 **/ 6576 static int i40e_reconstitute_veb(struct i40e_veb *veb) 6577 { 6578 struct i40e_vsi *ctl_vsi = NULL; 6579 struct i40e_pf *pf = veb->pf; 6580 int v, veb_idx; 6581 int ret; 6582 6583 /* build VSI that owns this VEB, temporarily attached to base VEB */ 6584 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 6585 if (pf->vsi[v] && 6586 pf->vsi[v]->veb_idx == veb->idx && 6587 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 6588 ctl_vsi = pf->vsi[v]; 6589 break; 6590 } 6591 } 6592 if (!ctl_vsi) { 6593 dev_info(&pf->pdev->dev, 6594 "missing owner VSI for veb_idx %d\n", veb->idx); 6595 ret = -ENOENT; 6596 goto end_reconstitute; 6597 } 6598 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 6599 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6600 ret = i40e_add_vsi(ctl_vsi); 6601 if (ret) { 6602 dev_info(&pf->pdev->dev, 6603 "rebuild of veb_idx %d owner VSI failed: %d\n", 6604 veb->idx, ret); 6605 goto end_reconstitute; 6606 } 6607 i40e_vsi_reset_stats(ctl_vsi); 6608 6609 /* create the VEB in the switch and move the VSI onto the VEB */ 6610 ret = i40e_add_veb(veb, ctl_vsi); 6611 if (ret) 6612 goto end_reconstitute; 6613 6614 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 6615 veb->bridge_mode = BRIDGE_MODE_VEB; 6616 else 6617 veb->bridge_mode = BRIDGE_MODE_VEPA; 6618 i40e_config_bridge_mode(veb); 6619 6620 /* create the remaining VSIs attached to this VEB */ 6621 for (v = 0; v < pf->num_alloc_vsi; v++) { 6622 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 6623 continue; 6624 6625 if (pf->vsi[v]->veb_idx == veb->idx) { 6626 struct i40e_vsi *vsi = pf->vsi[v]; 6627 6628 vsi->uplink_seid = veb->seid; 6629 ret = i40e_add_vsi(vsi); 6630 if (ret) { 6631 dev_info(&pf->pdev->dev, 6632 "rebuild of vsi_idx %d failed: %d\n", 6633 v, ret); 6634 goto end_reconstitute; 6635 } 6636 i40e_vsi_reset_stats(vsi); 6637 } 6638 } 6639 6640 /* create any VEBs attached to this VEB - RECURSION */ 6641 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 6642 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 6643 pf->veb[veb_idx]->uplink_seid = veb->seid; 6644 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 6645 if (ret) 6646 break; 6647 } 6648 } 6649 6650 end_reconstitute: 6651 return ret; 6652 } 6653 6654 /** 6655 * i40e_get_capabilities - get info about the HW 6656 * @pf: the PF struct 6657 **/ 6658 static int i40e_get_capabilities(struct i40e_pf *pf) 6659 { 6660 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 6661 u16 data_size; 6662 int buf_len; 6663 int err; 6664 6665 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 6666 do { 6667 cap_buf = kzalloc(buf_len, GFP_KERNEL); 6668 if (!cap_buf) 6669 return -ENOMEM; 6670 6671 /* this loads the data into the hw struct for us */ 6672 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 6673 &data_size, 6674 i40e_aqc_opc_list_func_capabilities, 6675 NULL); 6676 /* data loaded, buffer no longer needed */ 6677 kfree(cap_buf); 6678 6679 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 6680 /* retry with a larger buffer */ 6681 buf_len = data_size; 6682 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 6683 dev_info(&pf->pdev->dev, 6684 "capability discovery failed, err %s aq_err %s\n", 6685 i40e_stat_str(&pf->hw, err), 6686 i40e_aq_str(&pf->hw, 6687 pf->hw.aq.asq_last_status)); 6688 return -ENODEV; 6689 } 6690 } while (err); 6691 6692 if (pf->hw.debug_mask & I40E_DEBUG_USER) 6693 dev_info(&pf->pdev->dev, 6694 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 6695 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 6696 pf->hw.func_caps.num_msix_vectors, 6697 pf->hw.func_caps.num_msix_vectors_vf, 6698 pf->hw.func_caps.fd_filters_guaranteed, 6699 pf->hw.func_caps.fd_filters_best_effort, 6700 pf->hw.func_caps.num_tx_qp, 6701 pf->hw.func_caps.num_vsis); 6702 6703 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 6704 + pf->hw.func_caps.num_vfs) 6705 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 6706 dev_info(&pf->pdev->dev, 6707 "got num_vsis %d, setting num_vsis to %d\n", 6708 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 6709 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 6710 } 6711 6712 return 0; 6713 } 6714 6715 static int i40e_vsi_clear(struct i40e_vsi *vsi); 6716 6717 /** 6718 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 6719 * @pf: board private structure 6720 **/ 6721 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 6722 { 6723 struct i40e_vsi *vsi; 6724 6725 /* quick workaround for an NVM issue that leaves a critical register 6726 * uninitialized 6727 */ 6728 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 6729 static const u32 hkey[] = { 6730 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 6731 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 6732 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 6733 0x95b3a76d}; 6734 int i; 6735 6736 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 6737 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 6738 } 6739 6740 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6741 return; 6742 6743 /* find existing VSI and see if it needs configuring */ 6744 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6745 6746 /* create a new VSI if none exists */ 6747 if (!vsi) { 6748 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 6749 pf->vsi[pf->lan_vsi]->seid, 0); 6750 if (!vsi) { 6751 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 6752 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 6753 return; 6754 } 6755 } 6756 6757 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 6758 } 6759 6760 /** 6761 * i40e_fdir_teardown - release the Flow Director resources 6762 * @pf: board private structure 6763 **/ 6764 static void i40e_fdir_teardown(struct i40e_pf *pf) 6765 { 6766 struct i40e_vsi *vsi; 6767 6768 i40e_fdir_filter_exit(pf); 6769 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6770 if (vsi) 6771 i40e_vsi_release(vsi); 6772 } 6773 6774 /** 6775 * i40e_prep_for_reset - prep for the core to reset 6776 * @pf: board private structure 6777 * 6778 * Close up the VFs and other things in prep for PF Reset. 6779 **/ 6780 static void i40e_prep_for_reset(struct i40e_pf *pf) 6781 { 6782 struct i40e_hw *hw = &pf->hw; 6783 i40e_status ret = 0; 6784 u32 v; 6785 6786 clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 6787 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 6788 return; 6789 if (i40e_check_asq_alive(&pf->hw)) 6790 i40e_vc_notify_reset(pf); 6791 6792 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 6793 6794 /* quiesce the VSIs and their queues that are not already DOWN */ 6795 i40e_pf_quiesce_all_vsi(pf); 6796 6797 for (v = 0; v < pf->num_alloc_vsi; v++) { 6798 if (pf->vsi[v]) 6799 pf->vsi[v]->seid = 0; 6800 } 6801 6802 i40e_shutdown_adminq(&pf->hw); 6803 6804 /* call shutdown HMC */ 6805 if (hw->hmc.hmc_obj) { 6806 ret = i40e_shutdown_lan_hmc(hw); 6807 if (ret) 6808 dev_warn(&pf->pdev->dev, 6809 "shutdown_lan_hmc failed: %d\n", ret); 6810 } 6811 } 6812 6813 /** 6814 * i40e_send_version - update firmware with driver version 6815 * @pf: PF struct 6816 */ 6817 static void i40e_send_version(struct i40e_pf *pf) 6818 { 6819 struct i40e_driver_version dv; 6820 6821 dv.major_version = DRV_VERSION_MAJOR; 6822 dv.minor_version = DRV_VERSION_MINOR; 6823 dv.build_version = DRV_VERSION_BUILD; 6824 dv.subbuild_version = 0; 6825 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 6826 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 6827 } 6828 6829 /** 6830 * i40e_reset_and_rebuild - reset and rebuild using a saved config 6831 * @pf: board private structure 6832 * @reinit: if the Main VSI needs to re-initialized. 6833 **/ 6834 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit) 6835 { 6836 struct i40e_hw *hw = &pf->hw; 6837 u8 set_fc_aq_fail = 0; 6838 i40e_status ret; 6839 u32 val; 6840 u32 v; 6841 6842 /* Now we wait for GRST to settle out. 6843 * We don't have to delete the VEBs or VSIs from the hw switch 6844 * because the reset will make them disappear. 6845 */ 6846 ret = i40e_pf_reset(hw); 6847 if (ret) { 6848 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 6849 set_bit(__I40E_RESET_FAILED, &pf->state); 6850 goto clear_recovery; 6851 } 6852 pf->pfr_count++; 6853 6854 if (test_bit(__I40E_DOWN, &pf->state)) 6855 goto clear_recovery; 6856 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 6857 6858 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 6859 ret = i40e_init_adminq(&pf->hw); 6860 if (ret) { 6861 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 6862 i40e_stat_str(&pf->hw, ret), 6863 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6864 goto clear_recovery; 6865 } 6866 6867 /* re-verify the eeprom if we just had an EMP reset */ 6868 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state)) 6869 i40e_verify_eeprom(pf); 6870 6871 i40e_clear_pxe_mode(hw); 6872 ret = i40e_get_capabilities(pf); 6873 if (ret) 6874 goto end_core_reset; 6875 6876 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 6877 hw->func_caps.num_rx_qp, 6878 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 6879 if (ret) { 6880 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 6881 goto end_core_reset; 6882 } 6883 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 6884 if (ret) { 6885 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 6886 goto end_core_reset; 6887 } 6888 6889 #ifdef CONFIG_I40E_DCB 6890 ret = i40e_init_pf_dcb(pf); 6891 if (ret) { 6892 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 6893 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 6894 /* Continue without DCB enabled */ 6895 } 6896 #endif /* CONFIG_I40E_DCB */ 6897 #ifdef I40E_FCOE 6898 i40e_init_pf_fcoe(pf); 6899 6900 #endif 6901 /* do basic switch setup */ 6902 ret = i40e_setup_pf_switch(pf, reinit); 6903 if (ret) 6904 goto end_core_reset; 6905 6906 /* The driver only wants link up/down and module qualification 6907 * reports from firmware. Note the negative logic. 6908 */ 6909 ret = i40e_aq_set_phy_int_mask(&pf->hw, 6910 ~(I40E_AQ_EVENT_LINK_UPDOWN | 6911 I40E_AQ_EVENT_MEDIA_NA | 6912 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 6913 if (ret) 6914 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 6915 i40e_stat_str(&pf->hw, ret), 6916 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6917 6918 /* make sure our flow control settings are restored */ 6919 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 6920 if (ret) 6921 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 6922 i40e_stat_str(&pf->hw, ret), 6923 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6924 6925 /* Rebuild the VSIs and VEBs that existed before reset. 6926 * They are still in our local switch element arrays, so only 6927 * need to rebuild the switch model in the HW. 6928 * 6929 * If there were VEBs but the reconstitution failed, we'll try 6930 * try to recover minimal use by getting the basic PF VSI working. 6931 */ 6932 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 6933 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 6934 /* find the one VEB connected to the MAC, and find orphans */ 6935 for (v = 0; v < I40E_MAX_VEB; v++) { 6936 if (!pf->veb[v]) 6937 continue; 6938 6939 if (pf->veb[v]->uplink_seid == pf->mac_seid || 6940 pf->veb[v]->uplink_seid == 0) { 6941 ret = i40e_reconstitute_veb(pf->veb[v]); 6942 6943 if (!ret) 6944 continue; 6945 6946 /* If Main VEB failed, we're in deep doodoo, 6947 * so give up rebuilding the switch and set up 6948 * for minimal rebuild of PF VSI. 6949 * If orphan failed, we'll report the error 6950 * but try to keep going. 6951 */ 6952 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 6953 dev_info(&pf->pdev->dev, 6954 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 6955 ret); 6956 pf->vsi[pf->lan_vsi]->uplink_seid 6957 = pf->mac_seid; 6958 break; 6959 } else if (pf->veb[v]->uplink_seid == 0) { 6960 dev_info(&pf->pdev->dev, 6961 "rebuild of orphan VEB failed: %d\n", 6962 ret); 6963 } 6964 } 6965 } 6966 } 6967 6968 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 6969 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 6970 /* no VEB, so rebuild only the Main VSI */ 6971 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 6972 if (ret) { 6973 dev_info(&pf->pdev->dev, 6974 "rebuild of Main VSI failed: %d\n", ret); 6975 goto end_core_reset; 6976 } 6977 } 6978 6979 /* Reconfigure hardware for allowing smaller MSS in the case 6980 * of TSO, so that we avoid the MDD being fired and causing 6981 * a reset in the case of small MSS+TSO. 6982 */ 6983 #define I40E_REG_MSS 0x000E64DC 6984 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 6985 #define I40E_64BYTE_MSS 0x400000 6986 val = rd32(hw, I40E_REG_MSS); 6987 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 6988 val &= ~I40E_REG_MSS_MIN_MASK; 6989 val |= I40E_64BYTE_MSS; 6990 wr32(hw, I40E_REG_MSS, val); 6991 } 6992 6993 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 6994 msleep(75); 6995 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 6996 if (ret) 6997 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 6998 i40e_stat_str(&pf->hw, ret), 6999 i40e_aq_str(&pf->hw, 7000 pf->hw.aq.asq_last_status)); 7001 } 7002 /* reinit the misc interrupt */ 7003 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7004 ret = i40e_setup_misc_vector(pf); 7005 7006 /* Add a filter to drop all Flow control frames from any VSI from being 7007 * transmitted. By doing so we stop a malicious VF from sending out 7008 * PAUSE or PFC frames and potentially controlling traffic for other 7009 * PF/VF VSIs. 7010 * The FW can still send Flow control frames if enabled. 7011 */ 7012 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 7013 pf->main_vsi_seid); 7014 7015 /* restart the VSIs that were rebuilt and running before the reset */ 7016 i40e_pf_unquiesce_all_vsi(pf); 7017 7018 if (pf->num_alloc_vfs) { 7019 for (v = 0; v < pf->num_alloc_vfs; v++) 7020 i40e_reset_vf(&pf->vf[v], true); 7021 } 7022 7023 /* tell the firmware that we're starting */ 7024 i40e_send_version(pf); 7025 7026 end_core_reset: 7027 clear_bit(__I40E_RESET_FAILED, &pf->state); 7028 clear_recovery: 7029 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state); 7030 } 7031 7032 /** 7033 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 7034 * @pf: board private structure 7035 * 7036 * Close up the VFs and other things in prep for a Core Reset, 7037 * then get ready to rebuild the world. 7038 **/ 7039 static void i40e_handle_reset_warning(struct i40e_pf *pf) 7040 { 7041 i40e_prep_for_reset(pf); 7042 i40e_reset_and_rebuild(pf, false); 7043 } 7044 7045 /** 7046 * i40e_handle_mdd_event 7047 * @pf: pointer to the PF structure 7048 * 7049 * Called from the MDD irq handler to identify possibly malicious vfs 7050 **/ 7051 static void i40e_handle_mdd_event(struct i40e_pf *pf) 7052 { 7053 struct i40e_hw *hw = &pf->hw; 7054 bool mdd_detected = false; 7055 bool pf_mdd_detected = false; 7056 struct i40e_vf *vf; 7057 u32 reg; 7058 int i; 7059 7060 if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)) 7061 return; 7062 7063 /* find what triggered the MDD event */ 7064 reg = rd32(hw, I40E_GL_MDET_TX); 7065 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 7066 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 7067 I40E_GL_MDET_TX_PF_NUM_SHIFT; 7068 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 7069 I40E_GL_MDET_TX_VF_NUM_SHIFT; 7070 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 7071 I40E_GL_MDET_TX_EVENT_SHIFT; 7072 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 7073 I40E_GL_MDET_TX_QUEUE_SHIFT) - 7074 pf->hw.func_caps.base_queue; 7075 if (netif_msg_tx_err(pf)) 7076 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 7077 event, queue, pf_num, vf_num); 7078 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 7079 mdd_detected = true; 7080 } 7081 reg = rd32(hw, I40E_GL_MDET_RX); 7082 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 7083 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 7084 I40E_GL_MDET_RX_FUNCTION_SHIFT; 7085 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 7086 I40E_GL_MDET_RX_EVENT_SHIFT; 7087 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 7088 I40E_GL_MDET_RX_QUEUE_SHIFT) - 7089 pf->hw.func_caps.base_queue; 7090 if (netif_msg_rx_err(pf)) 7091 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 7092 event, queue, func); 7093 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 7094 mdd_detected = true; 7095 } 7096 7097 if (mdd_detected) { 7098 reg = rd32(hw, I40E_PF_MDET_TX); 7099 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 7100 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 7101 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 7102 pf_mdd_detected = true; 7103 } 7104 reg = rd32(hw, I40E_PF_MDET_RX); 7105 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 7106 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 7107 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 7108 pf_mdd_detected = true; 7109 } 7110 /* Queue belongs to the PF, initiate a reset */ 7111 if (pf_mdd_detected) { 7112 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 7113 i40e_service_event_schedule(pf); 7114 } 7115 } 7116 7117 /* see if one of the VFs needs its hand slapped */ 7118 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 7119 vf = &(pf->vf[i]); 7120 reg = rd32(hw, I40E_VP_MDET_TX(i)); 7121 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 7122 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 7123 vf->num_mdd_events++; 7124 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 7125 i); 7126 } 7127 7128 reg = rd32(hw, I40E_VP_MDET_RX(i)); 7129 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 7130 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 7131 vf->num_mdd_events++; 7132 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 7133 i); 7134 } 7135 7136 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 7137 dev_info(&pf->pdev->dev, 7138 "Too many MDD events on VF %d, disabled\n", i); 7139 dev_info(&pf->pdev->dev, 7140 "Use PF Control I/F to re-enable the VF\n"); 7141 set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states); 7142 } 7143 } 7144 7145 /* re-enable mdd interrupt cause */ 7146 clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 7147 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 7148 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 7149 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 7150 i40e_flush(hw); 7151 } 7152 7153 /** 7154 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 7155 * @pf: board private structure 7156 **/ 7157 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 7158 { 7159 struct i40e_hw *hw = &pf->hw; 7160 i40e_status ret; 7161 __be16 port; 7162 int i; 7163 7164 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 7165 return; 7166 7167 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 7168 7169 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7170 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 7171 pf->pending_udp_bitmap &= ~BIT_ULL(i); 7172 port = pf->udp_ports[i].index; 7173 if (port) 7174 ret = i40e_aq_add_udp_tunnel(hw, port, 7175 pf->udp_ports[i].type, 7176 NULL, NULL); 7177 else 7178 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 7179 7180 if (ret) { 7181 dev_dbg(&pf->pdev->dev, 7182 "%s %s port %d, index %d failed, err %s aq_err %s\n", 7183 pf->udp_ports[i].type ? "vxlan" : "geneve", 7184 port ? "add" : "delete", 7185 ntohs(port), i, 7186 i40e_stat_str(&pf->hw, ret), 7187 i40e_aq_str(&pf->hw, 7188 pf->hw.aq.asq_last_status)); 7189 pf->udp_ports[i].index = 0; 7190 } 7191 } 7192 } 7193 } 7194 7195 /** 7196 * i40e_service_task - Run the driver's async subtasks 7197 * @work: pointer to work_struct containing our data 7198 **/ 7199 static void i40e_service_task(struct work_struct *work) 7200 { 7201 struct i40e_pf *pf = container_of(work, 7202 struct i40e_pf, 7203 service_task); 7204 unsigned long start_time = jiffies; 7205 7206 /* don't bother with service tasks if a reset is in progress */ 7207 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 7208 i40e_service_event_complete(pf); 7209 return; 7210 } 7211 7212 i40e_detect_recover_hung(pf); 7213 i40e_sync_filters_subtask(pf); 7214 i40e_reset_subtask(pf); 7215 i40e_handle_mdd_event(pf); 7216 i40e_vc_process_vflr_event(pf); 7217 i40e_watchdog_subtask(pf); 7218 i40e_fdir_reinit_subtask(pf); 7219 i40e_client_subtask(pf); 7220 i40e_sync_filters_subtask(pf); 7221 i40e_sync_udp_filters_subtask(pf); 7222 i40e_clean_adminq_subtask(pf); 7223 7224 i40e_service_event_complete(pf); 7225 7226 /* If the tasks have taken longer than one timer cycle or there 7227 * is more work to be done, reschedule the service task now 7228 * rather than wait for the timer to tick again. 7229 */ 7230 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 7231 test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state) || 7232 test_bit(__I40E_MDD_EVENT_PENDING, &pf->state) || 7233 test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state)) 7234 i40e_service_event_schedule(pf); 7235 } 7236 7237 /** 7238 * i40e_service_timer - timer callback 7239 * @data: pointer to PF struct 7240 **/ 7241 static void i40e_service_timer(unsigned long data) 7242 { 7243 struct i40e_pf *pf = (struct i40e_pf *)data; 7244 7245 mod_timer(&pf->service_timer, 7246 round_jiffies(jiffies + pf->service_timer_period)); 7247 i40e_service_event_schedule(pf); 7248 } 7249 7250 /** 7251 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 7252 * @vsi: the VSI being configured 7253 **/ 7254 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 7255 { 7256 struct i40e_pf *pf = vsi->back; 7257 7258 switch (vsi->type) { 7259 case I40E_VSI_MAIN: 7260 vsi->alloc_queue_pairs = pf->num_lan_qps; 7261 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7262 I40E_REQ_DESCRIPTOR_MULTIPLE); 7263 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7264 vsi->num_q_vectors = pf->num_lan_msix; 7265 else 7266 vsi->num_q_vectors = 1; 7267 7268 break; 7269 7270 case I40E_VSI_FDIR: 7271 vsi->alloc_queue_pairs = 1; 7272 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 7273 I40E_REQ_DESCRIPTOR_MULTIPLE); 7274 vsi->num_q_vectors = pf->num_fdsb_msix; 7275 break; 7276 7277 case I40E_VSI_VMDQ2: 7278 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 7279 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7280 I40E_REQ_DESCRIPTOR_MULTIPLE); 7281 vsi->num_q_vectors = pf->num_vmdq_msix; 7282 break; 7283 7284 case I40E_VSI_SRIOV: 7285 vsi->alloc_queue_pairs = pf->num_vf_qps; 7286 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7287 I40E_REQ_DESCRIPTOR_MULTIPLE); 7288 break; 7289 7290 #ifdef I40E_FCOE 7291 case I40E_VSI_FCOE: 7292 vsi->alloc_queue_pairs = pf->num_fcoe_qps; 7293 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7294 I40E_REQ_DESCRIPTOR_MULTIPLE); 7295 vsi->num_q_vectors = pf->num_fcoe_msix; 7296 break; 7297 7298 #endif /* I40E_FCOE */ 7299 default: 7300 WARN_ON(1); 7301 return -ENODATA; 7302 } 7303 7304 return 0; 7305 } 7306 7307 /** 7308 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 7309 * @type: VSI pointer 7310 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 7311 * 7312 * On error: returns error code (negative) 7313 * On success: returns 0 7314 **/ 7315 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 7316 { 7317 int size; 7318 int ret = 0; 7319 7320 /* allocate memory for both Tx and Rx ring pointers */ 7321 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; 7322 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 7323 if (!vsi->tx_rings) 7324 return -ENOMEM; 7325 vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; 7326 7327 if (alloc_qvectors) { 7328 /* allocate memory for q_vector pointers */ 7329 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 7330 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 7331 if (!vsi->q_vectors) { 7332 ret = -ENOMEM; 7333 goto err_vectors; 7334 } 7335 } 7336 return ret; 7337 7338 err_vectors: 7339 kfree(vsi->tx_rings); 7340 return ret; 7341 } 7342 7343 /** 7344 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 7345 * @pf: board private structure 7346 * @type: type of VSI 7347 * 7348 * On error: returns error code (negative) 7349 * On success: returns vsi index in PF (positive) 7350 **/ 7351 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 7352 { 7353 int ret = -ENODEV; 7354 struct i40e_vsi *vsi; 7355 int vsi_idx; 7356 int i; 7357 7358 /* Need to protect the allocation of the VSIs at the PF level */ 7359 mutex_lock(&pf->switch_mutex); 7360 7361 /* VSI list may be fragmented if VSI creation/destruction has 7362 * been happening. We can afford to do a quick scan to look 7363 * for any free VSIs in the list. 7364 * 7365 * find next empty vsi slot, looping back around if necessary 7366 */ 7367 i = pf->next_vsi; 7368 while (i < pf->num_alloc_vsi && pf->vsi[i]) 7369 i++; 7370 if (i >= pf->num_alloc_vsi) { 7371 i = 0; 7372 while (i < pf->next_vsi && pf->vsi[i]) 7373 i++; 7374 } 7375 7376 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 7377 vsi_idx = i; /* Found one! */ 7378 } else { 7379 ret = -ENODEV; 7380 goto unlock_pf; /* out of VSI slots! */ 7381 } 7382 pf->next_vsi = ++i; 7383 7384 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 7385 if (!vsi) { 7386 ret = -ENOMEM; 7387 goto unlock_pf; 7388 } 7389 vsi->type = type; 7390 vsi->back = pf; 7391 set_bit(__I40E_DOWN, &vsi->state); 7392 vsi->flags = 0; 7393 vsi->idx = vsi_idx; 7394 vsi->int_rate_limit = 0; 7395 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 7396 pf->rss_table_size : 64; 7397 vsi->netdev_registered = false; 7398 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 7399 hash_init(vsi->mac_filter_hash); 7400 vsi->irqs_ready = false; 7401 7402 ret = i40e_set_num_rings_in_vsi(vsi); 7403 if (ret) 7404 goto err_rings; 7405 7406 ret = i40e_vsi_alloc_arrays(vsi, true); 7407 if (ret) 7408 goto err_rings; 7409 7410 /* Setup default MSIX irq handler for VSI */ 7411 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 7412 7413 /* Initialize VSI lock */ 7414 spin_lock_init(&vsi->mac_filter_hash_lock); 7415 pf->vsi[vsi_idx] = vsi; 7416 ret = vsi_idx; 7417 goto unlock_pf; 7418 7419 err_rings: 7420 pf->next_vsi = i - 1; 7421 kfree(vsi); 7422 unlock_pf: 7423 mutex_unlock(&pf->switch_mutex); 7424 return ret; 7425 } 7426 7427 /** 7428 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 7429 * @type: VSI pointer 7430 * @free_qvectors: a bool to specify if q_vectors need to be freed. 7431 * 7432 * On error: returns error code (negative) 7433 * On success: returns 0 7434 **/ 7435 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 7436 { 7437 /* free the ring and vector containers */ 7438 if (free_qvectors) { 7439 kfree(vsi->q_vectors); 7440 vsi->q_vectors = NULL; 7441 } 7442 kfree(vsi->tx_rings); 7443 vsi->tx_rings = NULL; 7444 vsi->rx_rings = NULL; 7445 } 7446 7447 /** 7448 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 7449 * and lookup table 7450 * @vsi: Pointer to VSI structure 7451 */ 7452 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 7453 { 7454 if (!vsi) 7455 return; 7456 7457 kfree(vsi->rss_hkey_user); 7458 vsi->rss_hkey_user = NULL; 7459 7460 kfree(vsi->rss_lut_user); 7461 vsi->rss_lut_user = NULL; 7462 } 7463 7464 /** 7465 * i40e_vsi_clear - Deallocate the VSI provided 7466 * @vsi: the VSI being un-configured 7467 **/ 7468 static int i40e_vsi_clear(struct i40e_vsi *vsi) 7469 { 7470 struct i40e_pf *pf; 7471 7472 if (!vsi) 7473 return 0; 7474 7475 if (!vsi->back) 7476 goto free_vsi; 7477 pf = vsi->back; 7478 7479 mutex_lock(&pf->switch_mutex); 7480 if (!pf->vsi[vsi->idx]) { 7481 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 7482 vsi->idx, vsi->idx, vsi, vsi->type); 7483 goto unlock_vsi; 7484 } 7485 7486 if (pf->vsi[vsi->idx] != vsi) { 7487 dev_err(&pf->pdev->dev, 7488 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 7489 pf->vsi[vsi->idx]->idx, 7490 pf->vsi[vsi->idx], 7491 pf->vsi[vsi->idx]->type, 7492 vsi->idx, vsi, vsi->type); 7493 goto unlock_vsi; 7494 } 7495 7496 /* updates the PF for this cleared vsi */ 7497 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7498 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 7499 7500 i40e_vsi_free_arrays(vsi, true); 7501 i40e_clear_rss_config_user(vsi); 7502 7503 pf->vsi[vsi->idx] = NULL; 7504 if (vsi->idx < pf->next_vsi) 7505 pf->next_vsi = vsi->idx; 7506 7507 unlock_vsi: 7508 mutex_unlock(&pf->switch_mutex); 7509 free_vsi: 7510 kfree(vsi); 7511 7512 return 0; 7513 } 7514 7515 /** 7516 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 7517 * @vsi: the VSI being cleaned 7518 **/ 7519 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 7520 { 7521 int i; 7522 7523 if (vsi->tx_rings && vsi->tx_rings[0]) { 7524 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7525 kfree_rcu(vsi->tx_rings[i], rcu); 7526 vsi->tx_rings[i] = NULL; 7527 vsi->rx_rings[i] = NULL; 7528 } 7529 } 7530 } 7531 7532 /** 7533 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 7534 * @vsi: the VSI being configured 7535 **/ 7536 static int i40e_alloc_rings(struct i40e_vsi *vsi) 7537 { 7538 struct i40e_ring *tx_ring, *rx_ring; 7539 struct i40e_pf *pf = vsi->back; 7540 int i; 7541 7542 /* Set basic values in the rings to be used later during open() */ 7543 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7544 /* allocate space for both Tx and Rx in one shot */ 7545 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); 7546 if (!tx_ring) 7547 goto err_out; 7548 7549 tx_ring->queue_index = i; 7550 tx_ring->reg_idx = vsi->base_queue + i; 7551 tx_ring->ring_active = false; 7552 tx_ring->vsi = vsi; 7553 tx_ring->netdev = vsi->netdev; 7554 tx_ring->dev = &pf->pdev->dev; 7555 tx_ring->count = vsi->num_desc; 7556 tx_ring->size = 0; 7557 tx_ring->dcb_tc = 0; 7558 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) 7559 tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7560 tx_ring->tx_itr_setting = pf->tx_itr_default; 7561 vsi->tx_rings[i] = tx_ring; 7562 7563 rx_ring = &tx_ring[1]; 7564 rx_ring->queue_index = i; 7565 rx_ring->reg_idx = vsi->base_queue + i; 7566 rx_ring->ring_active = false; 7567 rx_ring->vsi = vsi; 7568 rx_ring->netdev = vsi->netdev; 7569 rx_ring->dev = &pf->pdev->dev; 7570 rx_ring->count = vsi->num_desc; 7571 rx_ring->size = 0; 7572 rx_ring->dcb_tc = 0; 7573 rx_ring->rx_itr_setting = pf->rx_itr_default; 7574 vsi->rx_rings[i] = rx_ring; 7575 } 7576 7577 return 0; 7578 7579 err_out: 7580 i40e_vsi_clear_rings(vsi); 7581 return -ENOMEM; 7582 } 7583 7584 /** 7585 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 7586 * @pf: board private structure 7587 * @vectors: the number of MSI-X vectors to request 7588 * 7589 * Returns the number of vectors reserved, or error 7590 **/ 7591 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 7592 { 7593 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 7594 I40E_MIN_MSIX, vectors); 7595 if (vectors < 0) { 7596 dev_info(&pf->pdev->dev, 7597 "MSI-X vector reservation failed: %d\n", vectors); 7598 vectors = 0; 7599 } 7600 7601 return vectors; 7602 } 7603 7604 /** 7605 * i40e_init_msix - Setup the MSIX capability 7606 * @pf: board private structure 7607 * 7608 * Work with the OS to set up the MSIX vectors needed. 7609 * 7610 * Returns the number of vectors reserved or negative on failure 7611 **/ 7612 static int i40e_init_msix(struct i40e_pf *pf) 7613 { 7614 struct i40e_hw *hw = &pf->hw; 7615 int vectors_left; 7616 int v_budget, i; 7617 int v_actual; 7618 int iwarp_requested = 0; 7619 7620 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7621 return -ENODEV; 7622 7623 /* The number of vectors we'll request will be comprised of: 7624 * - Add 1 for "other" cause for Admin Queue events, etc. 7625 * - The number of LAN queue pairs 7626 * - Queues being used for RSS. 7627 * We don't need as many as max_rss_size vectors. 7628 * use rss_size instead in the calculation since that 7629 * is governed by number of cpus in the system. 7630 * - assumes symmetric Tx/Rx pairing 7631 * - The number of VMDq pairs 7632 * - The CPU count within the NUMA node if iWARP is enabled 7633 #ifdef I40E_FCOE 7634 * - The number of FCOE qps. 7635 #endif 7636 * Once we count this up, try the request. 7637 * 7638 * If we can't get what we want, we'll simplify to nearly nothing 7639 * and try again. If that still fails, we punt. 7640 */ 7641 vectors_left = hw->func_caps.num_msix_vectors; 7642 v_budget = 0; 7643 7644 /* reserve one vector for miscellaneous handler */ 7645 if (vectors_left) { 7646 v_budget++; 7647 vectors_left--; 7648 } 7649 7650 /* reserve vectors for the main PF traffic queues */ 7651 pf->num_lan_msix = min_t(int, num_online_cpus(), vectors_left); 7652 vectors_left -= pf->num_lan_msix; 7653 v_budget += pf->num_lan_msix; 7654 7655 /* reserve one vector for sideband flow director */ 7656 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7657 if (vectors_left) { 7658 pf->num_fdsb_msix = 1; 7659 v_budget++; 7660 vectors_left--; 7661 } else { 7662 pf->num_fdsb_msix = 0; 7663 } 7664 } 7665 7666 #ifdef I40E_FCOE 7667 /* can we reserve enough for FCoE? */ 7668 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7669 if (!vectors_left) 7670 pf->num_fcoe_msix = 0; 7671 else if (vectors_left >= pf->num_fcoe_qps) 7672 pf->num_fcoe_msix = pf->num_fcoe_qps; 7673 else 7674 pf->num_fcoe_msix = 1; 7675 v_budget += pf->num_fcoe_msix; 7676 vectors_left -= pf->num_fcoe_msix; 7677 } 7678 7679 #endif 7680 /* can we reserve enough for iWARP? */ 7681 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7682 iwarp_requested = pf->num_iwarp_msix; 7683 7684 if (!vectors_left) 7685 pf->num_iwarp_msix = 0; 7686 else if (vectors_left < pf->num_iwarp_msix) 7687 pf->num_iwarp_msix = 1; 7688 v_budget += pf->num_iwarp_msix; 7689 vectors_left -= pf->num_iwarp_msix; 7690 } 7691 7692 /* any vectors left over go for VMDq support */ 7693 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 7694 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 7695 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 7696 7697 if (!vectors_left) { 7698 pf->num_vmdq_msix = 0; 7699 pf->num_vmdq_qps = 0; 7700 } else { 7701 /* if we're short on vectors for what's desired, we limit 7702 * the queues per vmdq. If this is still more than are 7703 * available, the user will need to change the number of 7704 * queues/vectors used by the PF later with the ethtool 7705 * channels command 7706 */ 7707 if (vmdq_vecs < vmdq_vecs_wanted) 7708 pf->num_vmdq_qps = 1; 7709 pf->num_vmdq_msix = pf->num_vmdq_qps; 7710 7711 v_budget += vmdq_vecs; 7712 vectors_left -= vmdq_vecs; 7713 } 7714 } 7715 7716 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 7717 GFP_KERNEL); 7718 if (!pf->msix_entries) 7719 return -ENOMEM; 7720 7721 for (i = 0; i < v_budget; i++) 7722 pf->msix_entries[i].entry = i; 7723 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 7724 7725 if (v_actual < I40E_MIN_MSIX) { 7726 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 7727 kfree(pf->msix_entries); 7728 pf->msix_entries = NULL; 7729 pci_disable_msix(pf->pdev); 7730 return -ENODEV; 7731 7732 } else if (v_actual == I40E_MIN_MSIX) { 7733 /* Adjust for minimal MSIX use */ 7734 pf->num_vmdq_vsis = 0; 7735 pf->num_vmdq_qps = 0; 7736 pf->num_lan_qps = 1; 7737 pf->num_lan_msix = 1; 7738 7739 } else if (!vectors_left) { 7740 /* If we have limited resources, we will start with no vectors 7741 * for the special features and then allocate vectors to some 7742 * of these features based on the policy and at the end disable 7743 * the features that did not get any vectors. 7744 */ 7745 int vec; 7746 7747 dev_info(&pf->pdev->dev, 7748 "MSI-X vector limit reached, attempting to redistribute vectors\n"); 7749 /* reserve the misc vector */ 7750 vec = v_actual - 1; 7751 7752 /* Scale vector usage down */ 7753 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 7754 pf->num_vmdq_vsis = 1; 7755 pf->num_vmdq_qps = 1; 7756 #ifdef I40E_FCOE 7757 pf->num_fcoe_qps = 0; 7758 pf->num_fcoe_msix = 0; 7759 #endif 7760 7761 /* partition out the remaining vectors */ 7762 switch (vec) { 7763 case 2: 7764 pf->num_lan_msix = 1; 7765 break; 7766 case 3: 7767 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7768 pf->num_lan_msix = 1; 7769 pf->num_iwarp_msix = 1; 7770 } else { 7771 pf->num_lan_msix = 2; 7772 } 7773 #ifdef I40E_FCOE 7774 /* give one vector to FCoE */ 7775 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7776 pf->num_lan_msix = 1; 7777 pf->num_fcoe_msix = 1; 7778 } 7779 #endif 7780 break; 7781 default: 7782 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7783 pf->num_iwarp_msix = min_t(int, (vec / 3), 7784 iwarp_requested); 7785 pf->num_vmdq_vsis = min_t(int, (vec / 3), 7786 I40E_DEFAULT_NUM_VMDQ_VSI); 7787 } else { 7788 pf->num_vmdq_vsis = min_t(int, (vec / 2), 7789 I40E_DEFAULT_NUM_VMDQ_VSI); 7790 } 7791 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7792 pf->num_fdsb_msix = 1; 7793 vec--; 7794 } 7795 pf->num_lan_msix = min_t(int, 7796 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 7797 pf->num_lan_msix); 7798 pf->num_lan_qps = pf->num_lan_msix; 7799 #ifdef I40E_FCOE 7800 /* give one vector to FCoE */ 7801 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7802 pf->num_fcoe_msix = 1; 7803 vec--; 7804 } 7805 #endif 7806 break; 7807 } 7808 } 7809 7810 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 7811 (pf->num_fdsb_msix == 0)) { 7812 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 7813 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7814 } 7815 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 7816 (pf->num_vmdq_msix == 0)) { 7817 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 7818 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 7819 } 7820 7821 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 7822 (pf->num_iwarp_msix == 0)) { 7823 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 7824 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 7825 } 7826 #ifdef I40E_FCOE 7827 7828 if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) { 7829 dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n"); 7830 pf->flags &= ~I40E_FLAG_FCOE_ENABLED; 7831 } 7832 #endif 7833 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 7834 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 7835 pf->num_lan_msix, 7836 pf->num_vmdq_msix * pf->num_vmdq_vsis, 7837 pf->num_fdsb_msix, 7838 pf->num_iwarp_msix); 7839 7840 return v_actual; 7841 } 7842 7843 /** 7844 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 7845 * @vsi: the VSI being configured 7846 * @v_idx: index of the vector in the vsi struct 7847 * @cpu: cpu to be used on affinity_mask 7848 * 7849 * We allocate one q_vector. If allocation fails we return -ENOMEM. 7850 **/ 7851 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 7852 { 7853 struct i40e_q_vector *q_vector; 7854 7855 /* allocate q_vector */ 7856 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 7857 if (!q_vector) 7858 return -ENOMEM; 7859 7860 q_vector->vsi = vsi; 7861 q_vector->v_idx = v_idx; 7862 cpumask_set_cpu(cpu, &q_vector->affinity_mask); 7863 7864 if (vsi->netdev) 7865 netif_napi_add(vsi->netdev, &q_vector->napi, 7866 i40e_napi_poll, NAPI_POLL_WEIGHT); 7867 7868 q_vector->rx.latency_range = I40E_LOW_LATENCY; 7869 q_vector->tx.latency_range = I40E_LOW_LATENCY; 7870 7871 /* tie q_vector and vsi together */ 7872 vsi->q_vectors[v_idx] = q_vector; 7873 7874 return 0; 7875 } 7876 7877 /** 7878 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 7879 * @vsi: the VSI being configured 7880 * 7881 * We allocate one q_vector per queue interrupt. If allocation fails we 7882 * return -ENOMEM. 7883 **/ 7884 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 7885 { 7886 struct i40e_pf *pf = vsi->back; 7887 int err, v_idx, num_q_vectors, current_cpu; 7888 7889 /* if not MSIX, give the one vector only to the LAN VSI */ 7890 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7891 num_q_vectors = vsi->num_q_vectors; 7892 else if (vsi == pf->vsi[pf->lan_vsi]) 7893 num_q_vectors = 1; 7894 else 7895 return -EINVAL; 7896 7897 current_cpu = cpumask_first(cpu_online_mask); 7898 7899 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 7900 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 7901 if (err) 7902 goto err_out; 7903 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 7904 if (unlikely(current_cpu >= nr_cpu_ids)) 7905 current_cpu = cpumask_first(cpu_online_mask); 7906 } 7907 7908 return 0; 7909 7910 err_out: 7911 while (v_idx--) 7912 i40e_free_q_vector(vsi, v_idx); 7913 7914 return err; 7915 } 7916 7917 /** 7918 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 7919 * @pf: board private structure to initialize 7920 **/ 7921 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 7922 { 7923 int vectors = 0; 7924 ssize_t size; 7925 7926 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 7927 vectors = i40e_init_msix(pf); 7928 if (vectors < 0) { 7929 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 7930 I40E_FLAG_IWARP_ENABLED | 7931 #ifdef I40E_FCOE 7932 I40E_FLAG_FCOE_ENABLED | 7933 #endif 7934 I40E_FLAG_RSS_ENABLED | 7935 I40E_FLAG_DCB_CAPABLE | 7936 I40E_FLAG_DCB_ENABLED | 7937 I40E_FLAG_SRIOV_ENABLED | 7938 I40E_FLAG_FD_SB_ENABLED | 7939 I40E_FLAG_FD_ATR_ENABLED | 7940 I40E_FLAG_VMDQ_ENABLED); 7941 7942 /* rework the queue expectations without MSIX */ 7943 i40e_determine_queue_usage(pf); 7944 } 7945 } 7946 7947 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 7948 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 7949 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 7950 vectors = pci_enable_msi(pf->pdev); 7951 if (vectors < 0) { 7952 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 7953 vectors); 7954 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 7955 } 7956 vectors = 1; /* one MSI or Legacy vector */ 7957 } 7958 7959 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 7960 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 7961 7962 /* set up vector assignment tracking */ 7963 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 7964 pf->irq_pile = kzalloc(size, GFP_KERNEL); 7965 if (!pf->irq_pile) { 7966 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 7967 return -ENOMEM; 7968 } 7969 pf->irq_pile->num_entries = vectors; 7970 pf->irq_pile->search_hint = 0; 7971 7972 /* track first vector for misc interrupts, ignore return */ 7973 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 7974 7975 return 0; 7976 } 7977 7978 /** 7979 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 7980 * @pf: board private structure 7981 * 7982 * This sets up the handler for MSIX 0, which is used to manage the 7983 * non-queue interrupts, e.g. AdminQ and errors. This is not used 7984 * when in MSI or Legacy interrupt mode. 7985 **/ 7986 static int i40e_setup_misc_vector(struct i40e_pf *pf) 7987 { 7988 struct i40e_hw *hw = &pf->hw; 7989 int err = 0; 7990 7991 /* Only request the irq if this is the first time through, and 7992 * not when we're rebuilding after a Reset 7993 */ 7994 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 7995 err = request_irq(pf->msix_entries[0].vector, 7996 i40e_intr, 0, pf->int_name, pf); 7997 if (err) { 7998 dev_info(&pf->pdev->dev, 7999 "request_irq for %s failed: %d\n", 8000 pf->int_name, err); 8001 return -EFAULT; 8002 } 8003 } 8004 8005 i40e_enable_misc_int_causes(pf); 8006 8007 /* associate no queues to the misc vector */ 8008 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 8009 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 8010 8011 i40e_flush(hw); 8012 8013 i40e_irq_dynamic_enable_icr0(pf, true); 8014 8015 return err; 8016 } 8017 8018 /** 8019 * i40e_config_rss_aq - Prepare for RSS using AQ commands 8020 * @vsi: vsi structure 8021 * @seed: RSS hash seed 8022 **/ 8023 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8024 u8 *lut, u16 lut_size) 8025 { 8026 struct i40e_pf *pf = vsi->back; 8027 struct i40e_hw *hw = &pf->hw; 8028 int ret = 0; 8029 8030 if (seed) { 8031 struct i40e_aqc_get_set_rss_key_data *seed_dw = 8032 (struct i40e_aqc_get_set_rss_key_data *)seed; 8033 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 8034 if (ret) { 8035 dev_info(&pf->pdev->dev, 8036 "Cannot set RSS key, err %s aq_err %s\n", 8037 i40e_stat_str(hw, ret), 8038 i40e_aq_str(hw, hw->aq.asq_last_status)); 8039 return ret; 8040 } 8041 } 8042 if (lut) { 8043 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8044 8045 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8046 if (ret) { 8047 dev_info(&pf->pdev->dev, 8048 "Cannot set RSS lut, err %s aq_err %s\n", 8049 i40e_stat_str(hw, ret), 8050 i40e_aq_str(hw, hw->aq.asq_last_status)); 8051 return ret; 8052 } 8053 } 8054 return ret; 8055 } 8056 8057 /** 8058 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 8059 * @vsi: Pointer to vsi structure 8060 * @seed: Buffter to store the hash keys 8061 * @lut: Buffer to store the lookup table entries 8062 * @lut_size: Size of buffer to store the lookup table entries 8063 * 8064 * Return 0 on success, negative on failure 8065 */ 8066 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8067 u8 *lut, u16 lut_size) 8068 { 8069 struct i40e_pf *pf = vsi->back; 8070 struct i40e_hw *hw = &pf->hw; 8071 int ret = 0; 8072 8073 if (seed) { 8074 ret = i40e_aq_get_rss_key(hw, vsi->id, 8075 (struct i40e_aqc_get_set_rss_key_data *)seed); 8076 if (ret) { 8077 dev_info(&pf->pdev->dev, 8078 "Cannot get RSS key, err %s aq_err %s\n", 8079 i40e_stat_str(&pf->hw, ret), 8080 i40e_aq_str(&pf->hw, 8081 pf->hw.aq.asq_last_status)); 8082 return ret; 8083 } 8084 } 8085 8086 if (lut) { 8087 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8088 8089 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8090 if (ret) { 8091 dev_info(&pf->pdev->dev, 8092 "Cannot get RSS lut, err %s aq_err %s\n", 8093 i40e_stat_str(&pf->hw, ret), 8094 i40e_aq_str(&pf->hw, 8095 pf->hw.aq.asq_last_status)); 8096 return ret; 8097 } 8098 } 8099 8100 return ret; 8101 } 8102 8103 /** 8104 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 8105 * @vsi: VSI structure 8106 **/ 8107 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 8108 { 8109 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8110 struct i40e_pf *pf = vsi->back; 8111 u8 *lut; 8112 int ret; 8113 8114 if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)) 8115 return 0; 8116 8117 if (!vsi->rss_size) 8118 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8119 vsi->num_queue_pairs); 8120 if (!vsi->rss_size) 8121 return -EINVAL; 8122 8123 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8124 if (!lut) 8125 return -ENOMEM; 8126 /* Use the user configured hash keys and lookup table if there is one, 8127 * otherwise use default 8128 */ 8129 if (vsi->rss_lut_user) 8130 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8131 else 8132 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8133 if (vsi->rss_hkey_user) 8134 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8135 else 8136 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8137 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 8138 kfree(lut); 8139 8140 return ret; 8141 } 8142 8143 /** 8144 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 8145 * @vsi: Pointer to vsi structure 8146 * @seed: RSS hash seed 8147 * @lut: Lookup table 8148 * @lut_size: Lookup table size 8149 * 8150 * Returns 0 on success, negative on failure 8151 **/ 8152 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 8153 const u8 *lut, u16 lut_size) 8154 { 8155 struct i40e_pf *pf = vsi->back; 8156 struct i40e_hw *hw = &pf->hw; 8157 u16 vf_id = vsi->vf_id; 8158 u8 i; 8159 8160 /* Fill out hash function seed */ 8161 if (seed) { 8162 u32 *seed_dw = (u32 *)seed; 8163 8164 if (vsi->type == I40E_VSI_MAIN) { 8165 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8166 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), 8167 seed_dw[i]); 8168 } else if (vsi->type == I40E_VSI_SRIOV) { 8169 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 8170 i40e_write_rx_ctl(hw, 8171 I40E_VFQF_HKEY1(i, vf_id), 8172 seed_dw[i]); 8173 } else { 8174 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 8175 } 8176 } 8177 8178 if (lut) { 8179 u32 *lut_dw = (u32 *)lut; 8180 8181 if (vsi->type == I40E_VSI_MAIN) { 8182 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8183 return -EINVAL; 8184 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8185 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 8186 } else if (vsi->type == I40E_VSI_SRIOV) { 8187 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 8188 return -EINVAL; 8189 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8190 i40e_write_rx_ctl(hw, 8191 I40E_VFQF_HLUT1(i, vf_id), 8192 lut_dw[i]); 8193 } else { 8194 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8195 } 8196 } 8197 i40e_flush(hw); 8198 8199 return 0; 8200 } 8201 8202 /** 8203 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 8204 * @vsi: Pointer to VSI structure 8205 * @seed: Buffer to store the keys 8206 * @lut: Buffer to store the lookup table entries 8207 * @lut_size: Size of buffer to store the lookup table entries 8208 * 8209 * Returns 0 on success, negative on failure 8210 */ 8211 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 8212 u8 *lut, u16 lut_size) 8213 { 8214 struct i40e_pf *pf = vsi->back; 8215 struct i40e_hw *hw = &pf->hw; 8216 u16 i; 8217 8218 if (seed) { 8219 u32 *seed_dw = (u32 *)seed; 8220 8221 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8222 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 8223 } 8224 if (lut) { 8225 u32 *lut_dw = (u32 *)lut; 8226 8227 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8228 return -EINVAL; 8229 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8230 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 8231 } 8232 8233 return 0; 8234 } 8235 8236 /** 8237 * i40e_config_rss - Configure RSS keys and lut 8238 * @vsi: Pointer to VSI structure 8239 * @seed: RSS hash seed 8240 * @lut: Lookup table 8241 * @lut_size: Lookup table size 8242 * 8243 * Returns 0 on success, negative on failure 8244 */ 8245 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8246 { 8247 struct i40e_pf *pf = vsi->back; 8248 8249 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8250 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 8251 else 8252 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 8253 } 8254 8255 /** 8256 * i40e_get_rss - Get RSS keys and lut 8257 * @vsi: Pointer to VSI structure 8258 * @seed: Buffer to store the keys 8259 * @lut: Buffer to store the lookup table entries 8260 * lut_size: Size of buffer to store the lookup table entries 8261 * 8262 * Returns 0 on success, negative on failure 8263 */ 8264 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8265 { 8266 struct i40e_pf *pf = vsi->back; 8267 8268 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8269 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 8270 else 8271 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 8272 } 8273 8274 /** 8275 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 8276 * @pf: Pointer to board private structure 8277 * @lut: Lookup table 8278 * @rss_table_size: Lookup table size 8279 * @rss_size: Range of queue number for hashing 8280 */ 8281 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 8282 u16 rss_table_size, u16 rss_size) 8283 { 8284 u16 i; 8285 8286 for (i = 0; i < rss_table_size; i++) 8287 lut[i] = i % rss_size; 8288 } 8289 8290 /** 8291 * i40e_pf_config_rss - Prepare for RSS if used 8292 * @pf: board private structure 8293 **/ 8294 static int i40e_pf_config_rss(struct i40e_pf *pf) 8295 { 8296 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8297 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8298 u8 *lut; 8299 struct i40e_hw *hw = &pf->hw; 8300 u32 reg_val; 8301 u64 hena; 8302 int ret; 8303 8304 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 8305 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 8306 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 8307 hena |= i40e_pf_get_default_rss_hena(pf); 8308 8309 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 8310 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 8311 8312 /* Determine the RSS table size based on the hardware capabilities */ 8313 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 8314 reg_val = (pf->rss_table_size == 512) ? 8315 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 8316 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 8317 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 8318 8319 /* Determine the RSS size of the VSI */ 8320 if (!vsi->rss_size) 8321 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8322 vsi->num_queue_pairs); 8323 if (!vsi->rss_size) 8324 return -EINVAL; 8325 8326 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8327 if (!lut) 8328 return -ENOMEM; 8329 8330 /* Use user configured lut if there is one, otherwise use default */ 8331 if (vsi->rss_lut_user) 8332 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8333 else 8334 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8335 8336 /* Use user configured hash key if there is one, otherwise 8337 * use default. 8338 */ 8339 if (vsi->rss_hkey_user) 8340 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8341 else 8342 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8343 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 8344 kfree(lut); 8345 8346 return ret; 8347 } 8348 8349 /** 8350 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 8351 * @pf: board private structure 8352 * @queue_count: the requested queue count for rss. 8353 * 8354 * returns 0 if rss is not enabled, if enabled returns the final rss queue 8355 * count which may be different from the requested queue count. 8356 **/ 8357 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 8358 { 8359 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8360 int new_rss_size; 8361 8362 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 8363 return 0; 8364 8365 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 8366 8367 if (queue_count != vsi->num_queue_pairs) { 8368 vsi->req_queue_pairs = queue_count; 8369 i40e_prep_for_reset(pf); 8370 8371 pf->alloc_rss_size = new_rss_size; 8372 8373 i40e_reset_and_rebuild(pf, true); 8374 8375 /* Discard the user configured hash keys and lut, if less 8376 * queues are enabled. 8377 */ 8378 if (queue_count < vsi->rss_size) { 8379 i40e_clear_rss_config_user(vsi); 8380 dev_dbg(&pf->pdev->dev, 8381 "discard user configured hash keys and lut\n"); 8382 } 8383 8384 /* Reset vsi->rss_size, as number of enabled queues changed */ 8385 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8386 vsi->num_queue_pairs); 8387 8388 i40e_pf_config_rss(pf); 8389 } 8390 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 8391 vsi->req_queue_pairs, pf->rss_size_max); 8392 return pf->alloc_rss_size; 8393 } 8394 8395 /** 8396 * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition 8397 * @pf: board private structure 8398 **/ 8399 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf) 8400 { 8401 i40e_status status; 8402 bool min_valid, max_valid; 8403 u32 max_bw, min_bw; 8404 8405 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 8406 &min_valid, &max_valid); 8407 8408 if (!status) { 8409 if (min_valid) 8410 pf->npar_min_bw = min_bw; 8411 if (max_valid) 8412 pf->npar_max_bw = max_bw; 8413 } 8414 8415 return status; 8416 } 8417 8418 /** 8419 * i40e_set_npar_bw_setting - Set BW settings for this PF partition 8420 * @pf: board private structure 8421 **/ 8422 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf) 8423 { 8424 struct i40e_aqc_configure_partition_bw_data bw_data; 8425 i40e_status status; 8426 8427 /* Set the valid bit for this PF */ 8428 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 8429 bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK; 8430 bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK; 8431 8432 /* Set the new bandwidths */ 8433 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 8434 8435 return status; 8436 } 8437 8438 /** 8439 * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition 8440 * @pf: board private structure 8441 **/ 8442 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf) 8443 { 8444 /* Commit temporary BW setting to permanent NVM image */ 8445 enum i40e_admin_queue_err last_aq_status; 8446 i40e_status ret; 8447 u16 nvm_word; 8448 8449 if (pf->hw.partition_id != 1) { 8450 dev_info(&pf->pdev->dev, 8451 "Commit BW only works on partition 1! This is partition %d", 8452 pf->hw.partition_id); 8453 ret = I40E_NOT_SUPPORTED; 8454 goto bw_commit_out; 8455 } 8456 8457 /* Acquire NVM for read access */ 8458 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 8459 last_aq_status = pf->hw.aq.asq_last_status; 8460 if (ret) { 8461 dev_info(&pf->pdev->dev, 8462 "Cannot acquire NVM for read access, err %s aq_err %s\n", 8463 i40e_stat_str(&pf->hw, ret), 8464 i40e_aq_str(&pf->hw, last_aq_status)); 8465 goto bw_commit_out; 8466 } 8467 8468 /* Read word 0x10 of NVM - SW compatibility word 1 */ 8469 ret = i40e_aq_read_nvm(&pf->hw, 8470 I40E_SR_NVM_CONTROL_WORD, 8471 0x10, sizeof(nvm_word), &nvm_word, 8472 false, NULL); 8473 /* Save off last admin queue command status before releasing 8474 * the NVM 8475 */ 8476 last_aq_status = pf->hw.aq.asq_last_status; 8477 i40e_release_nvm(&pf->hw); 8478 if (ret) { 8479 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 8480 i40e_stat_str(&pf->hw, ret), 8481 i40e_aq_str(&pf->hw, last_aq_status)); 8482 goto bw_commit_out; 8483 } 8484 8485 /* Wait a bit for NVM release to complete */ 8486 msleep(50); 8487 8488 /* Acquire NVM for write access */ 8489 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 8490 last_aq_status = pf->hw.aq.asq_last_status; 8491 if (ret) { 8492 dev_info(&pf->pdev->dev, 8493 "Cannot acquire NVM for write access, err %s aq_err %s\n", 8494 i40e_stat_str(&pf->hw, ret), 8495 i40e_aq_str(&pf->hw, last_aq_status)); 8496 goto bw_commit_out; 8497 } 8498 /* Write it back out unchanged to initiate update NVM, 8499 * which will force a write of the shadow (alt) RAM to 8500 * the NVM - thus storing the bandwidth values permanently. 8501 */ 8502 ret = i40e_aq_update_nvm(&pf->hw, 8503 I40E_SR_NVM_CONTROL_WORD, 8504 0x10, sizeof(nvm_word), 8505 &nvm_word, true, NULL); 8506 /* Save off last admin queue command status before releasing 8507 * the NVM 8508 */ 8509 last_aq_status = pf->hw.aq.asq_last_status; 8510 i40e_release_nvm(&pf->hw); 8511 if (ret) 8512 dev_info(&pf->pdev->dev, 8513 "BW settings NOT SAVED, err %s aq_err %s\n", 8514 i40e_stat_str(&pf->hw, ret), 8515 i40e_aq_str(&pf->hw, last_aq_status)); 8516 bw_commit_out: 8517 8518 return ret; 8519 } 8520 8521 /** 8522 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 8523 * @pf: board private structure to initialize 8524 * 8525 * i40e_sw_init initializes the Adapter private data structure. 8526 * Fields are initialized based on PCI device information and 8527 * OS network device settings (MTU size). 8528 **/ 8529 static int i40e_sw_init(struct i40e_pf *pf) 8530 { 8531 int err = 0; 8532 int size; 8533 8534 /* Set default capability flags */ 8535 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 8536 I40E_FLAG_MSI_ENABLED | 8537 I40E_FLAG_MSIX_ENABLED; 8538 8539 /* Set default ITR */ 8540 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 8541 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 8542 8543 /* Depending on PF configurations, it is possible that the RSS 8544 * maximum might end up larger than the available queues 8545 */ 8546 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 8547 pf->alloc_rss_size = 1; 8548 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 8549 pf->rss_size_max = min_t(int, pf->rss_size_max, 8550 pf->hw.func_caps.num_tx_qp); 8551 if (pf->hw.func_caps.rss) { 8552 pf->flags |= I40E_FLAG_RSS_ENABLED; 8553 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 8554 num_online_cpus()); 8555 } 8556 8557 /* MFP mode enabled */ 8558 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 8559 pf->flags |= I40E_FLAG_MFP_ENABLED; 8560 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 8561 if (i40e_get_npar_bw_setting(pf)) 8562 dev_warn(&pf->pdev->dev, 8563 "Could not get NPAR bw settings\n"); 8564 else 8565 dev_info(&pf->pdev->dev, 8566 "Min BW = %8.8x, Max BW = %8.8x\n", 8567 pf->npar_min_bw, pf->npar_max_bw); 8568 } 8569 8570 /* FW/NVM is not yet fixed in this regard */ 8571 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 8572 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 8573 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 8574 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 8575 if (pf->flags & I40E_FLAG_MFP_ENABLED && 8576 pf->hw.num_partitions > 1) 8577 dev_info(&pf->pdev->dev, 8578 "Flow Director Sideband mode Disabled in MFP mode\n"); 8579 else 8580 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8581 pf->fdir_pf_filter_count = 8582 pf->hw.func_caps.fd_filters_guaranteed; 8583 pf->hw.fdir_shared_filter_count = 8584 pf->hw.func_caps.fd_filters_best_effort; 8585 } 8586 8587 if (i40e_is_mac_710(&pf->hw) && 8588 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 8589 (pf->hw.aq.fw_maj_ver < 4))) { 8590 pf->flags |= I40E_FLAG_RESTART_AUTONEG; 8591 /* No DCB support for FW < v4.33 */ 8592 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT; 8593 } 8594 8595 /* Disable FW LLDP if FW < v4.3 */ 8596 if (i40e_is_mac_710(&pf->hw) && 8597 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 8598 (pf->hw.aq.fw_maj_ver < 4))) 8599 pf->flags |= I40E_FLAG_STOP_FW_LLDP; 8600 8601 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 8602 if (i40e_is_mac_710(&pf->hw) && 8603 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 8604 (pf->hw.aq.fw_maj_ver >= 5))) 8605 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB; 8606 8607 if (pf->hw.func_caps.vmdq) { 8608 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 8609 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 8610 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 8611 } 8612 8613 if (pf->hw.func_caps.iwarp) { 8614 pf->flags |= I40E_FLAG_IWARP_ENABLED; 8615 /* IWARP needs one extra vector for CQP just like MISC.*/ 8616 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 8617 } 8618 8619 #ifdef I40E_FCOE 8620 i40e_init_pf_fcoe(pf); 8621 8622 #endif /* I40E_FCOE */ 8623 #ifdef CONFIG_PCI_IOV 8624 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 8625 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 8626 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 8627 pf->num_req_vfs = min_t(int, 8628 pf->hw.func_caps.num_vfs, 8629 I40E_MAX_VF_COUNT); 8630 } 8631 #endif /* CONFIG_PCI_IOV */ 8632 if (pf->hw.mac.type == I40E_MAC_X722) { 8633 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE | 8634 I40E_FLAG_128_QP_RSS_CAPABLE | 8635 I40E_FLAG_HW_ATR_EVICT_CAPABLE | 8636 I40E_FLAG_OUTER_UDP_CSUM_CAPABLE | 8637 I40E_FLAG_WB_ON_ITR_CAPABLE | 8638 I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE | 8639 I40E_FLAG_NO_PCI_LINK_CHECK | 8640 I40E_FLAG_USE_SET_LLDP_MIB | 8641 I40E_FLAG_GENEVE_OFFLOAD_CAPABLE; 8642 } else if ((pf->hw.aq.api_maj_ver > 1) || 8643 ((pf->hw.aq.api_maj_ver == 1) && 8644 (pf->hw.aq.api_min_ver > 4))) { 8645 /* Supported in FW API version higher than 1.4 */ 8646 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE; 8647 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8648 } else { 8649 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8650 } 8651 8652 pf->eeprom_version = 0xDEAD; 8653 pf->lan_veb = I40E_NO_VEB; 8654 pf->lan_vsi = I40E_NO_VSI; 8655 8656 /* By default FW has this off for performance reasons */ 8657 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 8658 8659 /* set up queue assignment tracking */ 8660 size = sizeof(struct i40e_lump_tracking) 8661 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 8662 pf->qp_pile = kzalloc(size, GFP_KERNEL); 8663 if (!pf->qp_pile) { 8664 err = -ENOMEM; 8665 goto sw_init_done; 8666 } 8667 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 8668 pf->qp_pile->search_hint = 0; 8669 8670 pf->tx_timeout_recovery_level = 1; 8671 8672 mutex_init(&pf->switch_mutex); 8673 8674 /* If NPAR is enabled nudge the Tx scheduler */ 8675 if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf))) 8676 i40e_set_npar_bw_setting(pf); 8677 8678 sw_init_done: 8679 return err; 8680 } 8681 8682 /** 8683 * i40e_set_ntuple - set the ntuple feature flag and take action 8684 * @pf: board private structure to initialize 8685 * @features: the feature set that the stack is suggesting 8686 * 8687 * returns a bool to indicate if reset needs to happen 8688 **/ 8689 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 8690 { 8691 bool need_reset = false; 8692 8693 /* Check if Flow Director n-tuple support was enabled or disabled. If 8694 * the state changed, we need to reset. 8695 */ 8696 if (features & NETIF_F_NTUPLE) { 8697 /* Enable filters and mark for reset */ 8698 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8699 need_reset = true; 8700 /* enable FD_SB only if there is MSI-X vector */ 8701 if (pf->num_fdsb_msix > 0) 8702 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8703 } else { 8704 /* turn off filters, mark for reset and clear SW filter list */ 8705 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8706 need_reset = true; 8707 i40e_fdir_filter_exit(pf); 8708 } 8709 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8710 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED; 8711 /* reset fd counters */ 8712 pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0; 8713 pf->fdir_pf_active_filters = 0; 8714 /* if ATR was auto disabled it can be re-enabled. */ 8715 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8716 (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) { 8717 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 8718 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8719 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 8720 } 8721 } 8722 return need_reset; 8723 } 8724 8725 /** 8726 * i40e_clear_rss_lut - clear the rx hash lookup table 8727 * @vsi: the VSI being configured 8728 **/ 8729 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 8730 { 8731 struct i40e_pf *pf = vsi->back; 8732 struct i40e_hw *hw = &pf->hw; 8733 u16 vf_id = vsi->vf_id; 8734 u8 i; 8735 8736 if (vsi->type == I40E_VSI_MAIN) { 8737 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8738 wr32(hw, I40E_PFQF_HLUT(i), 0); 8739 } else if (vsi->type == I40E_VSI_SRIOV) { 8740 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8741 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 8742 } else { 8743 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8744 } 8745 } 8746 8747 /** 8748 * i40e_set_features - set the netdev feature flags 8749 * @netdev: ptr to the netdev being adjusted 8750 * @features: the feature set that the stack is suggesting 8751 **/ 8752 static int i40e_set_features(struct net_device *netdev, 8753 netdev_features_t features) 8754 { 8755 struct i40e_netdev_priv *np = netdev_priv(netdev); 8756 struct i40e_vsi *vsi = np->vsi; 8757 struct i40e_pf *pf = vsi->back; 8758 bool need_reset; 8759 8760 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 8761 i40e_pf_config_rss(pf); 8762 else if (!(features & NETIF_F_RXHASH) && 8763 netdev->features & NETIF_F_RXHASH) 8764 i40e_clear_rss_lut(vsi); 8765 8766 if (features & NETIF_F_HW_VLAN_CTAG_RX) 8767 i40e_vlan_stripping_enable(vsi); 8768 else 8769 i40e_vlan_stripping_disable(vsi); 8770 8771 need_reset = i40e_set_ntuple(pf, features); 8772 8773 if (need_reset) 8774 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 8775 8776 return 0; 8777 } 8778 8779 /** 8780 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 8781 * @pf: board private structure 8782 * @port: The UDP port to look up 8783 * 8784 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 8785 **/ 8786 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, __be16 port) 8787 { 8788 u8 i; 8789 8790 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 8791 if (pf->udp_ports[i].index == port) 8792 return i; 8793 } 8794 8795 return i; 8796 } 8797 8798 /** 8799 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 8800 * @netdev: This physical port's netdev 8801 * @ti: Tunnel endpoint information 8802 **/ 8803 static void i40e_udp_tunnel_add(struct net_device *netdev, 8804 struct udp_tunnel_info *ti) 8805 { 8806 struct i40e_netdev_priv *np = netdev_priv(netdev); 8807 struct i40e_vsi *vsi = np->vsi; 8808 struct i40e_pf *pf = vsi->back; 8809 __be16 port = ti->port; 8810 u8 next_idx; 8811 u8 idx; 8812 8813 idx = i40e_get_udp_port_idx(pf, port); 8814 8815 /* Check if port already exists */ 8816 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8817 netdev_info(netdev, "port %d already offloaded\n", 8818 ntohs(port)); 8819 return; 8820 } 8821 8822 /* Now check if there is space to add the new port */ 8823 next_idx = i40e_get_udp_port_idx(pf, 0); 8824 8825 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8826 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 8827 ntohs(port)); 8828 return; 8829 } 8830 8831 switch (ti->type) { 8832 case UDP_TUNNEL_TYPE_VXLAN: 8833 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 8834 break; 8835 case UDP_TUNNEL_TYPE_GENEVE: 8836 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)) 8837 return; 8838 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 8839 break; 8840 default: 8841 return; 8842 } 8843 8844 /* New port: add it and mark its index in the bitmap */ 8845 pf->udp_ports[next_idx].index = port; 8846 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 8847 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8848 } 8849 8850 /** 8851 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 8852 * @netdev: This physical port's netdev 8853 * @ti: Tunnel endpoint information 8854 **/ 8855 static void i40e_udp_tunnel_del(struct net_device *netdev, 8856 struct udp_tunnel_info *ti) 8857 { 8858 struct i40e_netdev_priv *np = netdev_priv(netdev); 8859 struct i40e_vsi *vsi = np->vsi; 8860 struct i40e_pf *pf = vsi->back; 8861 __be16 port = ti->port; 8862 u8 idx; 8863 8864 idx = i40e_get_udp_port_idx(pf, port); 8865 8866 /* Check if port already exists */ 8867 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 8868 goto not_found; 8869 8870 switch (ti->type) { 8871 case UDP_TUNNEL_TYPE_VXLAN: 8872 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 8873 goto not_found; 8874 break; 8875 case UDP_TUNNEL_TYPE_GENEVE: 8876 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 8877 goto not_found; 8878 break; 8879 default: 8880 goto not_found; 8881 } 8882 8883 /* if port exists, set it to 0 (mark for deletion) 8884 * and make it pending 8885 */ 8886 pf->udp_ports[idx].index = 0; 8887 pf->pending_udp_bitmap |= BIT_ULL(idx); 8888 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8889 8890 return; 8891 not_found: 8892 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 8893 ntohs(port)); 8894 } 8895 8896 static int i40e_get_phys_port_id(struct net_device *netdev, 8897 struct netdev_phys_item_id *ppid) 8898 { 8899 struct i40e_netdev_priv *np = netdev_priv(netdev); 8900 struct i40e_pf *pf = np->vsi->back; 8901 struct i40e_hw *hw = &pf->hw; 8902 8903 if (!(pf->flags & I40E_FLAG_PORT_ID_VALID)) 8904 return -EOPNOTSUPP; 8905 8906 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 8907 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 8908 8909 return 0; 8910 } 8911 8912 /** 8913 * i40e_ndo_fdb_add - add an entry to the hardware database 8914 * @ndm: the input from the stack 8915 * @tb: pointer to array of nladdr (unused) 8916 * @dev: the net device pointer 8917 * @addr: the MAC address entry being added 8918 * @flags: instructions from stack about fdb operation 8919 */ 8920 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 8921 struct net_device *dev, 8922 const unsigned char *addr, u16 vid, 8923 u16 flags) 8924 { 8925 struct i40e_netdev_priv *np = netdev_priv(dev); 8926 struct i40e_pf *pf = np->vsi->back; 8927 int err = 0; 8928 8929 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 8930 return -EOPNOTSUPP; 8931 8932 if (vid) { 8933 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 8934 return -EINVAL; 8935 } 8936 8937 /* Hardware does not support aging addresses so if a 8938 * ndm_state is given only allow permanent addresses 8939 */ 8940 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 8941 netdev_info(dev, "FDB only supports static addresses\n"); 8942 return -EINVAL; 8943 } 8944 8945 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 8946 err = dev_uc_add_excl(dev, addr); 8947 else if (is_multicast_ether_addr(addr)) 8948 err = dev_mc_add_excl(dev, addr); 8949 else 8950 err = -EINVAL; 8951 8952 /* Only return duplicate errors if NLM_F_EXCL is set */ 8953 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 8954 err = 0; 8955 8956 return err; 8957 } 8958 8959 /** 8960 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 8961 * @dev: the netdev being configured 8962 * @nlh: RTNL message 8963 * 8964 * Inserts a new hardware bridge if not already created and 8965 * enables the bridging mode requested (VEB or VEPA). If the 8966 * hardware bridge has already been inserted and the request 8967 * is to change the mode then that requires a PF reset to 8968 * allow rebuild of the components with required hardware 8969 * bridge mode enabled. 8970 **/ 8971 static int i40e_ndo_bridge_setlink(struct net_device *dev, 8972 struct nlmsghdr *nlh, 8973 u16 flags) 8974 { 8975 struct i40e_netdev_priv *np = netdev_priv(dev); 8976 struct i40e_vsi *vsi = np->vsi; 8977 struct i40e_pf *pf = vsi->back; 8978 struct i40e_veb *veb = NULL; 8979 struct nlattr *attr, *br_spec; 8980 int i, rem; 8981 8982 /* Only for PF VSI for now */ 8983 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 8984 return -EOPNOTSUPP; 8985 8986 /* Find the HW bridge for PF VSI */ 8987 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 8988 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 8989 veb = pf->veb[i]; 8990 } 8991 8992 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8993 8994 nla_for_each_nested(attr, br_spec, rem) { 8995 __u16 mode; 8996 8997 if (nla_type(attr) != IFLA_BRIDGE_MODE) 8998 continue; 8999 9000 mode = nla_get_u16(attr); 9001 if ((mode != BRIDGE_MODE_VEPA) && 9002 (mode != BRIDGE_MODE_VEB)) 9003 return -EINVAL; 9004 9005 /* Insert a new HW bridge */ 9006 if (!veb) { 9007 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9008 vsi->tc_config.enabled_tc); 9009 if (veb) { 9010 veb->bridge_mode = mode; 9011 i40e_config_bridge_mode(veb); 9012 } else { 9013 /* No Bridge HW offload available */ 9014 return -ENOENT; 9015 } 9016 break; 9017 } else if (mode != veb->bridge_mode) { 9018 /* Existing HW bridge but different mode needs reset */ 9019 veb->bridge_mode = mode; 9020 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 9021 if (mode == BRIDGE_MODE_VEB) 9022 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 9023 else 9024 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9025 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 9026 break; 9027 } 9028 } 9029 9030 return 0; 9031 } 9032 9033 /** 9034 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 9035 * @skb: skb buff 9036 * @pid: process id 9037 * @seq: RTNL message seq # 9038 * @dev: the netdev being configured 9039 * @filter_mask: unused 9040 * @nlflags: netlink flags passed in 9041 * 9042 * Return the mode in which the hardware bridge is operating in 9043 * i.e VEB or VEPA. 9044 **/ 9045 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9046 struct net_device *dev, 9047 u32 __always_unused filter_mask, 9048 int nlflags) 9049 { 9050 struct i40e_netdev_priv *np = netdev_priv(dev); 9051 struct i40e_vsi *vsi = np->vsi; 9052 struct i40e_pf *pf = vsi->back; 9053 struct i40e_veb *veb = NULL; 9054 int i; 9055 9056 /* Only for PF VSI for now */ 9057 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9058 return -EOPNOTSUPP; 9059 9060 /* Find the HW bridge for the PF VSI */ 9061 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9062 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9063 veb = pf->veb[i]; 9064 } 9065 9066 if (!veb) 9067 return 0; 9068 9069 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 9070 0, 0, nlflags, filter_mask, NULL); 9071 } 9072 9073 /* Hardware supports L4 tunnel length of 128B (=2^7) which includes 9074 * inner mac plus all inner ethertypes. 9075 */ 9076 #define I40E_MAX_TUNNEL_HDR_LEN 128 9077 /** 9078 * i40e_features_check - Validate encapsulated packet conforms to limits 9079 * @skb: skb buff 9080 * @dev: This physical port's netdev 9081 * @features: Offload features that the stack believes apply 9082 **/ 9083 static netdev_features_t i40e_features_check(struct sk_buff *skb, 9084 struct net_device *dev, 9085 netdev_features_t features) 9086 { 9087 if (skb->encapsulation && 9088 ((skb_inner_network_header(skb) - skb_transport_header(skb)) > 9089 I40E_MAX_TUNNEL_HDR_LEN)) 9090 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9091 9092 return features; 9093 } 9094 9095 static const struct net_device_ops i40e_netdev_ops = { 9096 .ndo_open = i40e_open, 9097 .ndo_stop = i40e_close, 9098 .ndo_start_xmit = i40e_lan_xmit_frame, 9099 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 9100 .ndo_set_rx_mode = i40e_set_rx_mode, 9101 .ndo_validate_addr = eth_validate_addr, 9102 .ndo_set_mac_address = i40e_set_mac, 9103 .ndo_change_mtu = i40e_change_mtu, 9104 .ndo_do_ioctl = i40e_ioctl, 9105 .ndo_tx_timeout = i40e_tx_timeout, 9106 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 9107 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 9108 #ifdef CONFIG_NET_POLL_CONTROLLER 9109 .ndo_poll_controller = i40e_netpoll, 9110 #endif 9111 .ndo_setup_tc = __i40e_setup_tc, 9112 #ifdef I40E_FCOE 9113 .ndo_fcoe_enable = i40e_fcoe_enable, 9114 .ndo_fcoe_disable = i40e_fcoe_disable, 9115 #endif 9116 .ndo_set_features = i40e_set_features, 9117 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 9118 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 9119 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 9120 .ndo_get_vf_config = i40e_ndo_get_vf_config, 9121 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 9122 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 9123 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 9124 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 9125 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 9126 .ndo_get_phys_port_id = i40e_get_phys_port_id, 9127 .ndo_fdb_add = i40e_ndo_fdb_add, 9128 .ndo_features_check = i40e_features_check, 9129 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 9130 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 9131 }; 9132 9133 /** 9134 * i40e_config_netdev - Setup the netdev flags 9135 * @vsi: the VSI being configured 9136 * 9137 * Returns 0 on success, negative value on failure 9138 **/ 9139 static int i40e_config_netdev(struct i40e_vsi *vsi) 9140 { 9141 struct i40e_pf *pf = vsi->back; 9142 struct i40e_hw *hw = &pf->hw; 9143 struct i40e_netdev_priv *np; 9144 struct net_device *netdev; 9145 u8 mac_addr[ETH_ALEN]; 9146 int etherdev_size; 9147 9148 etherdev_size = sizeof(struct i40e_netdev_priv); 9149 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 9150 if (!netdev) 9151 return -ENOMEM; 9152 9153 vsi->netdev = netdev; 9154 np = netdev_priv(netdev); 9155 np->vsi = vsi; 9156 9157 netdev->hw_enc_features |= NETIF_F_SG | 9158 NETIF_F_IP_CSUM | 9159 NETIF_F_IPV6_CSUM | 9160 NETIF_F_HIGHDMA | 9161 NETIF_F_SOFT_FEATURES | 9162 NETIF_F_TSO | 9163 NETIF_F_TSO_ECN | 9164 NETIF_F_TSO6 | 9165 NETIF_F_GSO_GRE | 9166 NETIF_F_GSO_GRE_CSUM | 9167 NETIF_F_GSO_IPXIP4 | 9168 NETIF_F_GSO_IPXIP6 | 9169 NETIF_F_GSO_UDP_TUNNEL | 9170 NETIF_F_GSO_UDP_TUNNEL_CSUM | 9171 NETIF_F_GSO_PARTIAL | 9172 NETIF_F_SCTP_CRC | 9173 NETIF_F_RXHASH | 9174 NETIF_F_RXCSUM | 9175 0; 9176 9177 if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE)) 9178 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 9179 9180 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 9181 9182 /* record features VLANs can make use of */ 9183 netdev->vlan_features |= netdev->hw_enc_features | 9184 NETIF_F_TSO_MANGLEID; 9185 9186 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 9187 netdev->hw_features |= NETIF_F_NTUPLE; 9188 9189 netdev->hw_features |= netdev->hw_enc_features | 9190 NETIF_F_HW_VLAN_CTAG_TX | 9191 NETIF_F_HW_VLAN_CTAG_RX; 9192 9193 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 9194 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9195 9196 if (vsi->type == I40E_VSI_MAIN) { 9197 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 9198 ether_addr_copy(mac_addr, hw->mac.perm_addr); 9199 spin_lock_bh(&vsi->mac_filter_hash_lock); 9200 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY); 9201 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9202 } else { 9203 /* relate the VSI_VMDQ name to the VSI_MAIN name */ 9204 snprintf(netdev->name, IFNAMSIZ, "%sv%%d", 9205 pf->vsi[pf->lan_vsi]->netdev->name); 9206 random_ether_addr(mac_addr); 9207 9208 spin_lock_bh(&vsi->mac_filter_hash_lock); 9209 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY); 9210 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9211 } 9212 9213 ether_addr_copy(netdev->dev_addr, mac_addr); 9214 ether_addr_copy(netdev->perm_addr, mac_addr); 9215 9216 netdev->priv_flags |= IFF_UNICAST_FLT; 9217 netdev->priv_flags |= IFF_SUPP_NOFCS; 9218 /* Setup netdev TC information */ 9219 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 9220 9221 netdev->netdev_ops = &i40e_netdev_ops; 9222 netdev->watchdog_timeo = 5 * HZ; 9223 i40e_set_ethtool_ops(netdev); 9224 #ifdef I40E_FCOE 9225 i40e_fcoe_config_netdev(netdev, vsi); 9226 #endif 9227 9228 /* MTU range: 68 - 9706 */ 9229 netdev->min_mtu = ETH_MIN_MTU; 9230 netdev->max_mtu = I40E_MAX_RXBUFFER - 9231 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); 9232 9233 return 0; 9234 } 9235 9236 /** 9237 * i40e_vsi_delete - Delete a VSI from the switch 9238 * @vsi: the VSI being removed 9239 * 9240 * Returns 0 on success, negative value on failure 9241 **/ 9242 static void i40e_vsi_delete(struct i40e_vsi *vsi) 9243 { 9244 /* remove default VSI is not allowed */ 9245 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 9246 return; 9247 9248 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 9249 } 9250 9251 /** 9252 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 9253 * @vsi: the VSI being queried 9254 * 9255 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 9256 **/ 9257 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 9258 { 9259 struct i40e_veb *veb; 9260 struct i40e_pf *pf = vsi->back; 9261 9262 /* Uplink is not a bridge so default to VEB */ 9263 if (vsi->veb_idx == I40E_NO_VEB) 9264 return 1; 9265 9266 veb = pf->veb[vsi->veb_idx]; 9267 if (!veb) { 9268 dev_info(&pf->pdev->dev, 9269 "There is no veb associated with the bridge\n"); 9270 return -ENOENT; 9271 } 9272 9273 /* Uplink is a bridge in VEPA mode */ 9274 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 9275 return 0; 9276 } else { 9277 /* Uplink is a bridge in VEB mode */ 9278 return 1; 9279 } 9280 9281 /* VEPA is now default bridge, so return 0 */ 9282 return 0; 9283 } 9284 9285 /** 9286 * i40e_add_vsi - Add a VSI to the switch 9287 * @vsi: the VSI being configured 9288 * 9289 * This initializes a VSI context depending on the VSI type to be added and 9290 * passes it down to the add_vsi aq command. 9291 **/ 9292 static int i40e_add_vsi(struct i40e_vsi *vsi) 9293 { 9294 int ret = -ENODEV; 9295 i40e_status aq_ret = 0; 9296 struct i40e_pf *pf = vsi->back; 9297 struct i40e_hw *hw = &pf->hw; 9298 struct i40e_vsi_context ctxt; 9299 struct i40e_mac_filter *f; 9300 struct hlist_node *h; 9301 int bkt; 9302 9303 u8 enabled_tc = 0x1; /* TC0 enabled */ 9304 int f_count = 0; 9305 9306 memset(&ctxt, 0, sizeof(ctxt)); 9307 switch (vsi->type) { 9308 case I40E_VSI_MAIN: 9309 /* The PF's main VSI is already setup as part of the 9310 * device initialization, so we'll not bother with 9311 * the add_vsi call, but we will retrieve the current 9312 * VSI context. 9313 */ 9314 ctxt.seid = pf->main_vsi_seid; 9315 ctxt.pf_num = pf->hw.pf_id; 9316 ctxt.vf_num = 0; 9317 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9318 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9319 if (ret) { 9320 dev_info(&pf->pdev->dev, 9321 "couldn't get PF vsi config, err %s aq_err %s\n", 9322 i40e_stat_str(&pf->hw, ret), 9323 i40e_aq_str(&pf->hw, 9324 pf->hw.aq.asq_last_status)); 9325 return -ENOENT; 9326 } 9327 vsi->info = ctxt.info; 9328 vsi->info.valid_sections = 0; 9329 9330 vsi->seid = ctxt.seid; 9331 vsi->id = ctxt.vsi_number; 9332 9333 enabled_tc = i40e_pf_get_tc_map(pf); 9334 9335 /* MFP mode setup queue map and update VSI */ 9336 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 9337 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 9338 memset(&ctxt, 0, sizeof(ctxt)); 9339 ctxt.seid = pf->main_vsi_seid; 9340 ctxt.pf_num = pf->hw.pf_id; 9341 ctxt.vf_num = 0; 9342 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 9343 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 9344 if (ret) { 9345 dev_info(&pf->pdev->dev, 9346 "update vsi failed, err %s aq_err %s\n", 9347 i40e_stat_str(&pf->hw, ret), 9348 i40e_aq_str(&pf->hw, 9349 pf->hw.aq.asq_last_status)); 9350 ret = -ENOENT; 9351 goto err; 9352 } 9353 /* update the local VSI info queue map */ 9354 i40e_vsi_update_queue_map(vsi, &ctxt); 9355 vsi->info.valid_sections = 0; 9356 } else { 9357 /* Default/Main VSI is only enabled for TC0 9358 * reconfigure it to enable all TCs that are 9359 * available on the port in SFP mode. 9360 * For MFP case the iSCSI PF would use this 9361 * flow to enable LAN+iSCSI TC. 9362 */ 9363 ret = i40e_vsi_config_tc(vsi, enabled_tc); 9364 if (ret) { 9365 dev_info(&pf->pdev->dev, 9366 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 9367 enabled_tc, 9368 i40e_stat_str(&pf->hw, ret), 9369 i40e_aq_str(&pf->hw, 9370 pf->hw.aq.asq_last_status)); 9371 ret = -ENOENT; 9372 } 9373 } 9374 break; 9375 9376 case I40E_VSI_FDIR: 9377 ctxt.pf_num = hw->pf_id; 9378 ctxt.vf_num = 0; 9379 ctxt.uplink_seid = vsi->uplink_seid; 9380 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9381 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9382 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 9383 (i40e_is_vsi_uplink_mode_veb(vsi))) { 9384 ctxt.info.valid_sections |= 9385 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9386 ctxt.info.switch_id = 9387 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9388 } 9389 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9390 break; 9391 9392 case I40E_VSI_VMDQ2: 9393 ctxt.pf_num = hw->pf_id; 9394 ctxt.vf_num = 0; 9395 ctxt.uplink_seid = vsi->uplink_seid; 9396 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9397 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 9398 9399 /* This VSI is connected to VEB so the switch_id 9400 * should be set to zero by default. 9401 */ 9402 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9403 ctxt.info.valid_sections |= 9404 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9405 ctxt.info.switch_id = 9406 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9407 } 9408 9409 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9410 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9411 break; 9412 9413 case I40E_VSI_SRIOV: 9414 ctxt.pf_num = hw->pf_id; 9415 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 9416 ctxt.uplink_seid = vsi->uplink_seid; 9417 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9418 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 9419 9420 /* This VSI is connected to VEB so the switch_id 9421 * should be set to zero by default. 9422 */ 9423 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9424 ctxt.info.valid_sections |= 9425 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9426 ctxt.info.switch_id = 9427 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9428 } 9429 9430 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 9431 ctxt.info.valid_sections |= 9432 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 9433 ctxt.info.queueing_opt_flags |= 9434 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 9435 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 9436 } 9437 9438 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 9439 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 9440 if (pf->vf[vsi->vf_id].spoofchk) { 9441 ctxt.info.valid_sections |= 9442 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 9443 ctxt.info.sec_flags |= 9444 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 9445 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 9446 } 9447 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9448 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9449 break; 9450 9451 #ifdef I40E_FCOE 9452 case I40E_VSI_FCOE: 9453 ret = i40e_fcoe_vsi_init(vsi, &ctxt); 9454 if (ret) { 9455 dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n"); 9456 return ret; 9457 } 9458 break; 9459 9460 #endif /* I40E_FCOE */ 9461 case I40E_VSI_IWARP: 9462 /* send down message to iWARP */ 9463 break; 9464 9465 default: 9466 return -ENODEV; 9467 } 9468 9469 if (vsi->type != I40E_VSI_MAIN) { 9470 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 9471 if (ret) { 9472 dev_info(&vsi->back->pdev->dev, 9473 "add vsi failed, err %s aq_err %s\n", 9474 i40e_stat_str(&pf->hw, ret), 9475 i40e_aq_str(&pf->hw, 9476 pf->hw.aq.asq_last_status)); 9477 ret = -ENOENT; 9478 goto err; 9479 } 9480 vsi->info = ctxt.info; 9481 vsi->info.valid_sections = 0; 9482 vsi->seid = ctxt.seid; 9483 vsi->id = ctxt.vsi_number; 9484 } 9485 /* Except FDIR VSI, for all othet VSI set the broadcast filter */ 9486 if (vsi->type != I40E_VSI_FDIR) { 9487 aq_ret = i40e_aq_set_vsi_broadcast(hw, vsi->seid, true, NULL); 9488 if (aq_ret) { 9489 ret = i40e_aq_rc_to_posix(aq_ret, 9490 hw->aq.asq_last_status); 9491 dev_info(&pf->pdev->dev, 9492 "set brdcast promisc failed, err %s, aq_err %s\n", 9493 i40e_stat_str(hw, aq_ret), 9494 i40e_aq_str(hw, hw->aq.asq_last_status)); 9495 } 9496 } 9497 9498 vsi->active_filters = 0; 9499 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 9500 spin_lock_bh(&vsi->mac_filter_hash_lock); 9501 /* If macvlan filters already exist, force them to get loaded */ 9502 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 9503 f->state = I40E_FILTER_NEW; 9504 f_count++; 9505 } 9506 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9507 9508 if (f_count) { 9509 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 9510 pf->flags |= I40E_FLAG_FILTER_SYNC; 9511 } 9512 9513 /* Update VSI BW information */ 9514 ret = i40e_vsi_get_bw_info(vsi); 9515 if (ret) { 9516 dev_info(&pf->pdev->dev, 9517 "couldn't get vsi bw info, err %s aq_err %s\n", 9518 i40e_stat_str(&pf->hw, ret), 9519 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9520 /* VSI is already added so not tearing that up */ 9521 ret = 0; 9522 } 9523 9524 err: 9525 return ret; 9526 } 9527 9528 /** 9529 * i40e_vsi_release - Delete a VSI and free its resources 9530 * @vsi: the VSI being removed 9531 * 9532 * Returns 0 on success or < 0 on error 9533 **/ 9534 int i40e_vsi_release(struct i40e_vsi *vsi) 9535 { 9536 struct i40e_mac_filter *f; 9537 struct hlist_node *h; 9538 struct i40e_veb *veb = NULL; 9539 struct i40e_pf *pf; 9540 u16 uplink_seid; 9541 int i, n, bkt; 9542 9543 pf = vsi->back; 9544 9545 /* release of a VEB-owner or last VSI is not allowed */ 9546 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 9547 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 9548 vsi->seid, vsi->uplink_seid); 9549 return -ENODEV; 9550 } 9551 if (vsi == pf->vsi[pf->lan_vsi] && 9552 !test_bit(__I40E_DOWN, &pf->state)) { 9553 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 9554 return -ENODEV; 9555 } 9556 9557 uplink_seid = vsi->uplink_seid; 9558 if (vsi->type != I40E_VSI_SRIOV) { 9559 if (vsi->netdev_registered) { 9560 vsi->netdev_registered = false; 9561 if (vsi->netdev) { 9562 /* results in a call to i40e_close() */ 9563 unregister_netdev(vsi->netdev); 9564 } 9565 } else { 9566 i40e_vsi_close(vsi); 9567 } 9568 i40e_vsi_disable_irq(vsi); 9569 } 9570 9571 spin_lock_bh(&vsi->mac_filter_hash_lock); 9572 9573 /* clear the sync flag on all filters */ 9574 if (vsi->netdev) { 9575 __dev_uc_unsync(vsi->netdev, NULL); 9576 __dev_mc_unsync(vsi->netdev, NULL); 9577 } 9578 9579 /* make sure any remaining filters are marked for deletion */ 9580 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 9581 __i40e_del_filter(vsi, f); 9582 9583 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9584 9585 i40e_sync_vsi_filters(vsi); 9586 9587 i40e_vsi_delete(vsi); 9588 i40e_vsi_free_q_vectors(vsi); 9589 if (vsi->netdev) { 9590 free_netdev(vsi->netdev); 9591 vsi->netdev = NULL; 9592 } 9593 i40e_vsi_clear_rings(vsi); 9594 i40e_vsi_clear(vsi); 9595 9596 /* If this was the last thing on the VEB, except for the 9597 * controlling VSI, remove the VEB, which puts the controlling 9598 * VSI onto the next level down in the switch. 9599 * 9600 * Well, okay, there's one more exception here: don't remove 9601 * the orphan VEBs yet. We'll wait for an explicit remove request 9602 * from up the network stack. 9603 */ 9604 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 9605 if (pf->vsi[i] && 9606 pf->vsi[i]->uplink_seid == uplink_seid && 9607 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 9608 n++; /* count the VSIs */ 9609 } 9610 } 9611 for (i = 0; i < I40E_MAX_VEB; i++) { 9612 if (!pf->veb[i]) 9613 continue; 9614 if (pf->veb[i]->uplink_seid == uplink_seid) 9615 n++; /* count the VEBs */ 9616 if (pf->veb[i]->seid == uplink_seid) 9617 veb = pf->veb[i]; 9618 } 9619 if (n == 0 && veb && veb->uplink_seid != 0) 9620 i40e_veb_release(veb); 9621 9622 return 0; 9623 } 9624 9625 /** 9626 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 9627 * @vsi: ptr to the VSI 9628 * 9629 * This should only be called after i40e_vsi_mem_alloc() which allocates the 9630 * corresponding SW VSI structure and initializes num_queue_pairs for the 9631 * newly allocated VSI. 9632 * 9633 * Returns 0 on success or negative on failure 9634 **/ 9635 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 9636 { 9637 int ret = -ENOENT; 9638 struct i40e_pf *pf = vsi->back; 9639 9640 if (vsi->q_vectors[0]) { 9641 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 9642 vsi->seid); 9643 return -EEXIST; 9644 } 9645 9646 if (vsi->base_vector) { 9647 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 9648 vsi->seid, vsi->base_vector); 9649 return -EEXIST; 9650 } 9651 9652 ret = i40e_vsi_alloc_q_vectors(vsi); 9653 if (ret) { 9654 dev_info(&pf->pdev->dev, 9655 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 9656 vsi->num_q_vectors, vsi->seid, ret); 9657 vsi->num_q_vectors = 0; 9658 goto vector_setup_out; 9659 } 9660 9661 /* In Legacy mode, we do not have to get any other vector since we 9662 * piggyback on the misc/ICR0 for queue interrupts. 9663 */ 9664 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 9665 return ret; 9666 if (vsi->num_q_vectors) 9667 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 9668 vsi->num_q_vectors, vsi->idx); 9669 if (vsi->base_vector < 0) { 9670 dev_info(&pf->pdev->dev, 9671 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 9672 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 9673 i40e_vsi_free_q_vectors(vsi); 9674 ret = -ENOENT; 9675 goto vector_setup_out; 9676 } 9677 9678 vector_setup_out: 9679 return ret; 9680 } 9681 9682 /** 9683 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 9684 * @vsi: pointer to the vsi. 9685 * 9686 * This re-allocates a vsi's queue resources. 9687 * 9688 * Returns pointer to the successfully allocated and configured VSI sw struct 9689 * on success, otherwise returns NULL on failure. 9690 **/ 9691 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 9692 { 9693 struct i40e_pf *pf; 9694 u8 enabled_tc; 9695 int ret; 9696 9697 if (!vsi) 9698 return NULL; 9699 9700 pf = vsi->back; 9701 9702 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9703 i40e_vsi_clear_rings(vsi); 9704 9705 i40e_vsi_free_arrays(vsi, false); 9706 i40e_set_num_rings_in_vsi(vsi); 9707 ret = i40e_vsi_alloc_arrays(vsi, false); 9708 if (ret) 9709 goto err_vsi; 9710 9711 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); 9712 if (ret < 0) { 9713 dev_info(&pf->pdev->dev, 9714 "failed to get tracking for %d queues for VSI %d err %d\n", 9715 vsi->alloc_queue_pairs, vsi->seid, ret); 9716 goto err_vsi; 9717 } 9718 vsi->base_queue = ret; 9719 9720 /* Update the FW view of the VSI. Force a reset of TC and queue 9721 * layout configurations. 9722 */ 9723 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 9724 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 9725 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 9726 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 9727 9728 /* assign it some queues */ 9729 ret = i40e_alloc_rings(vsi); 9730 if (ret) 9731 goto err_rings; 9732 9733 /* map all of the rings to the q_vectors */ 9734 i40e_vsi_map_rings_to_vectors(vsi); 9735 return vsi; 9736 9737 err_rings: 9738 i40e_vsi_free_q_vectors(vsi); 9739 if (vsi->netdev_registered) { 9740 vsi->netdev_registered = false; 9741 unregister_netdev(vsi->netdev); 9742 free_netdev(vsi->netdev); 9743 vsi->netdev = NULL; 9744 } 9745 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 9746 err_vsi: 9747 i40e_vsi_clear(vsi); 9748 return NULL; 9749 } 9750 9751 /** 9752 * i40e_vsi_setup - Set up a VSI by a given type 9753 * @pf: board private structure 9754 * @type: VSI type 9755 * @uplink_seid: the switch element to link to 9756 * @param1: usage depends upon VSI type. For VF types, indicates VF id 9757 * 9758 * This allocates the sw VSI structure and its queue resources, then add a VSI 9759 * to the identified VEB. 9760 * 9761 * Returns pointer to the successfully allocated and configure VSI sw struct on 9762 * success, otherwise returns NULL on failure. 9763 **/ 9764 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 9765 u16 uplink_seid, u32 param1) 9766 { 9767 struct i40e_vsi *vsi = NULL; 9768 struct i40e_veb *veb = NULL; 9769 int ret, i; 9770 int v_idx; 9771 9772 /* The requested uplink_seid must be either 9773 * - the PF's port seid 9774 * no VEB is needed because this is the PF 9775 * or this is a Flow Director special case VSI 9776 * - seid of an existing VEB 9777 * - seid of a VSI that owns an existing VEB 9778 * - seid of a VSI that doesn't own a VEB 9779 * a new VEB is created and the VSI becomes the owner 9780 * - seid of the PF VSI, which is what creates the first VEB 9781 * this is a special case of the previous 9782 * 9783 * Find which uplink_seid we were given and create a new VEB if needed 9784 */ 9785 for (i = 0; i < I40E_MAX_VEB; i++) { 9786 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 9787 veb = pf->veb[i]; 9788 break; 9789 } 9790 } 9791 9792 if (!veb && uplink_seid != pf->mac_seid) { 9793 9794 for (i = 0; i < pf->num_alloc_vsi; i++) { 9795 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 9796 vsi = pf->vsi[i]; 9797 break; 9798 } 9799 } 9800 if (!vsi) { 9801 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 9802 uplink_seid); 9803 return NULL; 9804 } 9805 9806 if (vsi->uplink_seid == pf->mac_seid) 9807 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 9808 vsi->tc_config.enabled_tc); 9809 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 9810 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9811 vsi->tc_config.enabled_tc); 9812 if (veb) { 9813 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 9814 dev_info(&vsi->back->pdev->dev, 9815 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 9816 return NULL; 9817 } 9818 /* We come up by default in VEPA mode if SRIOV is not 9819 * already enabled, in which case we can't force VEPA 9820 * mode. 9821 */ 9822 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 9823 veb->bridge_mode = BRIDGE_MODE_VEPA; 9824 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9825 } 9826 i40e_config_bridge_mode(veb); 9827 } 9828 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9829 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9830 veb = pf->veb[i]; 9831 } 9832 if (!veb) { 9833 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 9834 return NULL; 9835 } 9836 9837 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 9838 uplink_seid = veb->seid; 9839 } 9840 9841 /* get vsi sw struct */ 9842 v_idx = i40e_vsi_mem_alloc(pf, type); 9843 if (v_idx < 0) 9844 goto err_alloc; 9845 vsi = pf->vsi[v_idx]; 9846 if (!vsi) 9847 goto err_alloc; 9848 vsi->type = type; 9849 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 9850 9851 if (type == I40E_VSI_MAIN) 9852 pf->lan_vsi = v_idx; 9853 else if (type == I40E_VSI_SRIOV) 9854 vsi->vf_id = param1; 9855 /* assign it some queues */ 9856 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, 9857 vsi->idx); 9858 if (ret < 0) { 9859 dev_info(&pf->pdev->dev, 9860 "failed to get tracking for %d queues for VSI %d err=%d\n", 9861 vsi->alloc_queue_pairs, vsi->seid, ret); 9862 goto err_vsi; 9863 } 9864 vsi->base_queue = ret; 9865 9866 /* get a VSI from the hardware */ 9867 vsi->uplink_seid = uplink_seid; 9868 ret = i40e_add_vsi(vsi); 9869 if (ret) 9870 goto err_vsi; 9871 9872 switch (vsi->type) { 9873 /* setup the netdev if needed */ 9874 case I40E_VSI_MAIN: 9875 /* Apply relevant filters if a platform-specific mac 9876 * address was selected. 9877 */ 9878 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 9879 ret = i40e_macaddr_init(vsi, pf->hw.mac.addr); 9880 if (ret) { 9881 dev_warn(&pf->pdev->dev, 9882 "could not set up macaddr; err %d\n", 9883 ret); 9884 } 9885 } 9886 case I40E_VSI_VMDQ2: 9887 case I40E_VSI_FCOE: 9888 ret = i40e_config_netdev(vsi); 9889 if (ret) 9890 goto err_netdev; 9891 ret = register_netdev(vsi->netdev); 9892 if (ret) 9893 goto err_netdev; 9894 vsi->netdev_registered = true; 9895 netif_carrier_off(vsi->netdev); 9896 #ifdef CONFIG_I40E_DCB 9897 /* Setup DCB netlink interface */ 9898 i40e_dcbnl_setup(vsi); 9899 #endif /* CONFIG_I40E_DCB */ 9900 /* fall through */ 9901 9902 case I40E_VSI_FDIR: 9903 /* set up vectors and rings if needed */ 9904 ret = i40e_vsi_setup_vectors(vsi); 9905 if (ret) 9906 goto err_msix; 9907 9908 ret = i40e_alloc_rings(vsi); 9909 if (ret) 9910 goto err_rings; 9911 9912 /* map all of the rings to the q_vectors */ 9913 i40e_vsi_map_rings_to_vectors(vsi); 9914 9915 i40e_vsi_reset_stats(vsi); 9916 break; 9917 9918 default: 9919 /* no netdev or rings for the other VSI types */ 9920 break; 9921 } 9922 9923 if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) && 9924 (vsi->type == I40E_VSI_VMDQ2)) { 9925 ret = i40e_vsi_config_rss(vsi); 9926 } 9927 return vsi; 9928 9929 err_rings: 9930 i40e_vsi_free_q_vectors(vsi); 9931 err_msix: 9932 if (vsi->netdev_registered) { 9933 vsi->netdev_registered = false; 9934 unregister_netdev(vsi->netdev); 9935 free_netdev(vsi->netdev); 9936 vsi->netdev = NULL; 9937 } 9938 err_netdev: 9939 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 9940 err_vsi: 9941 i40e_vsi_clear(vsi); 9942 err_alloc: 9943 return NULL; 9944 } 9945 9946 /** 9947 * i40e_veb_get_bw_info - Query VEB BW information 9948 * @veb: the veb to query 9949 * 9950 * Query the Tx scheduler BW configuration data for given VEB 9951 **/ 9952 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 9953 { 9954 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 9955 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 9956 struct i40e_pf *pf = veb->pf; 9957 struct i40e_hw *hw = &pf->hw; 9958 u32 tc_bw_max; 9959 int ret = 0; 9960 int i; 9961 9962 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 9963 &bw_data, NULL); 9964 if (ret) { 9965 dev_info(&pf->pdev->dev, 9966 "query veb bw config failed, err %s aq_err %s\n", 9967 i40e_stat_str(&pf->hw, ret), 9968 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 9969 goto out; 9970 } 9971 9972 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 9973 &ets_data, NULL); 9974 if (ret) { 9975 dev_info(&pf->pdev->dev, 9976 "query veb bw ets config failed, err %s aq_err %s\n", 9977 i40e_stat_str(&pf->hw, ret), 9978 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 9979 goto out; 9980 } 9981 9982 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 9983 veb->bw_max_quanta = ets_data.tc_bw_max; 9984 veb->is_abs_credits = bw_data.absolute_credits_enable; 9985 veb->enabled_tc = ets_data.tc_valid_bits; 9986 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 9987 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 9988 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 9989 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 9990 veb->bw_tc_limit_credits[i] = 9991 le16_to_cpu(bw_data.tc_bw_limits[i]); 9992 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 9993 } 9994 9995 out: 9996 return ret; 9997 } 9998 9999 /** 10000 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 10001 * @pf: board private structure 10002 * 10003 * On error: returns error code (negative) 10004 * On success: returns vsi index in PF (positive) 10005 **/ 10006 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 10007 { 10008 int ret = -ENOENT; 10009 struct i40e_veb *veb; 10010 int i; 10011 10012 /* Need to protect the allocation of switch elements at the PF level */ 10013 mutex_lock(&pf->switch_mutex); 10014 10015 /* VEB list may be fragmented if VEB creation/destruction has 10016 * been happening. We can afford to do a quick scan to look 10017 * for any free slots in the list. 10018 * 10019 * find next empty veb slot, looping back around if necessary 10020 */ 10021 i = 0; 10022 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 10023 i++; 10024 if (i >= I40E_MAX_VEB) { 10025 ret = -ENOMEM; 10026 goto err_alloc_veb; /* out of VEB slots! */ 10027 } 10028 10029 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 10030 if (!veb) { 10031 ret = -ENOMEM; 10032 goto err_alloc_veb; 10033 } 10034 veb->pf = pf; 10035 veb->idx = i; 10036 veb->enabled_tc = 1; 10037 10038 pf->veb[i] = veb; 10039 ret = i; 10040 err_alloc_veb: 10041 mutex_unlock(&pf->switch_mutex); 10042 return ret; 10043 } 10044 10045 /** 10046 * i40e_switch_branch_release - Delete a branch of the switch tree 10047 * @branch: where to start deleting 10048 * 10049 * This uses recursion to find the tips of the branch to be 10050 * removed, deleting until we get back to and can delete this VEB. 10051 **/ 10052 static void i40e_switch_branch_release(struct i40e_veb *branch) 10053 { 10054 struct i40e_pf *pf = branch->pf; 10055 u16 branch_seid = branch->seid; 10056 u16 veb_idx = branch->idx; 10057 int i; 10058 10059 /* release any VEBs on this VEB - RECURSION */ 10060 for (i = 0; i < I40E_MAX_VEB; i++) { 10061 if (!pf->veb[i]) 10062 continue; 10063 if (pf->veb[i]->uplink_seid == branch->seid) 10064 i40e_switch_branch_release(pf->veb[i]); 10065 } 10066 10067 /* Release the VSIs on this VEB, but not the owner VSI. 10068 * 10069 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 10070 * the VEB itself, so don't use (*branch) after this loop. 10071 */ 10072 for (i = 0; i < pf->num_alloc_vsi; i++) { 10073 if (!pf->vsi[i]) 10074 continue; 10075 if (pf->vsi[i]->uplink_seid == branch_seid && 10076 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10077 i40e_vsi_release(pf->vsi[i]); 10078 } 10079 } 10080 10081 /* There's one corner case where the VEB might not have been 10082 * removed, so double check it here and remove it if needed. 10083 * This case happens if the veb was created from the debugfs 10084 * commands and no VSIs were added to it. 10085 */ 10086 if (pf->veb[veb_idx]) 10087 i40e_veb_release(pf->veb[veb_idx]); 10088 } 10089 10090 /** 10091 * i40e_veb_clear - remove veb struct 10092 * @veb: the veb to remove 10093 **/ 10094 static void i40e_veb_clear(struct i40e_veb *veb) 10095 { 10096 if (!veb) 10097 return; 10098 10099 if (veb->pf) { 10100 struct i40e_pf *pf = veb->pf; 10101 10102 mutex_lock(&pf->switch_mutex); 10103 if (pf->veb[veb->idx] == veb) 10104 pf->veb[veb->idx] = NULL; 10105 mutex_unlock(&pf->switch_mutex); 10106 } 10107 10108 kfree(veb); 10109 } 10110 10111 /** 10112 * i40e_veb_release - Delete a VEB and free its resources 10113 * @veb: the VEB being removed 10114 **/ 10115 void i40e_veb_release(struct i40e_veb *veb) 10116 { 10117 struct i40e_vsi *vsi = NULL; 10118 struct i40e_pf *pf; 10119 int i, n = 0; 10120 10121 pf = veb->pf; 10122 10123 /* find the remaining VSI and check for extras */ 10124 for (i = 0; i < pf->num_alloc_vsi; i++) { 10125 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 10126 n++; 10127 vsi = pf->vsi[i]; 10128 } 10129 } 10130 if (n != 1) { 10131 dev_info(&pf->pdev->dev, 10132 "can't remove VEB %d with %d VSIs left\n", 10133 veb->seid, n); 10134 return; 10135 } 10136 10137 /* move the remaining VSI to uplink veb */ 10138 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 10139 if (veb->uplink_seid) { 10140 vsi->uplink_seid = veb->uplink_seid; 10141 if (veb->uplink_seid == pf->mac_seid) 10142 vsi->veb_idx = I40E_NO_VEB; 10143 else 10144 vsi->veb_idx = veb->veb_idx; 10145 } else { 10146 /* floating VEB */ 10147 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10148 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 10149 } 10150 10151 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10152 i40e_veb_clear(veb); 10153 } 10154 10155 /** 10156 * i40e_add_veb - create the VEB in the switch 10157 * @veb: the VEB to be instantiated 10158 * @vsi: the controlling VSI 10159 **/ 10160 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 10161 { 10162 struct i40e_pf *pf = veb->pf; 10163 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 10164 int ret; 10165 10166 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 10167 veb->enabled_tc, false, 10168 &veb->seid, enable_stats, NULL); 10169 10170 /* get a VEB from the hardware */ 10171 if (ret) { 10172 dev_info(&pf->pdev->dev, 10173 "couldn't add VEB, err %s aq_err %s\n", 10174 i40e_stat_str(&pf->hw, ret), 10175 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10176 return -EPERM; 10177 } 10178 10179 /* get statistics counter */ 10180 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 10181 &veb->stats_idx, NULL, NULL, NULL); 10182 if (ret) { 10183 dev_info(&pf->pdev->dev, 10184 "couldn't get VEB statistics idx, err %s aq_err %s\n", 10185 i40e_stat_str(&pf->hw, ret), 10186 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10187 return -EPERM; 10188 } 10189 ret = i40e_veb_get_bw_info(veb); 10190 if (ret) { 10191 dev_info(&pf->pdev->dev, 10192 "couldn't get VEB bw info, err %s aq_err %s\n", 10193 i40e_stat_str(&pf->hw, ret), 10194 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10195 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10196 return -ENOENT; 10197 } 10198 10199 vsi->uplink_seid = veb->seid; 10200 vsi->veb_idx = veb->idx; 10201 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10202 10203 return 0; 10204 } 10205 10206 /** 10207 * i40e_veb_setup - Set up a VEB 10208 * @pf: board private structure 10209 * @flags: VEB setup flags 10210 * @uplink_seid: the switch element to link to 10211 * @vsi_seid: the initial VSI seid 10212 * @enabled_tc: Enabled TC bit-map 10213 * 10214 * This allocates the sw VEB structure and links it into the switch 10215 * It is possible and legal for this to be a duplicate of an already 10216 * existing VEB. It is also possible for both uplink and vsi seids 10217 * to be zero, in order to create a floating VEB. 10218 * 10219 * Returns pointer to the successfully allocated VEB sw struct on 10220 * success, otherwise returns NULL on failure. 10221 **/ 10222 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 10223 u16 uplink_seid, u16 vsi_seid, 10224 u8 enabled_tc) 10225 { 10226 struct i40e_veb *veb, *uplink_veb = NULL; 10227 int vsi_idx, veb_idx; 10228 int ret; 10229 10230 /* if one seid is 0, the other must be 0 to create a floating relay */ 10231 if ((uplink_seid == 0 || vsi_seid == 0) && 10232 (uplink_seid + vsi_seid != 0)) { 10233 dev_info(&pf->pdev->dev, 10234 "one, not both seid's are 0: uplink=%d vsi=%d\n", 10235 uplink_seid, vsi_seid); 10236 return NULL; 10237 } 10238 10239 /* make sure there is such a vsi and uplink */ 10240 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 10241 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 10242 break; 10243 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 10244 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 10245 vsi_seid); 10246 return NULL; 10247 } 10248 10249 if (uplink_seid && uplink_seid != pf->mac_seid) { 10250 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10251 if (pf->veb[veb_idx] && 10252 pf->veb[veb_idx]->seid == uplink_seid) { 10253 uplink_veb = pf->veb[veb_idx]; 10254 break; 10255 } 10256 } 10257 if (!uplink_veb) { 10258 dev_info(&pf->pdev->dev, 10259 "uplink seid %d not found\n", uplink_seid); 10260 return NULL; 10261 } 10262 } 10263 10264 /* get veb sw struct */ 10265 veb_idx = i40e_veb_mem_alloc(pf); 10266 if (veb_idx < 0) 10267 goto err_alloc; 10268 veb = pf->veb[veb_idx]; 10269 veb->flags = flags; 10270 veb->uplink_seid = uplink_seid; 10271 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 10272 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 10273 10274 /* create the VEB in the switch */ 10275 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 10276 if (ret) 10277 goto err_veb; 10278 if (vsi_idx == pf->lan_vsi) 10279 pf->lan_veb = veb->idx; 10280 10281 return veb; 10282 10283 err_veb: 10284 i40e_veb_clear(veb); 10285 err_alloc: 10286 return NULL; 10287 } 10288 10289 /** 10290 * i40e_setup_pf_switch_element - set PF vars based on switch type 10291 * @pf: board private structure 10292 * @ele: element we are building info from 10293 * @num_reported: total number of elements 10294 * @printconfig: should we print the contents 10295 * 10296 * helper function to assist in extracting a few useful SEID values. 10297 **/ 10298 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 10299 struct i40e_aqc_switch_config_element_resp *ele, 10300 u16 num_reported, bool printconfig) 10301 { 10302 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 10303 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 10304 u8 element_type = ele->element_type; 10305 u16 seid = le16_to_cpu(ele->seid); 10306 10307 if (printconfig) 10308 dev_info(&pf->pdev->dev, 10309 "type=%d seid=%d uplink=%d downlink=%d\n", 10310 element_type, seid, uplink_seid, downlink_seid); 10311 10312 switch (element_type) { 10313 case I40E_SWITCH_ELEMENT_TYPE_MAC: 10314 pf->mac_seid = seid; 10315 break; 10316 case I40E_SWITCH_ELEMENT_TYPE_VEB: 10317 /* Main VEB? */ 10318 if (uplink_seid != pf->mac_seid) 10319 break; 10320 if (pf->lan_veb == I40E_NO_VEB) { 10321 int v; 10322 10323 /* find existing or else empty VEB */ 10324 for (v = 0; v < I40E_MAX_VEB; v++) { 10325 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 10326 pf->lan_veb = v; 10327 break; 10328 } 10329 } 10330 if (pf->lan_veb == I40E_NO_VEB) { 10331 v = i40e_veb_mem_alloc(pf); 10332 if (v < 0) 10333 break; 10334 pf->lan_veb = v; 10335 } 10336 } 10337 10338 pf->veb[pf->lan_veb]->seid = seid; 10339 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 10340 pf->veb[pf->lan_veb]->pf = pf; 10341 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 10342 break; 10343 case I40E_SWITCH_ELEMENT_TYPE_VSI: 10344 if (num_reported != 1) 10345 break; 10346 /* This is immediately after a reset so we can assume this is 10347 * the PF's VSI 10348 */ 10349 pf->mac_seid = uplink_seid; 10350 pf->pf_seid = downlink_seid; 10351 pf->main_vsi_seid = seid; 10352 if (printconfig) 10353 dev_info(&pf->pdev->dev, 10354 "pf_seid=%d main_vsi_seid=%d\n", 10355 pf->pf_seid, pf->main_vsi_seid); 10356 break; 10357 case I40E_SWITCH_ELEMENT_TYPE_PF: 10358 case I40E_SWITCH_ELEMENT_TYPE_VF: 10359 case I40E_SWITCH_ELEMENT_TYPE_EMP: 10360 case I40E_SWITCH_ELEMENT_TYPE_BMC: 10361 case I40E_SWITCH_ELEMENT_TYPE_PE: 10362 case I40E_SWITCH_ELEMENT_TYPE_PA: 10363 /* ignore these for now */ 10364 break; 10365 default: 10366 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 10367 element_type, seid); 10368 break; 10369 } 10370 } 10371 10372 /** 10373 * i40e_fetch_switch_configuration - Get switch config from firmware 10374 * @pf: board private structure 10375 * @printconfig: should we print the contents 10376 * 10377 * Get the current switch configuration from the device and 10378 * extract a few useful SEID values. 10379 **/ 10380 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 10381 { 10382 struct i40e_aqc_get_switch_config_resp *sw_config; 10383 u16 next_seid = 0; 10384 int ret = 0; 10385 u8 *aq_buf; 10386 int i; 10387 10388 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 10389 if (!aq_buf) 10390 return -ENOMEM; 10391 10392 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 10393 do { 10394 u16 num_reported, num_total; 10395 10396 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 10397 I40E_AQ_LARGE_BUF, 10398 &next_seid, NULL); 10399 if (ret) { 10400 dev_info(&pf->pdev->dev, 10401 "get switch config failed err %s aq_err %s\n", 10402 i40e_stat_str(&pf->hw, ret), 10403 i40e_aq_str(&pf->hw, 10404 pf->hw.aq.asq_last_status)); 10405 kfree(aq_buf); 10406 return -ENOENT; 10407 } 10408 10409 num_reported = le16_to_cpu(sw_config->header.num_reported); 10410 num_total = le16_to_cpu(sw_config->header.num_total); 10411 10412 if (printconfig) 10413 dev_info(&pf->pdev->dev, 10414 "header: %d reported %d total\n", 10415 num_reported, num_total); 10416 10417 for (i = 0; i < num_reported; i++) { 10418 struct i40e_aqc_switch_config_element_resp *ele = 10419 &sw_config->element[i]; 10420 10421 i40e_setup_pf_switch_element(pf, ele, num_reported, 10422 printconfig); 10423 } 10424 } while (next_seid != 0); 10425 10426 kfree(aq_buf); 10427 return ret; 10428 } 10429 10430 /** 10431 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 10432 * @pf: board private structure 10433 * @reinit: if the Main VSI needs to re-initialized. 10434 * 10435 * Returns 0 on success, negative value on failure 10436 **/ 10437 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 10438 { 10439 u16 flags = 0; 10440 int ret; 10441 10442 /* find out what's out there already */ 10443 ret = i40e_fetch_switch_configuration(pf, false); 10444 if (ret) { 10445 dev_info(&pf->pdev->dev, 10446 "couldn't fetch switch config, err %s aq_err %s\n", 10447 i40e_stat_str(&pf->hw, ret), 10448 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10449 return ret; 10450 } 10451 i40e_pf_reset_stats(pf); 10452 10453 /* set the switch config bit for the whole device to 10454 * support limited promisc or true promisc 10455 * when user requests promisc. The default is limited 10456 * promisc. 10457 */ 10458 10459 if ((pf->hw.pf_id == 0) && 10460 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) 10461 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10462 10463 if (pf->hw.pf_id == 0) { 10464 u16 valid_flags; 10465 10466 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10467 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 10468 NULL); 10469 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 10470 dev_info(&pf->pdev->dev, 10471 "couldn't set switch config bits, err %s aq_err %s\n", 10472 i40e_stat_str(&pf->hw, ret), 10473 i40e_aq_str(&pf->hw, 10474 pf->hw.aq.asq_last_status)); 10475 /* not a fatal problem, just keep going */ 10476 } 10477 } 10478 10479 /* first time setup */ 10480 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 10481 struct i40e_vsi *vsi = NULL; 10482 u16 uplink_seid; 10483 10484 /* Set up the PF VSI associated with the PF's main VSI 10485 * that is already in the HW switch 10486 */ 10487 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 10488 uplink_seid = pf->veb[pf->lan_veb]->seid; 10489 else 10490 uplink_seid = pf->mac_seid; 10491 if (pf->lan_vsi == I40E_NO_VSI) 10492 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 10493 else if (reinit) 10494 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 10495 if (!vsi) { 10496 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 10497 i40e_fdir_teardown(pf); 10498 return -EAGAIN; 10499 } 10500 } else { 10501 /* force a reset of TC and queue layout configurations */ 10502 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 10503 10504 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 10505 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 10506 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 10507 } 10508 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 10509 10510 i40e_fdir_sb_setup(pf); 10511 10512 /* Setup static PF queue filter control settings */ 10513 ret = i40e_setup_pf_filter_control(pf); 10514 if (ret) { 10515 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 10516 ret); 10517 /* Failure here should not stop continuing other steps */ 10518 } 10519 10520 /* enable RSS in the HW, even for only one queue, as the stack can use 10521 * the hash 10522 */ 10523 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 10524 i40e_pf_config_rss(pf); 10525 10526 /* fill in link information and enable LSE reporting */ 10527 i40e_update_link_info(&pf->hw); 10528 i40e_link_event(pf); 10529 10530 /* Initialize user-specific link properties */ 10531 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 10532 I40E_AQ_AN_COMPLETED) ? true : false); 10533 10534 i40e_ptp_init(pf); 10535 10536 return ret; 10537 } 10538 10539 /** 10540 * i40e_determine_queue_usage - Work out queue distribution 10541 * @pf: board private structure 10542 **/ 10543 static void i40e_determine_queue_usage(struct i40e_pf *pf) 10544 { 10545 int queues_left; 10546 10547 pf->num_lan_qps = 0; 10548 #ifdef I40E_FCOE 10549 pf->num_fcoe_qps = 0; 10550 #endif 10551 10552 /* Find the max queues to be put into basic use. We'll always be 10553 * using TC0, whether or not DCB is running, and TC0 will get the 10554 * big RSS set. 10555 */ 10556 queues_left = pf->hw.func_caps.num_tx_qp; 10557 10558 if ((queues_left == 1) || 10559 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 10560 /* one qp for PF, no queues for anything else */ 10561 queues_left = 0; 10562 pf->alloc_rss_size = pf->num_lan_qps = 1; 10563 10564 /* make sure all the fancies are disabled */ 10565 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10566 I40E_FLAG_IWARP_ENABLED | 10567 #ifdef I40E_FCOE 10568 I40E_FLAG_FCOE_ENABLED | 10569 #endif 10570 I40E_FLAG_FD_SB_ENABLED | 10571 I40E_FLAG_FD_ATR_ENABLED | 10572 I40E_FLAG_DCB_CAPABLE | 10573 I40E_FLAG_DCB_ENABLED | 10574 I40E_FLAG_SRIOV_ENABLED | 10575 I40E_FLAG_VMDQ_ENABLED); 10576 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 10577 I40E_FLAG_FD_SB_ENABLED | 10578 I40E_FLAG_FD_ATR_ENABLED | 10579 I40E_FLAG_DCB_CAPABLE))) { 10580 /* one qp for PF */ 10581 pf->alloc_rss_size = pf->num_lan_qps = 1; 10582 queues_left -= pf->num_lan_qps; 10583 10584 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10585 I40E_FLAG_IWARP_ENABLED | 10586 #ifdef I40E_FCOE 10587 I40E_FLAG_FCOE_ENABLED | 10588 #endif 10589 I40E_FLAG_FD_SB_ENABLED | 10590 I40E_FLAG_FD_ATR_ENABLED | 10591 I40E_FLAG_DCB_ENABLED | 10592 I40E_FLAG_VMDQ_ENABLED); 10593 } else { 10594 /* Not enough queues for all TCs */ 10595 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 10596 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 10597 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 10598 I40E_FLAG_DCB_ENABLED); 10599 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 10600 } 10601 pf->num_lan_qps = max_t(int, pf->rss_size_max, 10602 num_online_cpus()); 10603 pf->num_lan_qps = min_t(int, pf->num_lan_qps, 10604 pf->hw.func_caps.num_tx_qp); 10605 10606 queues_left -= pf->num_lan_qps; 10607 } 10608 10609 #ifdef I40E_FCOE 10610 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 10611 if (I40E_DEFAULT_FCOE <= queues_left) { 10612 pf->num_fcoe_qps = I40E_DEFAULT_FCOE; 10613 } else if (I40E_MINIMUM_FCOE <= queues_left) { 10614 pf->num_fcoe_qps = I40E_MINIMUM_FCOE; 10615 } else { 10616 pf->num_fcoe_qps = 0; 10617 pf->flags &= ~I40E_FLAG_FCOE_ENABLED; 10618 dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n"); 10619 } 10620 10621 queues_left -= pf->num_fcoe_qps; 10622 } 10623 10624 #endif 10625 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10626 if (queues_left > 1) { 10627 queues_left -= 1; /* save 1 queue for FD */ 10628 } else { 10629 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10630 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 10631 } 10632 } 10633 10634 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 10635 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 10636 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 10637 (queues_left / pf->num_vf_qps)); 10638 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 10639 } 10640 10641 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10642 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 10643 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 10644 (queues_left / pf->num_vmdq_qps)); 10645 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 10646 } 10647 10648 pf->queues_left = queues_left; 10649 dev_dbg(&pf->pdev->dev, 10650 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 10651 pf->hw.func_caps.num_tx_qp, 10652 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 10653 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 10654 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 10655 queues_left); 10656 #ifdef I40E_FCOE 10657 dev_dbg(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps); 10658 #endif 10659 } 10660 10661 /** 10662 * i40e_setup_pf_filter_control - Setup PF static filter control 10663 * @pf: PF to be setup 10664 * 10665 * i40e_setup_pf_filter_control sets up a PF's initial filter control 10666 * settings. If PE/FCoE are enabled then it will also set the per PF 10667 * based filter sizes required for them. It also enables Flow director, 10668 * ethertype and macvlan type filter settings for the pf. 10669 * 10670 * Returns 0 on success, negative on failure 10671 **/ 10672 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 10673 { 10674 struct i40e_filter_control_settings *settings = &pf->filter_settings; 10675 10676 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 10677 10678 /* Flow Director is enabled */ 10679 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 10680 settings->enable_fdir = true; 10681 10682 /* Ethtype and MACVLAN filters enabled for PF */ 10683 settings->enable_ethtype = true; 10684 settings->enable_macvlan = true; 10685 10686 if (i40e_set_filter_control(&pf->hw, settings)) 10687 return -ENOENT; 10688 10689 return 0; 10690 } 10691 10692 #define INFO_STRING_LEN 255 10693 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 10694 static void i40e_print_features(struct i40e_pf *pf) 10695 { 10696 struct i40e_hw *hw = &pf->hw; 10697 char *buf; 10698 int i; 10699 10700 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 10701 if (!buf) 10702 return; 10703 10704 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 10705 #ifdef CONFIG_PCI_IOV 10706 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 10707 #endif 10708 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 10709 pf->hw.func_caps.num_vsis, 10710 pf->vsi[pf->lan_vsi]->num_queue_pairs); 10711 if (pf->flags & I40E_FLAG_RSS_ENABLED) 10712 i += snprintf(&buf[i], REMAIN(i), " RSS"); 10713 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 10714 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 10715 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10716 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 10717 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 10718 } 10719 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 10720 i += snprintf(&buf[i], REMAIN(i), " DCB"); 10721 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 10722 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 10723 if (pf->flags & I40E_FLAG_PTP) 10724 i += snprintf(&buf[i], REMAIN(i), " PTP"); 10725 #ifdef I40E_FCOE 10726 if (pf->flags & I40E_FLAG_FCOE_ENABLED) 10727 i += snprintf(&buf[i], REMAIN(i), " FCOE"); 10728 #endif 10729 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 10730 i += snprintf(&buf[i], REMAIN(i), " VEB"); 10731 else 10732 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 10733 10734 dev_info(&pf->pdev->dev, "%s\n", buf); 10735 kfree(buf); 10736 WARN_ON(i > INFO_STRING_LEN); 10737 } 10738 10739 /** 10740 * i40e_get_platform_mac_addr - get platform-specific MAC address 10741 * 10742 * @pdev: PCI device information struct 10743 * @pf: board private structure 10744 * 10745 * Look up the MAC address in Open Firmware on systems that support it, 10746 * and use IDPROM on SPARC if no OF address is found. On return, the 10747 * I40E_FLAG_PF_MAC will be wset in pf->flags if a platform-specific value 10748 * has been selected. 10749 **/ 10750 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 10751 { 10752 pf->flags &= ~I40E_FLAG_PF_MAC; 10753 if (!eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 10754 pf->flags |= I40E_FLAG_PF_MAC; 10755 } 10756 10757 /** 10758 * i40e_probe - Device initialization routine 10759 * @pdev: PCI device information struct 10760 * @ent: entry in i40e_pci_tbl 10761 * 10762 * i40e_probe initializes a PF identified by a pci_dev structure. 10763 * The OS initialization, configuring of the PF private structure, 10764 * and a hardware reset occur. 10765 * 10766 * Returns 0 on success, negative on failure 10767 **/ 10768 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 10769 { 10770 struct i40e_aq_get_phy_abilities_resp abilities; 10771 struct i40e_pf *pf; 10772 struct i40e_hw *hw; 10773 static u16 pfs_found; 10774 u16 wol_nvm_bits; 10775 u16 link_status; 10776 int err; 10777 u32 val; 10778 u32 i; 10779 u8 set_fc_aq_fail; 10780 10781 err = pci_enable_device_mem(pdev); 10782 if (err) 10783 return err; 10784 10785 /* set up for high or low dma */ 10786 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 10787 if (err) { 10788 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 10789 if (err) { 10790 dev_err(&pdev->dev, 10791 "DMA configuration failed: 0x%x\n", err); 10792 goto err_dma; 10793 } 10794 } 10795 10796 /* set up pci connections */ 10797 err = pci_request_mem_regions(pdev, i40e_driver_name); 10798 if (err) { 10799 dev_info(&pdev->dev, 10800 "pci_request_selected_regions failed %d\n", err); 10801 goto err_pci_reg; 10802 } 10803 10804 pci_enable_pcie_error_reporting(pdev); 10805 pci_set_master(pdev); 10806 10807 /* Now that we have a PCI connection, we need to do the 10808 * low level device setup. This is primarily setting up 10809 * the Admin Queue structures and then querying for the 10810 * device's current profile information. 10811 */ 10812 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 10813 if (!pf) { 10814 err = -ENOMEM; 10815 goto err_pf_alloc; 10816 } 10817 pf->next_vsi = 0; 10818 pf->pdev = pdev; 10819 set_bit(__I40E_DOWN, &pf->state); 10820 10821 hw = &pf->hw; 10822 hw->back = pf; 10823 10824 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 10825 I40E_MAX_CSR_SPACE); 10826 10827 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 10828 if (!hw->hw_addr) { 10829 err = -EIO; 10830 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 10831 (unsigned int)pci_resource_start(pdev, 0), 10832 pf->ioremap_len, err); 10833 goto err_ioremap; 10834 } 10835 hw->vendor_id = pdev->vendor; 10836 hw->device_id = pdev->device; 10837 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 10838 hw->subsystem_vendor_id = pdev->subsystem_vendor; 10839 hw->subsystem_device_id = pdev->subsystem_device; 10840 hw->bus.device = PCI_SLOT(pdev->devfn); 10841 hw->bus.func = PCI_FUNC(pdev->devfn); 10842 pf->instance = pfs_found; 10843 10844 /* set up the locks for the AQ, do this only once in probe 10845 * and destroy them only once in remove 10846 */ 10847 mutex_init(&hw->aq.asq_mutex); 10848 mutex_init(&hw->aq.arq_mutex); 10849 10850 pf->msg_enable = netif_msg_init(debug, 10851 NETIF_MSG_DRV | 10852 NETIF_MSG_PROBE | 10853 NETIF_MSG_LINK); 10854 if (debug < -1) 10855 pf->hw.debug_mask = debug; 10856 10857 /* do a special CORER for clearing PXE mode once at init */ 10858 if (hw->revision_id == 0 && 10859 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 10860 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 10861 i40e_flush(hw); 10862 msleep(200); 10863 pf->corer_count++; 10864 10865 i40e_clear_pxe_mode(hw); 10866 } 10867 10868 /* Reset here to make sure all is clean and to define PF 'n' */ 10869 i40e_clear_hw(hw); 10870 err = i40e_pf_reset(hw); 10871 if (err) { 10872 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 10873 goto err_pf_reset; 10874 } 10875 pf->pfr_count++; 10876 10877 hw->aq.num_arq_entries = I40E_AQ_LEN; 10878 hw->aq.num_asq_entries = I40E_AQ_LEN; 10879 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 10880 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 10881 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 10882 10883 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 10884 "%s-%s:misc", 10885 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 10886 10887 err = i40e_init_shared_code(hw); 10888 if (err) { 10889 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 10890 err); 10891 goto err_pf_reset; 10892 } 10893 10894 /* set up a default setting for link flow control */ 10895 pf->hw.fc.requested_mode = I40E_FC_NONE; 10896 10897 err = i40e_init_adminq(hw); 10898 if (err) { 10899 if (err == I40E_ERR_FIRMWARE_API_VERSION) 10900 dev_info(&pdev->dev, 10901 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 10902 else 10903 dev_info(&pdev->dev, 10904 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 10905 10906 goto err_pf_reset; 10907 } 10908 10909 /* provide nvm, fw, api versions */ 10910 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 10911 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 10912 hw->aq.api_maj_ver, hw->aq.api_min_ver, 10913 i40e_nvm_version_str(hw)); 10914 10915 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 10916 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 10917 dev_info(&pdev->dev, 10918 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 10919 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 10920 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 10921 dev_info(&pdev->dev, 10922 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 10923 10924 i40e_verify_eeprom(pf); 10925 10926 /* Rev 0 hardware was never productized */ 10927 if (hw->revision_id < 1) 10928 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 10929 10930 i40e_clear_pxe_mode(hw); 10931 err = i40e_get_capabilities(pf); 10932 if (err) 10933 goto err_adminq_setup; 10934 10935 err = i40e_sw_init(pf); 10936 if (err) { 10937 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 10938 goto err_sw_init; 10939 } 10940 10941 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10942 hw->func_caps.num_rx_qp, 10943 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 10944 if (err) { 10945 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 10946 goto err_init_lan_hmc; 10947 } 10948 10949 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10950 if (err) { 10951 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 10952 err = -ENOENT; 10953 goto err_configure_lan_hmc; 10954 } 10955 10956 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 10957 * Ignore error return codes because if it was already disabled via 10958 * hardware settings this will fail 10959 */ 10960 if (pf->flags & I40E_FLAG_STOP_FW_LLDP) { 10961 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 10962 i40e_aq_stop_lldp(hw, true, NULL); 10963 } 10964 10965 i40e_get_mac_addr(hw, hw->mac.addr); 10966 /* allow a platform config to override the HW addr */ 10967 i40e_get_platform_mac_addr(pdev, pf); 10968 if (!is_valid_ether_addr(hw->mac.addr)) { 10969 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 10970 err = -EIO; 10971 goto err_mac_addr; 10972 } 10973 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 10974 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 10975 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 10976 if (is_valid_ether_addr(hw->mac.port_addr)) 10977 pf->flags |= I40E_FLAG_PORT_ID_VALID; 10978 #ifdef I40E_FCOE 10979 err = i40e_get_san_mac_addr(hw, hw->mac.san_addr); 10980 if (err) 10981 dev_info(&pdev->dev, 10982 "(non-fatal) SAN MAC retrieval failed: %d\n", err); 10983 if (!is_valid_ether_addr(hw->mac.san_addr)) { 10984 dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n", 10985 hw->mac.san_addr); 10986 ether_addr_copy(hw->mac.san_addr, hw->mac.addr); 10987 } 10988 dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr); 10989 #endif /* I40E_FCOE */ 10990 10991 pci_set_drvdata(pdev, pf); 10992 pci_save_state(pdev); 10993 #ifdef CONFIG_I40E_DCB 10994 err = i40e_init_pf_dcb(pf); 10995 if (err) { 10996 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 10997 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 10998 /* Continue without DCB enabled */ 10999 } 11000 #endif /* CONFIG_I40E_DCB */ 11001 11002 /* set up periodic task facility */ 11003 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 11004 pf->service_timer_period = HZ; 11005 11006 INIT_WORK(&pf->service_task, i40e_service_task); 11007 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 11008 pf->flags |= I40E_FLAG_NEED_LINK_UPDATE; 11009 11010 /* NVM bit on means WoL disabled for the port */ 11011 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 11012 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 11013 pf->wol_en = false; 11014 else 11015 pf->wol_en = true; 11016 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 11017 11018 /* set up the main switch operations */ 11019 i40e_determine_queue_usage(pf); 11020 err = i40e_init_interrupt_scheme(pf); 11021 if (err) 11022 goto err_switch_setup; 11023 11024 /* The number of VSIs reported by the FW is the minimum guaranteed 11025 * to us; HW supports far more and we share the remaining pool with 11026 * the other PFs. We allocate space for more than the guarantee with 11027 * the understanding that we might not get them all later. 11028 */ 11029 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 11030 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 11031 else 11032 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 11033 11034 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 11035 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 11036 GFP_KERNEL); 11037 if (!pf->vsi) { 11038 err = -ENOMEM; 11039 goto err_switch_setup; 11040 } 11041 11042 #ifdef CONFIG_PCI_IOV 11043 /* prep for VF support */ 11044 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11045 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11046 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11047 if (pci_num_vf(pdev)) 11048 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11049 } 11050 #endif 11051 err = i40e_setup_pf_switch(pf, false); 11052 if (err) { 11053 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 11054 goto err_vsis; 11055 } 11056 11057 /* Make sure flow control is set according to current settings */ 11058 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 11059 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 11060 dev_dbg(&pf->pdev->dev, 11061 "Set fc with err %s aq_err %s on get_phy_cap\n", 11062 i40e_stat_str(hw, err), 11063 i40e_aq_str(hw, hw->aq.asq_last_status)); 11064 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 11065 dev_dbg(&pf->pdev->dev, 11066 "Set fc with err %s aq_err %s on set_phy_config\n", 11067 i40e_stat_str(hw, err), 11068 i40e_aq_str(hw, hw->aq.asq_last_status)); 11069 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 11070 dev_dbg(&pf->pdev->dev, 11071 "Set fc with err %s aq_err %s on get_link_info\n", 11072 i40e_stat_str(hw, err), 11073 i40e_aq_str(hw, hw->aq.asq_last_status)); 11074 11075 /* if FDIR VSI was set up, start it now */ 11076 for (i = 0; i < pf->num_alloc_vsi; i++) { 11077 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 11078 i40e_vsi_open(pf->vsi[i]); 11079 break; 11080 } 11081 } 11082 11083 /* The driver only wants link up/down and module qualification 11084 * reports from firmware. Note the negative logic. 11085 */ 11086 err = i40e_aq_set_phy_int_mask(&pf->hw, 11087 ~(I40E_AQ_EVENT_LINK_UPDOWN | 11088 I40E_AQ_EVENT_MEDIA_NA | 11089 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 11090 if (err) 11091 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 11092 i40e_stat_str(&pf->hw, err), 11093 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11094 11095 /* Reconfigure hardware for allowing smaller MSS in the case 11096 * of TSO, so that we avoid the MDD being fired and causing 11097 * a reset in the case of small MSS+TSO. 11098 */ 11099 val = rd32(hw, I40E_REG_MSS); 11100 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11101 val &= ~I40E_REG_MSS_MIN_MASK; 11102 val |= I40E_64BYTE_MSS; 11103 wr32(hw, I40E_REG_MSS, val); 11104 } 11105 11106 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 11107 msleep(75); 11108 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11109 if (err) 11110 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 11111 i40e_stat_str(&pf->hw, err), 11112 i40e_aq_str(&pf->hw, 11113 pf->hw.aq.asq_last_status)); 11114 } 11115 /* The main driver is (mostly) up and happy. We need to set this state 11116 * before setting up the misc vector or we get a race and the vector 11117 * ends up disabled forever. 11118 */ 11119 clear_bit(__I40E_DOWN, &pf->state); 11120 11121 /* In case of MSIX we are going to setup the misc vector right here 11122 * to handle admin queue events etc. In case of legacy and MSI 11123 * the misc functionality and queue processing is combined in 11124 * the same vector and that gets setup at open. 11125 */ 11126 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11127 err = i40e_setup_misc_vector(pf); 11128 if (err) { 11129 dev_info(&pdev->dev, 11130 "setup of misc vector failed: %d\n", err); 11131 goto err_vsis; 11132 } 11133 } 11134 11135 #ifdef CONFIG_PCI_IOV 11136 /* prep for VF support */ 11137 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11138 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11139 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11140 /* disable link interrupts for VFs */ 11141 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 11142 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 11143 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 11144 i40e_flush(hw); 11145 11146 if (pci_num_vf(pdev)) { 11147 dev_info(&pdev->dev, 11148 "Active VFs found, allocating resources.\n"); 11149 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 11150 if (err) 11151 dev_info(&pdev->dev, 11152 "Error %d allocating resources for existing VFs\n", 11153 err); 11154 } 11155 } 11156 #endif /* CONFIG_PCI_IOV */ 11157 11158 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11159 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 11160 pf->num_iwarp_msix, 11161 I40E_IWARP_IRQ_PILE_ID); 11162 if (pf->iwarp_base_vector < 0) { 11163 dev_info(&pdev->dev, 11164 "failed to get tracking for %d vectors for IWARP err=%d\n", 11165 pf->num_iwarp_msix, pf->iwarp_base_vector); 11166 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11167 } 11168 } 11169 11170 i40e_dbg_pf_init(pf); 11171 11172 /* tell the firmware that we're starting */ 11173 i40e_send_version(pf); 11174 11175 /* since everything's happy, start the service_task timer */ 11176 mod_timer(&pf->service_timer, 11177 round_jiffies(jiffies + pf->service_timer_period)); 11178 11179 /* add this PF to client device list and launch a client service task */ 11180 err = i40e_lan_add_device(pf); 11181 if (err) 11182 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 11183 err); 11184 11185 #ifdef I40E_FCOE 11186 /* create FCoE interface */ 11187 i40e_fcoe_vsi_setup(pf); 11188 11189 #endif 11190 #define PCI_SPEED_SIZE 8 11191 #define PCI_WIDTH_SIZE 8 11192 /* Devices on the IOSF bus do not have this information 11193 * and will report PCI Gen 1 x 1 by default so don't bother 11194 * checking them. 11195 */ 11196 if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) { 11197 char speed[PCI_SPEED_SIZE] = "Unknown"; 11198 char width[PCI_WIDTH_SIZE] = "Unknown"; 11199 11200 /* Get the negotiated link width and speed from PCI config 11201 * space 11202 */ 11203 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 11204 &link_status); 11205 11206 i40e_set_pci_config_data(hw, link_status); 11207 11208 switch (hw->bus.speed) { 11209 case i40e_bus_speed_8000: 11210 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 11211 case i40e_bus_speed_5000: 11212 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 11213 case i40e_bus_speed_2500: 11214 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 11215 default: 11216 break; 11217 } 11218 switch (hw->bus.width) { 11219 case i40e_bus_width_pcie_x8: 11220 strncpy(width, "8", PCI_WIDTH_SIZE); break; 11221 case i40e_bus_width_pcie_x4: 11222 strncpy(width, "4", PCI_WIDTH_SIZE); break; 11223 case i40e_bus_width_pcie_x2: 11224 strncpy(width, "2", PCI_WIDTH_SIZE); break; 11225 case i40e_bus_width_pcie_x1: 11226 strncpy(width, "1", PCI_WIDTH_SIZE); break; 11227 default: 11228 break; 11229 } 11230 11231 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 11232 speed, width); 11233 11234 if (hw->bus.width < i40e_bus_width_pcie_x8 || 11235 hw->bus.speed < i40e_bus_speed_8000) { 11236 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 11237 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 11238 } 11239 } 11240 11241 /* get the requested speeds from the fw */ 11242 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 11243 if (err) 11244 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 11245 i40e_stat_str(&pf->hw, err), 11246 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11247 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 11248 11249 /* get the supported phy types from the fw */ 11250 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 11251 if (err) 11252 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 11253 i40e_stat_str(&pf->hw, err), 11254 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11255 pf->hw.phy.phy_types = le32_to_cpu(abilities.phy_type); 11256 11257 /* Add a filter to drop all Flow control frames from any VSI from being 11258 * transmitted. By doing so we stop a malicious VF from sending out 11259 * PAUSE or PFC frames and potentially controlling traffic for other 11260 * PF/VF VSIs. 11261 * The FW can still send Flow control frames if enabled. 11262 */ 11263 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11264 pf->main_vsi_seid); 11265 11266 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 11267 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 11268 pf->flags |= I40E_FLAG_HAVE_10GBASET_PHY; 11269 11270 /* print a string summarizing features */ 11271 i40e_print_features(pf); 11272 11273 return 0; 11274 11275 /* Unwind what we've done if something failed in the setup */ 11276 err_vsis: 11277 set_bit(__I40E_DOWN, &pf->state); 11278 i40e_clear_interrupt_scheme(pf); 11279 kfree(pf->vsi); 11280 err_switch_setup: 11281 i40e_reset_interrupt_capability(pf); 11282 del_timer_sync(&pf->service_timer); 11283 err_mac_addr: 11284 err_configure_lan_hmc: 11285 (void)i40e_shutdown_lan_hmc(hw); 11286 err_init_lan_hmc: 11287 kfree(pf->qp_pile); 11288 err_sw_init: 11289 err_adminq_setup: 11290 err_pf_reset: 11291 iounmap(hw->hw_addr); 11292 err_ioremap: 11293 kfree(pf); 11294 err_pf_alloc: 11295 pci_disable_pcie_error_reporting(pdev); 11296 pci_release_mem_regions(pdev); 11297 err_pci_reg: 11298 err_dma: 11299 pci_disable_device(pdev); 11300 return err; 11301 } 11302 11303 /** 11304 * i40e_remove - Device removal routine 11305 * @pdev: PCI device information struct 11306 * 11307 * i40e_remove is called by the PCI subsystem to alert the driver 11308 * that is should release a PCI device. This could be caused by a 11309 * Hot-Plug event, or because the driver is going to be removed from 11310 * memory. 11311 **/ 11312 static void i40e_remove(struct pci_dev *pdev) 11313 { 11314 struct i40e_pf *pf = pci_get_drvdata(pdev); 11315 struct i40e_hw *hw = &pf->hw; 11316 i40e_status ret_code; 11317 int i; 11318 11319 i40e_dbg_pf_exit(pf); 11320 11321 i40e_ptp_stop(pf); 11322 11323 /* Disable RSS in hw */ 11324 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 11325 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 11326 11327 /* no more scheduling of any task */ 11328 set_bit(__I40E_SUSPENDED, &pf->state); 11329 set_bit(__I40E_DOWN, &pf->state); 11330 if (pf->service_timer.data) 11331 del_timer_sync(&pf->service_timer); 11332 if (pf->service_task.func) 11333 cancel_work_sync(&pf->service_task); 11334 11335 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 11336 i40e_free_vfs(pf); 11337 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 11338 } 11339 11340 i40e_fdir_teardown(pf); 11341 11342 /* If there is a switch structure or any orphans, remove them. 11343 * This will leave only the PF's VSI remaining. 11344 */ 11345 for (i = 0; i < I40E_MAX_VEB; i++) { 11346 if (!pf->veb[i]) 11347 continue; 11348 11349 if (pf->veb[i]->uplink_seid == pf->mac_seid || 11350 pf->veb[i]->uplink_seid == 0) 11351 i40e_switch_branch_release(pf->veb[i]); 11352 } 11353 11354 /* Now we can shutdown the PF's VSI, just before we kill 11355 * adminq and hmc. 11356 */ 11357 if (pf->vsi[pf->lan_vsi]) 11358 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 11359 11360 /* remove attached clients */ 11361 ret_code = i40e_lan_del_device(pf); 11362 if (ret_code) { 11363 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 11364 ret_code); 11365 } 11366 11367 /* shutdown and destroy the HMC */ 11368 if (hw->hmc.hmc_obj) { 11369 ret_code = i40e_shutdown_lan_hmc(hw); 11370 if (ret_code) 11371 dev_warn(&pdev->dev, 11372 "Failed to destroy the HMC resources: %d\n", 11373 ret_code); 11374 } 11375 11376 /* shutdown the adminq */ 11377 i40e_shutdown_adminq(hw); 11378 11379 /* destroy the locks only once, here */ 11380 mutex_destroy(&hw->aq.arq_mutex); 11381 mutex_destroy(&hw->aq.asq_mutex); 11382 11383 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 11384 i40e_clear_interrupt_scheme(pf); 11385 for (i = 0; i < pf->num_alloc_vsi; i++) { 11386 if (pf->vsi[i]) { 11387 i40e_vsi_clear_rings(pf->vsi[i]); 11388 i40e_vsi_clear(pf->vsi[i]); 11389 pf->vsi[i] = NULL; 11390 } 11391 } 11392 11393 for (i = 0; i < I40E_MAX_VEB; i++) { 11394 kfree(pf->veb[i]); 11395 pf->veb[i] = NULL; 11396 } 11397 11398 kfree(pf->qp_pile); 11399 kfree(pf->vsi); 11400 11401 iounmap(hw->hw_addr); 11402 kfree(pf); 11403 pci_release_mem_regions(pdev); 11404 11405 pci_disable_pcie_error_reporting(pdev); 11406 pci_disable_device(pdev); 11407 } 11408 11409 /** 11410 * i40e_pci_error_detected - warning that something funky happened in PCI land 11411 * @pdev: PCI device information struct 11412 * 11413 * Called to warn that something happened and the error handling steps 11414 * are in progress. Allows the driver to quiesce things, be ready for 11415 * remediation. 11416 **/ 11417 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 11418 enum pci_channel_state error) 11419 { 11420 struct i40e_pf *pf = pci_get_drvdata(pdev); 11421 11422 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 11423 11424 if (!pf) { 11425 dev_info(&pdev->dev, 11426 "Cannot recover - error happened during device probe\n"); 11427 return PCI_ERS_RESULT_DISCONNECT; 11428 } 11429 11430 /* shutdown all operations */ 11431 if (!test_bit(__I40E_SUSPENDED, &pf->state)) { 11432 rtnl_lock(); 11433 i40e_prep_for_reset(pf); 11434 rtnl_unlock(); 11435 } 11436 11437 /* Request a slot reset */ 11438 return PCI_ERS_RESULT_NEED_RESET; 11439 } 11440 11441 /** 11442 * i40e_pci_error_slot_reset - a PCI slot reset just happened 11443 * @pdev: PCI device information struct 11444 * 11445 * Called to find if the driver can work with the device now that 11446 * the pci slot has been reset. If a basic connection seems good 11447 * (registers are readable and have sane content) then return a 11448 * happy little PCI_ERS_RESULT_xxx. 11449 **/ 11450 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 11451 { 11452 struct i40e_pf *pf = pci_get_drvdata(pdev); 11453 pci_ers_result_t result; 11454 int err; 11455 u32 reg; 11456 11457 dev_dbg(&pdev->dev, "%s\n", __func__); 11458 if (pci_enable_device_mem(pdev)) { 11459 dev_info(&pdev->dev, 11460 "Cannot re-enable PCI device after reset.\n"); 11461 result = PCI_ERS_RESULT_DISCONNECT; 11462 } else { 11463 pci_set_master(pdev); 11464 pci_restore_state(pdev); 11465 pci_save_state(pdev); 11466 pci_wake_from_d3(pdev, false); 11467 11468 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 11469 if (reg == 0) 11470 result = PCI_ERS_RESULT_RECOVERED; 11471 else 11472 result = PCI_ERS_RESULT_DISCONNECT; 11473 } 11474 11475 err = pci_cleanup_aer_uncorrect_error_status(pdev); 11476 if (err) { 11477 dev_info(&pdev->dev, 11478 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 11479 err); 11480 /* non-fatal, continue */ 11481 } 11482 11483 return result; 11484 } 11485 11486 /** 11487 * i40e_pci_error_resume - restart operations after PCI error recovery 11488 * @pdev: PCI device information struct 11489 * 11490 * Called to allow the driver to bring things back up after PCI error 11491 * and/or reset recovery has finished. 11492 **/ 11493 static void i40e_pci_error_resume(struct pci_dev *pdev) 11494 { 11495 struct i40e_pf *pf = pci_get_drvdata(pdev); 11496 11497 dev_dbg(&pdev->dev, "%s\n", __func__); 11498 if (test_bit(__I40E_SUSPENDED, &pf->state)) 11499 return; 11500 11501 rtnl_lock(); 11502 i40e_handle_reset_warning(pf); 11503 rtnl_unlock(); 11504 } 11505 11506 /** 11507 * i40e_shutdown - PCI callback for shutting down 11508 * @pdev: PCI device information struct 11509 **/ 11510 static void i40e_shutdown(struct pci_dev *pdev) 11511 { 11512 struct i40e_pf *pf = pci_get_drvdata(pdev); 11513 struct i40e_hw *hw = &pf->hw; 11514 11515 set_bit(__I40E_SUSPENDED, &pf->state); 11516 set_bit(__I40E_DOWN, &pf->state); 11517 rtnl_lock(); 11518 i40e_prep_for_reset(pf); 11519 rtnl_unlock(); 11520 11521 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11522 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11523 11524 del_timer_sync(&pf->service_timer); 11525 cancel_work_sync(&pf->service_task); 11526 i40e_fdir_teardown(pf); 11527 11528 rtnl_lock(); 11529 i40e_prep_for_reset(pf); 11530 rtnl_unlock(); 11531 11532 wr32(hw, I40E_PFPM_APM, 11533 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11534 wr32(hw, I40E_PFPM_WUFC, 11535 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11536 11537 i40e_clear_interrupt_scheme(pf); 11538 11539 if (system_state == SYSTEM_POWER_OFF) { 11540 pci_wake_from_d3(pdev, pf->wol_en); 11541 pci_set_power_state(pdev, PCI_D3hot); 11542 } 11543 } 11544 11545 #ifdef CONFIG_PM 11546 /** 11547 * i40e_suspend - PCI callback for moving to D3 11548 * @pdev: PCI device information struct 11549 **/ 11550 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state) 11551 { 11552 struct i40e_pf *pf = pci_get_drvdata(pdev); 11553 struct i40e_hw *hw = &pf->hw; 11554 int retval = 0; 11555 11556 set_bit(__I40E_SUSPENDED, &pf->state); 11557 set_bit(__I40E_DOWN, &pf->state); 11558 11559 rtnl_lock(); 11560 i40e_prep_for_reset(pf); 11561 rtnl_unlock(); 11562 11563 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11564 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11565 11566 i40e_stop_misc_vector(pf); 11567 11568 retval = pci_save_state(pdev); 11569 if (retval) 11570 return retval; 11571 11572 pci_wake_from_d3(pdev, pf->wol_en); 11573 pci_set_power_state(pdev, PCI_D3hot); 11574 11575 return retval; 11576 } 11577 11578 /** 11579 * i40e_resume - PCI callback for waking up from D3 11580 * @pdev: PCI device information struct 11581 **/ 11582 static int i40e_resume(struct pci_dev *pdev) 11583 { 11584 struct i40e_pf *pf = pci_get_drvdata(pdev); 11585 u32 err; 11586 11587 pci_set_power_state(pdev, PCI_D0); 11588 pci_restore_state(pdev); 11589 /* pci_restore_state() clears dev->state_saves, so 11590 * call pci_save_state() again to restore it. 11591 */ 11592 pci_save_state(pdev); 11593 11594 err = pci_enable_device_mem(pdev); 11595 if (err) { 11596 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 11597 return err; 11598 } 11599 pci_set_master(pdev); 11600 11601 /* no wakeup events while running */ 11602 pci_wake_from_d3(pdev, false); 11603 11604 /* handling the reset will rebuild the device state */ 11605 if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) { 11606 clear_bit(__I40E_DOWN, &pf->state); 11607 rtnl_lock(); 11608 i40e_reset_and_rebuild(pf, false); 11609 rtnl_unlock(); 11610 } 11611 11612 return 0; 11613 } 11614 11615 #endif 11616 static const struct pci_error_handlers i40e_err_handler = { 11617 .error_detected = i40e_pci_error_detected, 11618 .slot_reset = i40e_pci_error_slot_reset, 11619 .resume = i40e_pci_error_resume, 11620 }; 11621 11622 static struct pci_driver i40e_driver = { 11623 .name = i40e_driver_name, 11624 .id_table = i40e_pci_tbl, 11625 .probe = i40e_probe, 11626 .remove = i40e_remove, 11627 #ifdef CONFIG_PM 11628 .suspend = i40e_suspend, 11629 .resume = i40e_resume, 11630 #endif 11631 .shutdown = i40e_shutdown, 11632 .err_handler = &i40e_err_handler, 11633 .sriov_configure = i40e_pci_sriov_configure, 11634 }; 11635 11636 /** 11637 * i40e_init_module - Driver registration routine 11638 * 11639 * i40e_init_module is the first routine called when the driver is 11640 * loaded. All it does is register with the PCI subsystem. 11641 **/ 11642 static int __init i40e_init_module(void) 11643 { 11644 pr_info("%s: %s - version %s\n", i40e_driver_name, 11645 i40e_driver_string, i40e_driver_version_str); 11646 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 11647 11648 /* we will see if single thread per module is enough for now, 11649 * it can't be any worse than using the system workqueue which 11650 * was already single threaded 11651 */ 11652 i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 11653 i40e_driver_name); 11654 if (!i40e_wq) { 11655 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 11656 return -ENOMEM; 11657 } 11658 11659 i40e_dbg_init(); 11660 return pci_register_driver(&i40e_driver); 11661 } 11662 module_init(i40e_init_module); 11663 11664 /** 11665 * i40e_exit_module - Driver exit cleanup routine 11666 * 11667 * i40e_exit_module is called just before the driver is removed 11668 * from memory. 11669 **/ 11670 static void __exit i40e_exit_module(void) 11671 { 11672 pci_unregister_driver(&i40e_driver); 11673 destroy_workqueue(i40e_wq); 11674 i40e_dbg_exit(); 11675 } 11676 module_exit(i40e_exit_module); 11677