1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <generated/utsrelease.h> 9 #include "ice.h" 10 #include "ice_base.h" 11 #include "ice_lib.h" 12 #include "ice_fltr.h" 13 #include "ice_dcb_lib.h" 14 #include "ice_dcb_nl.h" 15 #include "ice_devlink.h" 16 17 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 18 static const char ice_driver_string[] = DRV_SUMMARY; 19 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 20 21 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 22 #define ICE_DDP_PKG_PATH "intel/ice/ddp/" 23 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 24 25 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 26 MODULE_DESCRIPTION(DRV_SUMMARY); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 29 30 static int debug = -1; 31 module_param(debug, int, 0644); 32 #ifndef CONFIG_DYNAMIC_DEBUG 33 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 34 #else 35 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 36 #endif /* !CONFIG_DYNAMIC_DEBUG */ 37 38 static struct workqueue_struct *ice_wq; 39 static const struct net_device_ops ice_netdev_safe_mode_ops; 40 static const struct net_device_ops ice_netdev_ops; 41 static int ice_vsi_open(struct ice_vsi *vsi); 42 43 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 44 45 static void ice_vsi_release_all(struct ice_pf *pf); 46 47 /** 48 * ice_get_tx_pending - returns number of Tx descriptors not processed 49 * @ring: the ring of descriptors 50 */ 51 static u16 ice_get_tx_pending(struct ice_ring *ring) 52 { 53 u16 head, tail; 54 55 head = ring->next_to_clean; 56 tail = ring->next_to_use; 57 58 if (head != tail) 59 return (head < tail) ? 60 tail - head : (tail + ring->count - head); 61 return 0; 62 } 63 64 /** 65 * ice_check_for_hang_subtask - check for and recover hung queues 66 * @pf: pointer to PF struct 67 */ 68 static void ice_check_for_hang_subtask(struct ice_pf *pf) 69 { 70 struct ice_vsi *vsi = NULL; 71 struct ice_hw *hw; 72 unsigned int i; 73 int packets; 74 u32 v; 75 76 ice_for_each_vsi(pf, v) 77 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 78 vsi = pf->vsi[v]; 79 break; 80 } 81 82 if (!vsi || test_bit(__ICE_DOWN, vsi->state)) 83 return; 84 85 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 86 return; 87 88 hw = &vsi->back->hw; 89 90 for (i = 0; i < vsi->num_txq; i++) { 91 struct ice_ring *tx_ring = vsi->tx_rings[i]; 92 93 if (tx_ring && tx_ring->desc) { 94 /* If packet counter has not changed the queue is 95 * likely stalled, so force an interrupt for this 96 * queue. 97 * 98 * prev_pkt would be negative if there was no 99 * pending work. 100 */ 101 packets = tx_ring->stats.pkts & INT_MAX; 102 if (tx_ring->tx_stats.prev_pkt == packets) { 103 /* Trigger sw interrupt to revive the queue */ 104 ice_trigger_sw_intr(hw, tx_ring->q_vector); 105 continue; 106 } 107 108 /* Memory barrier between read of packet count and call 109 * to ice_get_tx_pending() 110 */ 111 smp_rmb(); 112 tx_ring->tx_stats.prev_pkt = 113 ice_get_tx_pending(tx_ring) ? packets : -1; 114 } 115 } 116 } 117 118 /** 119 * ice_init_mac_fltr - Set initial MAC filters 120 * @pf: board private structure 121 * 122 * Set initial set of MAC filters for PF VSI; configure filters for permanent 123 * address and broadcast address. If an error is encountered, netdevice will be 124 * unregistered. 125 */ 126 static int ice_init_mac_fltr(struct ice_pf *pf) 127 { 128 enum ice_status status; 129 struct ice_vsi *vsi; 130 u8 *perm_addr; 131 132 vsi = ice_get_main_vsi(pf); 133 if (!vsi) 134 return -EINVAL; 135 136 perm_addr = vsi->port_info->mac.perm_addr; 137 status = ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 138 if (!status) 139 return 0; 140 141 /* We aren't useful with no MAC filters, so unregister if we 142 * had an error 143 */ 144 if (vsi->netdev->reg_state == NETREG_REGISTERED) { 145 dev_err(ice_pf_to_dev(pf), "Could not add MAC filters error %s. Unregistering device\n", 146 ice_stat_str(status)); 147 unregister_netdev(vsi->netdev); 148 free_netdev(vsi->netdev); 149 vsi->netdev = NULL; 150 } 151 152 return -EIO; 153 } 154 155 /** 156 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 157 * @netdev: the net device on which the sync is happening 158 * @addr: MAC address to sync 159 * 160 * This is a callback function which is called by the in kernel device sync 161 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 162 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 163 * MAC filters from the hardware. 164 */ 165 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 166 { 167 struct ice_netdev_priv *np = netdev_priv(netdev); 168 struct ice_vsi *vsi = np->vsi; 169 170 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 171 ICE_FWD_TO_VSI)) 172 return -EINVAL; 173 174 return 0; 175 } 176 177 /** 178 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 179 * @netdev: the net device on which the unsync is happening 180 * @addr: MAC address to unsync 181 * 182 * This is a callback function which is called by the in kernel device unsync 183 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 184 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 185 * delete the MAC filters from the hardware. 186 */ 187 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 188 { 189 struct ice_netdev_priv *np = netdev_priv(netdev); 190 struct ice_vsi *vsi = np->vsi; 191 192 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 193 ICE_FWD_TO_VSI)) 194 return -EINVAL; 195 196 return 0; 197 } 198 199 /** 200 * ice_vsi_fltr_changed - check if filter state changed 201 * @vsi: VSI to be checked 202 * 203 * returns true if filter state has changed, false otherwise. 204 */ 205 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 206 { 207 return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) || 208 test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) || 209 test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 210 } 211 212 /** 213 * ice_cfg_promisc - Enable or disable promiscuous mode for a given PF 214 * @vsi: the VSI being configured 215 * @promisc_m: mask of promiscuous config bits 216 * @set_promisc: enable or disable promisc flag request 217 * 218 */ 219 static int ice_cfg_promisc(struct ice_vsi *vsi, u8 promisc_m, bool set_promisc) 220 { 221 struct ice_hw *hw = &vsi->back->hw; 222 enum ice_status status = 0; 223 224 if (vsi->type != ICE_VSI_PF) 225 return 0; 226 227 if (vsi->vlan_ena) { 228 status = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_m, 229 set_promisc); 230 } else { 231 if (set_promisc) 232 status = ice_set_vsi_promisc(hw, vsi->idx, promisc_m, 233 0); 234 else 235 status = ice_clear_vsi_promisc(hw, vsi->idx, promisc_m, 236 0); 237 } 238 239 if (status) 240 return -EIO; 241 242 return 0; 243 } 244 245 /** 246 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 247 * @vsi: ptr to the VSI 248 * 249 * Push any outstanding VSI filter changes through the AdminQ. 250 */ 251 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 252 { 253 struct device *dev = ice_pf_to_dev(vsi->back); 254 struct net_device *netdev = vsi->netdev; 255 bool promisc_forced_on = false; 256 struct ice_pf *pf = vsi->back; 257 struct ice_hw *hw = &pf->hw; 258 enum ice_status status = 0; 259 u32 changed_flags = 0; 260 u8 promisc_m; 261 int err = 0; 262 263 if (!vsi->netdev) 264 return -EINVAL; 265 266 while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) 267 usleep_range(1000, 2000); 268 269 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 270 vsi->current_netdev_flags = vsi->netdev->flags; 271 272 INIT_LIST_HEAD(&vsi->tmp_sync_list); 273 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 274 275 if (ice_vsi_fltr_changed(vsi)) { 276 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 277 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 278 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 279 280 /* grab the netdev's addr_list_lock */ 281 netif_addr_lock_bh(netdev); 282 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 283 ice_add_mac_to_unsync_list); 284 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 285 ice_add_mac_to_unsync_list); 286 /* our temp lists are populated. release lock */ 287 netif_addr_unlock_bh(netdev); 288 } 289 290 /* Remove MAC addresses in the unsync list */ 291 status = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 292 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 293 if (status) { 294 netdev_err(netdev, "Failed to delete MAC filters\n"); 295 /* if we failed because of alloc failures, just bail */ 296 if (status == ICE_ERR_NO_MEMORY) { 297 err = -ENOMEM; 298 goto out; 299 } 300 } 301 302 /* Add MAC addresses in the sync list */ 303 status = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 304 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 305 /* If filter is added successfully or already exists, do not go into 306 * 'if' condition and report it as error. Instead continue processing 307 * rest of the function. 308 */ 309 if (status && status != ICE_ERR_ALREADY_EXISTS) { 310 netdev_err(netdev, "Failed to add MAC filters\n"); 311 /* If there is no more space for new umac filters, VSI 312 * should go into promiscuous mode. There should be some 313 * space reserved for promiscuous filters. 314 */ 315 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 316 !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC, 317 vsi->state)) { 318 promisc_forced_on = true; 319 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 320 vsi->vsi_num); 321 } else { 322 err = -EIO; 323 goto out; 324 } 325 } 326 /* check for changes in promiscuous modes */ 327 if (changed_flags & IFF_ALLMULTI) { 328 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 329 if (vsi->vlan_ena) 330 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 331 else 332 promisc_m = ICE_MCAST_PROMISC_BITS; 333 334 err = ice_cfg_promisc(vsi, promisc_m, true); 335 if (err) { 336 netdev_err(netdev, "Error setting Multicast promiscuous mode on VSI %i\n", 337 vsi->vsi_num); 338 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 339 goto out_promisc; 340 } 341 } else { 342 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 343 if (vsi->vlan_ena) 344 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 345 else 346 promisc_m = ICE_MCAST_PROMISC_BITS; 347 348 err = ice_cfg_promisc(vsi, promisc_m, false); 349 if (err) { 350 netdev_err(netdev, "Error clearing Multicast promiscuous mode on VSI %i\n", 351 vsi->vsi_num); 352 vsi->current_netdev_flags |= IFF_ALLMULTI; 353 goto out_promisc; 354 } 355 } 356 } 357 358 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 359 test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) { 360 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 361 if (vsi->current_netdev_flags & IFF_PROMISC) { 362 /* Apply Rx filter rule to get traffic from wire */ 363 if (!ice_is_dflt_vsi_in_use(pf->first_sw)) { 364 err = ice_set_dflt_vsi(pf->first_sw, vsi); 365 if (err && err != -EEXIST) { 366 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 367 err, vsi->vsi_num); 368 vsi->current_netdev_flags &= 369 ~IFF_PROMISC; 370 goto out_promisc; 371 } 372 } 373 } else { 374 /* Clear Rx filter to remove traffic from wire */ 375 if (ice_is_vsi_dflt_vsi(pf->first_sw, vsi)) { 376 err = ice_clear_dflt_vsi(pf->first_sw); 377 if (err) { 378 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 379 err, vsi->vsi_num); 380 vsi->current_netdev_flags |= 381 IFF_PROMISC; 382 goto out_promisc; 383 } 384 } 385 } 386 } 387 goto exit; 388 389 out_promisc: 390 set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 391 goto exit; 392 out: 393 /* if something went wrong then set the changed flag so we try again */ 394 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 395 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 396 exit: 397 clear_bit(__ICE_CFG_BUSY, vsi->state); 398 return err; 399 } 400 401 /** 402 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 403 * @pf: board private structure 404 */ 405 static void ice_sync_fltr_subtask(struct ice_pf *pf) 406 { 407 int v; 408 409 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 410 return; 411 412 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 413 414 ice_for_each_vsi(pf, v) 415 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 416 ice_vsi_sync_fltr(pf->vsi[v])) { 417 /* come back and try again later */ 418 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 419 break; 420 } 421 } 422 423 /** 424 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 425 * @pf: the PF 426 * @locked: is the rtnl_lock already held 427 */ 428 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 429 { 430 int v; 431 432 ice_for_each_vsi(pf, v) 433 if (pf->vsi[v]) 434 ice_dis_vsi(pf->vsi[v], locked); 435 } 436 437 /** 438 * ice_prepare_for_reset - prep for the core to reset 439 * @pf: board private structure 440 * 441 * Inform or close all dependent features in prep for reset. 442 */ 443 static void 444 ice_prepare_for_reset(struct ice_pf *pf) 445 { 446 struct ice_hw *hw = &pf->hw; 447 unsigned int i; 448 449 /* already prepared for reset */ 450 if (test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) 451 return; 452 453 /* Notify VFs of impending reset */ 454 if (ice_check_sq_alive(hw, &hw->mailboxq)) 455 ice_vc_notify_reset(pf); 456 457 /* Disable VFs until reset is completed */ 458 ice_for_each_vf(pf, i) 459 ice_set_vf_state_qs_dis(&pf->vf[i]); 460 461 /* clear SW filtering DB */ 462 ice_clear_hw_tbls(hw); 463 /* disable the VSIs and their queues that are not already DOWN */ 464 ice_pf_dis_all_vsi(pf, false); 465 466 if (hw->port_info) 467 ice_sched_clear_port(hw->port_info); 468 469 ice_shutdown_all_ctrlq(hw); 470 471 set_bit(__ICE_PREPARED_FOR_RESET, pf->state); 472 } 473 474 /** 475 * ice_do_reset - Initiate one of many types of resets 476 * @pf: board private structure 477 * @reset_type: reset type requested 478 * before this function was called. 479 */ 480 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 481 { 482 struct device *dev = ice_pf_to_dev(pf); 483 struct ice_hw *hw = &pf->hw; 484 485 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 486 WARN_ON(in_interrupt()); 487 488 ice_prepare_for_reset(pf); 489 490 /* trigger the reset */ 491 if (ice_reset(hw, reset_type)) { 492 dev_err(dev, "reset %d failed\n", reset_type); 493 set_bit(__ICE_RESET_FAILED, pf->state); 494 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 495 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 496 clear_bit(__ICE_PFR_REQ, pf->state); 497 clear_bit(__ICE_CORER_REQ, pf->state); 498 clear_bit(__ICE_GLOBR_REQ, pf->state); 499 return; 500 } 501 502 /* PFR is a bit of a special case because it doesn't result in an OICR 503 * interrupt. So for PFR, rebuild after the reset and clear the reset- 504 * associated state bits. 505 */ 506 if (reset_type == ICE_RESET_PFR) { 507 pf->pfr_count++; 508 ice_rebuild(pf, reset_type); 509 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 510 clear_bit(__ICE_PFR_REQ, pf->state); 511 ice_reset_all_vfs(pf, true); 512 } 513 } 514 515 /** 516 * ice_reset_subtask - Set up for resetting the device and driver 517 * @pf: board private structure 518 */ 519 static void ice_reset_subtask(struct ice_pf *pf) 520 { 521 enum ice_reset_req reset_type = ICE_RESET_INVAL; 522 523 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 524 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 525 * of reset is pending and sets bits in pf->state indicating the reset 526 * type and __ICE_RESET_OICR_RECV. So, if the latter bit is set 527 * prepare for pending reset if not already (for PF software-initiated 528 * global resets the software should already be prepared for it as 529 * indicated by __ICE_PREPARED_FOR_RESET; for global resets initiated 530 * by firmware or software on other PFs, that bit is not set so prepare 531 * for the reset now), poll for reset done, rebuild and return. 532 */ 533 if (test_bit(__ICE_RESET_OICR_RECV, pf->state)) { 534 /* Perform the largest reset requested */ 535 if (test_and_clear_bit(__ICE_CORER_RECV, pf->state)) 536 reset_type = ICE_RESET_CORER; 537 if (test_and_clear_bit(__ICE_GLOBR_RECV, pf->state)) 538 reset_type = ICE_RESET_GLOBR; 539 if (test_and_clear_bit(__ICE_EMPR_RECV, pf->state)) 540 reset_type = ICE_RESET_EMPR; 541 /* return if no valid reset type requested */ 542 if (reset_type == ICE_RESET_INVAL) 543 return; 544 ice_prepare_for_reset(pf); 545 546 /* make sure we are ready to rebuild */ 547 if (ice_check_reset(&pf->hw)) { 548 set_bit(__ICE_RESET_FAILED, pf->state); 549 } else { 550 /* done with reset. start rebuild */ 551 pf->hw.reset_ongoing = false; 552 ice_rebuild(pf, reset_type); 553 /* clear bit to resume normal operations, but 554 * ICE_NEEDS_RESTART bit is set in case rebuild failed 555 */ 556 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 557 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 558 clear_bit(__ICE_PFR_REQ, pf->state); 559 clear_bit(__ICE_CORER_REQ, pf->state); 560 clear_bit(__ICE_GLOBR_REQ, pf->state); 561 ice_reset_all_vfs(pf, true); 562 } 563 564 return; 565 } 566 567 /* No pending resets to finish processing. Check for new resets */ 568 if (test_bit(__ICE_PFR_REQ, pf->state)) 569 reset_type = ICE_RESET_PFR; 570 if (test_bit(__ICE_CORER_REQ, pf->state)) 571 reset_type = ICE_RESET_CORER; 572 if (test_bit(__ICE_GLOBR_REQ, pf->state)) 573 reset_type = ICE_RESET_GLOBR; 574 /* If no valid reset type requested just return */ 575 if (reset_type == ICE_RESET_INVAL) 576 return; 577 578 /* reset if not already down or busy */ 579 if (!test_bit(__ICE_DOWN, pf->state) && 580 !test_bit(__ICE_CFG_BUSY, pf->state)) { 581 ice_do_reset(pf, reset_type); 582 } 583 } 584 585 /** 586 * ice_print_topo_conflict - print topology conflict message 587 * @vsi: the VSI whose topology status is being checked 588 */ 589 static void ice_print_topo_conflict(struct ice_vsi *vsi) 590 { 591 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 592 case ICE_AQ_LINK_TOPO_CONFLICT: 593 case ICE_AQ_LINK_MEDIA_CONFLICT: 594 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 595 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 596 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 597 netdev_info(vsi->netdev, "Possible mis-configuration of the Ethernet port detected, please use the Intel(R) Ethernet Port Configuration Tool application to address the issue.\n"); 598 break; 599 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 600 netdev_info(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 601 break; 602 default: 603 break; 604 } 605 } 606 607 /** 608 * ice_print_link_msg - print link up or down message 609 * @vsi: the VSI whose link status is being queried 610 * @isup: boolean for if the link is now up or down 611 */ 612 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 613 { 614 struct ice_aqc_get_phy_caps_data *caps; 615 const char *an_advertised; 616 enum ice_status status; 617 const char *fec_req; 618 const char *speed; 619 const char *fec; 620 const char *fc; 621 const char *an; 622 623 if (!vsi) 624 return; 625 626 if (vsi->current_isup == isup) 627 return; 628 629 vsi->current_isup = isup; 630 631 if (!isup) { 632 netdev_info(vsi->netdev, "NIC Link is Down\n"); 633 return; 634 } 635 636 switch (vsi->port_info->phy.link_info.link_speed) { 637 case ICE_AQ_LINK_SPEED_100GB: 638 speed = "100 G"; 639 break; 640 case ICE_AQ_LINK_SPEED_50GB: 641 speed = "50 G"; 642 break; 643 case ICE_AQ_LINK_SPEED_40GB: 644 speed = "40 G"; 645 break; 646 case ICE_AQ_LINK_SPEED_25GB: 647 speed = "25 G"; 648 break; 649 case ICE_AQ_LINK_SPEED_20GB: 650 speed = "20 G"; 651 break; 652 case ICE_AQ_LINK_SPEED_10GB: 653 speed = "10 G"; 654 break; 655 case ICE_AQ_LINK_SPEED_5GB: 656 speed = "5 G"; 657 break; 658 case ICE_AQ_LINK_SPEED_2500MB: 659 speed = "2.5 G"; 660 break; 661 case ICE_AQ_LINK_SPEED_1000MB: 662 speed = "1 G"; 663 break; 664 case ICE_AQ_LINK_SPEED_100MB: 665 speed = "100 M"; 666 break; 667 default: 668 speed = "Unknown"; 669 break; 670 } 671 672 switch (vsi->port_info->fc.current_mode) { 673 case ICE_FC_FULL: 674 fc = "Rx/Tx"; 675 break; 676 case ICE_FC_TX_PAUSE: 677 fc = "Tx"; 678 break; 679 case ICE_FC_RX_PAUSE: 680 fc = "Rx"; 681 break; 682 case ICE_FC_NONE: 683 fc = "None"; 684 break; 685 default: 686 fc = "Unknown"; 687 break; 688 } 689 690 /* Get FEC mode based on negotiated link info */ 691 switch (vsi->port_info->phy.link_info.fec_info) { 692 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 693 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 694 fec = "RS-FEC"; 695 break; 696 case ICE_AQ_LINK_25G_KR_FEC_EN: 697 fec = "FC-FEC/BASE-R"; 698 break; 699 default: 700 fec = "NONE"; 701 break; 702 } 703 704 /* check if autoneg completed, might be false due to not supported */ 705 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 706 an = "True"; 707 else 708 an = "False"; 709 710 /* Get FEC mode requested based on PHY caps last SW configuration */ 711 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 712 if (!caps) { 713 fec_req = "Unknown"; 714 an_advertised = "Unknown"; 715 goto done; 716 } 717 718 status = ice_aq_get_phy_caps(vsi->port_info, false, 719 ICE_AQC_REPORT_SW_CFG, caps, NULL); 720 if (status) 721 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 722 723 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 724 725 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 726 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 727 fec_req = "RS-FEC"; 728 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 729 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 730 fec_req = "FC-FEC/BASE-R"; 731 else 732 fec_req = "NONE"; 733 734 kfree(caps); 735 736 done: 737 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 738 speed, fec_req, fec, an_advertised, an, fc); 739 ice_print_topo_conflict(vsi); 740 } 741 742 /** 743 * ice_vsi_link_event - update the VSI's netdev 744 * @vsi: the VSI on which the link event occurred 745 * @link_up: whether or not the VSI needs to be set up or down 746 */ 747 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 748 { 749 if (!vsi) 750 return; 751 752 if (test_bit(__ICE_DOWN, vsi->state) || !vsi->netdev) 753 return; 754 755 if (vsi->type == ICE_VSI_PF) { 756 if (link_up == netif_carrier_ok(vsi->netdev)) 757 return; 758 759 if (link_up) { 760 netif_carrier_on(vsi->netdev); 761 netif_tx_wake_all_queues(vsi->netdev); 762 } else { 763 netif_carrier_off(vsi->netdev); 764 netif_tx_stop_all_queues(vsi->netdev); 765 } 766 } 767 } 768 769 /** 770 * ice_link_event - process the link event 771 * @pf: PF that the link event is associated with 772 * @pi: port_info for the port that the link event is associated with 773 * @link_up: true if the physical link is up and false if it is down 774 * @link_speed: current link speed received from the link event 775 * 776 * Returns 0 on success and negative on failure 777 */ 778 static int 779 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 780 u16 link_speed) 781 { 782 struct device *dev = ice_pf_to_dev(pf); 783 struct ice_phy_info *phy_info; 784 struct ice_vsi *vsi; 785 u16 old_link_speed; 786 bool old_link; 787 int result; 788 789 phy_info = &pi->phy; 790 phy_info->link_info_old = phy_info->link_info; 791 792 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 793 old_link_speed = phy_info->link_info_old.link_speed; 794 795 /* update the link info structures and re-enable link events, 796 * don't bail on failure due to other book keeping needed 797 */ 798 result = ice_update_link_info(pi); 799 if (result) 800 dev_dbg(dev, "Failed to update link status and re-enable link events for port %d\n", 801 pi->lport); 802 803 vsi = ice_get_main_vsi(pf); 804 if (!vsi || !vsi->port_info) 805 return -EINVAL; 806 807 /* turn off PHY if media was removed */ 808 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 809 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 810 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 811 812 result = ice_aq_set_link_restart_an(pi, false, NULL); 813 if (result) { 814 dev_dbg(dev, "Failed to set link down, VSI %d error %d\n", 815 vsi->vsi_num, result); 816 return result; 817 } 818 } 819 820 /* if the old link up/down and speed is the same as the new */ 821 if (link_up == old_link && link_speed == old_link_speed) 822 return result; 823 824 ice_dcb_rebuild(pf); 825 ice_vsi_link_event(vsi, link_up); 826 ice_print_link_msg(vsi, link_up); 827 828 ice_vc_notify_link_state(pf); 829 830 return result; 831 } 832 833 /** 834 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 835 * @pf: board private structure 836 */ 837 static void ice_watchdog_subtask(struct ice_pf *pf) 838 { 839 int i; 840 841 /* if interface is down do nothing */ 842 if (test_bit(__ICE_DOWN, pf->state) || 843 test_bit(__ICE_CFG_BUSY, pf->state)) 844 return; 845 846 /* make sure we don't do these things too often */ 847 if (time_before(jiffies, 848 pf->serv_tmr_prev + pf->serv_tmr_period)) 849 return; 850 851 pf->serv_tmr_prev = jiffies; 852 853 /* Update the stats for active netdevs so the network stack 854 * can look at updated numbers whenever it cares to 855 */ 856 ice_update_pf_stats(pf); 857 ice_for_each_vsi(pf, i) 858 if (pf->vsi[i] && pf->vsi[i]->netdev) 859 ice_update_vsi_stats(pf->vsi[i]); 860 } 861 862 /** 863 * ice_init_link_events - enable/initialize link events 864 * @pi: pointer to the port_info instance 865 * 866 * Returns -EIO on failure, 0 on success 867 */ 868 static int ice_init_link_events(struct ice_port_info *pi) 869 { 870 u16 mask; 871 872 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 873 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL)); 874 875 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 876 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 877 pi->lport); 878 return -EIO; 879 } 880 881 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 882 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 883 pi->lport); 884 return -EIO; 885 } 886 887 return 0; 888 } 889 890 /** 891 * ice_handle_link_event - handle link event via ARQ 892 * @pf: PF that the link event is associated with 893 * @event: event structure containing link status info 894 */ 895 static int 896 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 897 { 898 struct ice_aqc_get_link_status_data *link_data; 899 struct ice_port_info *port_info; 900 int status; 901 902 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 903 port_info = pf->hw.port_info; 904 if (!port_info) 905 return -EINVAL; 906 907 status = ice_link_event(pf, port_info, 908 !!(link_data->link_info & ICE_AQ_LINK_UP), 909 le16_to_cpu(link_data->link_speed)); 910 if (status) 911 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 912 status); 913 914 return status; 915 } 916 917 /** 918 * __ice_clean_ctrlq - helper function to clean controlq rings 919 * @pf: ptr to struct ice_pf 920 * @q_type: specific Control queue type 921 */ 922 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 923 { 924 struct device *dev = ice_pf_to_dev(pf); 925 struct ice_rq_event_info event; 926 struct ice_hw *hw = &pf->hw; 927 struct ice_ctl_q_info *cq; 928 u16 pending, i = 0; 929 const char *qtype; 930 u32 oldval, val; 931 932 /* Do not clean control queue if/when PF reset fails */ 933 if (test_bit(__ICE_RESET_FAILED, pf->state)) 934 return 0; 935 936 switch (q_type) { 937 case ICE_CTL_Q_ADMIN: 938 cq = &hw->adminq; 939 qtype = "Admin"; 940 break; 941 case ICE_CTL_Q_MAILBOX: 942 cq = &hw->mailboxq; 943 qtype = "Mailbox"; 944 break; 945 default: 946 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 947 return 0; 948 } 949 950 /* check for error indications - PF_xx_AxQLEN register layout for 951 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 952 */ 953 val = rd32(hw, cq->rq.len); 954 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 955 PF_FW_ARQLEN_ARQCRIT_M)) { 956 oldval = val; 957 if (val & PF_FW_ARQLEN_ARQVFE_M) 958 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 959 qtype); 960 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 961 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 962 qtype); 963 } 964 if (val & PF_FW_ARQLEN_ARQCRIT_M) 965 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 966 qtype); 967 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 968 PF_FW_ARQLEN_ARQCRIT_M); 969 if (oldval != val) 970 wr32(hw, cq->rq.len, val); 971 } 972 973 val = rd32(hw, cq->sq.len); 974 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 975 PF_FW_ATQLEN_ATQCRIT_M)) { 976 oldval = val; 977 if (val & PF_FW_ATQLEN_ATQVFE_M) 978 dev_dbg(dev, "%s Send Queue VF Error detected\n", 979 qtype); 980 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 981 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 982 qtype); 983 } 984 if (val & PF_FW_ATQLEN_ATQCRIT_M) 985 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 986 qtype); 987 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 988 PF_FW_ATQLEN_ATQCRIT_M); 989 if (oldval != val) 990 wr32(hw, cq->sq.len, val); 991 } 992 993 event.buf_len = cq->rq_buf_size; 994 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 995 if (!event.msg_buf) 996 return 0; 997 998 do { 999 enum ice_status ret; 1000 u16 opcode; 1001 1002 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1003 if (ret == ICE_ERR_AQ_NO_WORK) 1004 break; 1005 if (ret) { 1006 dev_err(dev, "%s Receive Queue event error %s\n", qtype, 1007 ice_stat_str(ret)); 1008 break; 1009 } 1010 1011 opcode = le16_to_cpu(event.desc.opcode); 1012 1013 switch (opcode) { 1014 case ice_aqc_opc_get_link_status: 1015 if (ice_handle_link_event(pf, &event)) 1016 dev_err(dev, "Could not handle link event\n"); 1017 break; 1018 case ice_aqc_opc_event_lan_overflow: 1019 ice_vf_lan_overflow_event(pf, &event); 1020 break; 1021 case ice_mbx_opc_send_msg_to_pf: 1022 ice_vc_process_vf_msg(pf, &event); 1023 break; 1024 case ice_aqc_opc_fw_logging: 1025 ice_output_fw_log(hw, &event.desc, event.msg_buf); 1026 break; 1027 case ice_aqc_opc_lldp_set_mib_change: 1028 ice_dcb_process_lldp_set_mib_change(pf, &event); 1029 break; 1030 default: 1031 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1032 qtype, opcode); 1033 break; 1034 } 1035 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1036 1037 kfree(event.msg_buf); 1038 1039 return pending && (i == ICE_DFLT_IRQ_WORK); 1040 } 1041 1042 /** 1043 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1044 * @hw: pointer to hardware info 1045 * @cq: control queue information 1046 * 1047 * returns true if there are pending messages in a queue, false if there aren't 1048 */ 1049 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1050 { 1051 u16 ntu; 1052 1053 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1054 return cq->rq.next_to_clean != ntu; 1055 } 1056 1057 /** 1058 * ice_clean_adminq_subtask - clean the AdminQ rings 1059 * @pf: board private structure 1060 */ 1061 static void ice_clean_adminq_subtask(struct ice_pf *pf) 1062 { 1063 struct ice_hw *hw = &pf->hw; 1064 1065 if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1066 return; 1067 1068 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1069 return; 1070 1071 clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 1072 1073 /* There might be a situation where new messages arrive to a control 1074 * queue between processing the last message and clearing the 1075 * EVENT_PENDING bit. So before exiting, check queue head again (using 1076 * ice_ctrlq_pending) and process new messages if any. 1077 */ 1078 if (ice_ctrlq_pending(hw, &hw->adminq)) 1079 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1080 1081 ice_flush(hw); 1082 } 1083 1084 /** 1085 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1086 * @pf: board private structure 1087 */ 1088 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1089 { 1090 struct ice_hw *hw = &pf->hw; 1091 1092 if (!test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1093 return; 1094 1095 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1096 return; 1097 1098 clear_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1099 1100 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1101 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1102 1103 ice_flush(hw); 1104 } 1105 1106 /** 1107 * ice_service_task_schedule - schedule the service task to wake up 1108 * @pf: board private structure 1109 * 1110 * If not already scheduled, this puts the task into the work queue. 1111 */ 1112 void ice_service_task_schedule(struct ice_pf *pf) 1113 { 1114 if (!test_bit(__ICE_SERVICE_DIS, pf->state) && 1115 !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) && 1116 !test_bit(__ICE_NEEDS_RESTART, pf->state)) 1117 queue_work(ice_wq, &pf->serv_task); 1118 } 1119 1120 /** 1121 * ice_service_task_complete - finish up the service task 1122 * @pf: board private structure 1123 */ 1124 static void ice_service_task_complete(struct ice_pf *pf) 1125 { 1126 WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state)); 1127 1128 /* force memory (pf->state) to sync before next service task */ 1129 smp_mb__before_atomic(); 1130 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1131 } 1132 1133 /** 1134 * ice_service_task_stop - stop service task and cancel works 1135 * @pf: board private structure 1136 * 1137 * Return 0 if the __ICE_SERVICE_DIS bit was not already set, 1138 * 1 otherwise. 1139 */ 1140 static int ice_service_task_stop(struct ice_pf *pf) 1141 { 1142 int ret; 1143 1144 ret = test_and_set_bit(__ICE_SERVICE_DIS, pf->state); 1145 1146 if (pf->serv_tmr.function) 1147 del_timer_sync(&pf->serv_tmr); 1148 if (pf->serv_task.func) 1149 cancel_work_sync(&pf->serv_task); 1150 1151 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1152 return ret; 1153 } 1154 1155 /** 1156 * ice_service_task_restart - restart service task and schedule works 1157 * @pf: board private structure 1158 * 1159 * This function is needed for suspend and resume works (e.g WoL scenario) 1160 */ 1161 static void ice_service_task_restart(struct ice_pf *pf) 1162 { 1163 clear_bit(__ICE_SERVICE_DIS, pf->state); 1164 ice_service_task_schedule(pf); 1165 } 1166 1167 /** 1168 * ice_service_timer - timer callback to schedule service task 1169 * @t: pointer to timer_list 1170 */ 1171 static void ice_service_timer(struct timer_list *t) 1172 { 1173 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1174 1175 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1176 ice_service_task_schedule(pf); 1177 } 1178 1179 /** 1180 * ice_handle_mdd_event - handle malicious driver detect event 1181 * @pf: pointer to the PF structure 1182 * 1183 * Called from service task. OICR interrupt handler indicates MDD event. 1184 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1185 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1186 * disable the queue, the PF can be configured to reset the VF using ethtool 1187 * private flag mdd-auto-reset-vf. 1188 */ 1189 static void ice_handle_mdd_event(struct ice_pf *pf) 1190 { 1191 struct device *dev = ice_pf_to_dev(pf); 1192 struct ice_hw *hw = &pf->hw; 1193 unsigned int i; 1194 u32 reg; 1195 1196 if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state)) { 1197 /* Since the VF MDD event logging is rate limited, check if 1198 * there are pending MDD events. 1199 */ 1200 ice_print_vfs_mdd_events(pf); 1201 return; 1202 } 1203 1204 /* find what triggered an MDD event */ 1205 reg = rd32(hw, GL_MDET_TX_PQM); 1206 if (reg & GL_MDET_TX_PQM_VALID_M) { 1207 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1208 GL_MDET_TX_PQM_PF_NUM_S; 1209 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1210 GL_MDET_TX_PQM_VF_NUM_S; 1211 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1212 GL_MDET_TX_PQM_MAL_TYPE_S; 1213 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1214 GL_MDET_TX_PQM_QNUM_S); 1215 1216 if (netif_msg_tx_err(pf)) 1217 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1218 event, queue, pf_num, vf_num); 1219 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1220 } 1221 1222 reg = rd32(hw, GL_MDET_TX_TCLAN); 1223 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1224 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1225 GL_MDET_TX_TCLAN_PF_NUM_S; 1226 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1227 GL_MDET_TX_TCLAN_VF_NUM_S; 1228 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1229 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1230 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1231 GL_MDET_TX_TCLAN_QNUM_S); 1232 1233 if (netif_msg_tx_err(pf)) 1234 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1235 event, queue, pf_num, vf_num); 1236 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1237 } 1238 1239 reg = rd32(hw, GL_MDET_RX); 1240 if (reg & GL_MDET_RX_VALID_M) { 1241 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1242 GL_MDET_RX_PF_NUM_S; 1243 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1244 GL_MDET_RX_VF_NUM_S; 1245 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1246 GL_MDET_RX_MAL_TYPE_S; 1247 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1248 GL_MDET_RX_QNUM_S); 1249 1250 if (netif_msg_rx_err(pf)) 1251 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1252 event, queue, pf_num, vf_num); 1253 wr32(hw, GL_MDET_RX, 0xffffffff); 1254 } 1255 1256 /* check to see if this PF caused an MDD event */ 1257 reg = rd32(hw, PF_MDET_TX_PQM); 1258 if (reg & PF_MDET_TX_PQM_VALID_M) { 1259 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1260 if (netif_msg_tx_err(pf)) 1261 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1262 } 1263 1264 reg = rd32(hw, PF_MDET_TX_TCLAN); 1265 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1266 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1267 if (netif_msg_tx_err(pf)) 1268 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1269 } 1270 1271 reg = rd32(hw, PF_MDET_RX); 1272 if (reg & PF_MDET_RX_VALID_M) { 1273 wr32(hw, PF_MDET_RX, 0xFFFF); 1274 if (netif_msg_rx_err(pf)) 1275 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1276 } 1277 1278 /* Check to see if one of the VFs caused an MDD event, and then 1279 * increment counters and set print pending 1280 */ 1281 ice_for_each_vf(pf, i) { 1282 struct ice_vf *vf = &pf->vf[i]; 1283 1284 reg = rd32(hw, VP_MDET_TX_PQM(i)); 1285 if (reg & VP_MDET_TX_PQM_VALID_M) { 1286 wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF); 1287 vf->mdd_tx_events.count++; 1288 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1289 if (netif_msg_tx_err(pf)) 1290 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1291 i); 1292 } 1293 1294 reg = rd32(hw, VP_MDET_TX_TCLAN(i)); 1295 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1296 wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF); 1297 vf->mdd_tx_events.count++; 1298 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1299 if (netif_msg_tx_err(pf)) 1300 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1301 i); 1302 } 1303 1304 reg = rd32(hw, VP_MDET_TX_TDPU(i)); 1305 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1306 wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF); 1307 vf->mdd_tx_events.count++; 1308 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1309 if (netif_msg_tx_err(pf)) 1310 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1311 i); 1312 } 1313 1314 reg = rd32(hw, VP_MDET_RX(i)); 1315 if (reg & VP_MDET_RX_VALID_M) { 1316 wr32(hw, VP_MDET_RX(i), 0xFFFF); 1317 vf->mdd_rx_events.count++; 1318 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1319 if (netif_msg_rx_err(pf)) 1320 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1321 i); 1322 1323 /* Since the queue is disabled on VF Rx MDD events, the 1324 * PF can be configured to reset the VF through ethtool 1325 * private flag mdd-auto-reset-vf. 1326 */ 1327 if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 1328 /* VF MDD event counters will be cleared by 1329 * reset, so print the event prior to reset. 1330 */ 1331 ice_print_vf_rx_mdd_event(vf); 1332 ice_reset_vf(&pf->vf[i], false); 1333 } 1334 } 1335 } 1336 1337 ice_print_vfs_mdd_events(pf); 1338 } 1339 1340 /** 1341 * ice_force_phys_link_state - Force the physical link state 1342 * @vsi: VSI to force the physical link state to up/down 1343 * @link_up: true/false indicates to set the physical link to up/down 1344 * 1345 * Force the physical link state by getting the current PHY capabilities from 1346 * hardware and setting the PHY config based on the determined capabilities. If 1347 * link changes a link event will be triggered because both the Enable Automatic 1348 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1349 * 1350 * Returns 0 on success, negative on failure 1351 */ 1352 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1353 { 1354 struct ice_aqc_get_phy_caps_data *pcaps; 1355 struct ice_aqc_set_phy_cfg_data *cfg; 1356 struct ice_port_info *pi; 1357 struct device *dev; 1358 int retcode; 1359 1360 if (!vsi || !vsi->port_info || !vsi->back) 1361 return -EINVAL; 1362 if (vsi->type != ICE_VSI_PF) 1363 return 0; 1364 1365 dev = ice_pf_to_dev(vsi->back); 1366 1367 pi = vsi->port_info; 1368 1369 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1370 if (!pcaps) 1371 return -ENOMEM; 1372 1373 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1374 NULL); 1375 if (retcode) { 1376 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1377 vsi->vsi_num, retcode); 1378 retcode = -EIO; 1379 goto out; 1380 } 1381 1382 /* No change in link */ 1383 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1384 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1385 goto out; 1386 1387 /* Use the current user PHY configuration. The current user PHY 1388 * configuration is initialized during probe from PHY capabilities 1389 * software mode, and updated on set PHY configuration. 1390 */ 1391 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1392 if (!cfg) { 1393 retcode = -ENOMEM; 1394 goto out; 1395 } 1396 1397 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1398 if (link_up) 1399 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1400 else 1401 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1402 1403 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1404 if (retcode) { 1405 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1406 vsi->vsi_num, retcode); 1407 retcode = -EIO; 1408 } 1409 1410 kfree(cfg); 1411 out: 1412 kfree(pcaps); 1413 return retcode; 1414 } 1415 1416 /** 1417 * ice_init_nvm_phy_type - Initialize the NVM PHY type 1418 * @pi: port info structure 1419 * 1420 * Initialize nvm_phy_type_[low|high] for link lenient mode support 1421 */ 1422 static int ice_init_nvm_phy_type(struct ice_port_info *pi) 1423 { 1424 struct ice_aqc_get_phy_caps_data *pcaps; 1425 struct ice_pf *pf = pi->hw->back; 1426 enum ice_status status; 1427 int err = 0; 1428 1429 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1430 if (!pcaps) 1431 return -ENOMEM; 1432 1433 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_NVM_CAP, pcaps, 1434 NULL); 1435 1436 if (status) { 1437 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1438 err = -EIO; 1439 goto out; 1440 } 1441 1442 pf->nvm_phy_type_hi = pcaps->phy_type_high; 1443 pf->nvm_phy_type_lo = pcaps->phy_type_low; 1444 1445 out: 1446 kfree(pcaps); 1447 return err; 1448 } 1449 1450 /** 1451 * ice_init_link_dflt_override - Initialize link default override 1452 * @pi: port info structure 1453 * 1454 * Initialize link default override and PHY total port shutdown during probe 1455 */ 1456 static void ice_init_link_dflt_override(struct ice_port_info *pi) 1457 { 1458 struct ice_link_default_override_tlv *ldo; 1459 struct ice_pf *pf = pi->hw->back; 1460 1461 ldo = &pf->link_dflt_override; 1462 if (ice_get_link_default_override(ldo, pi)) 1463 return; 1464 1465 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 1466 return; 1467 1468 /* Enable Total Port Shutdown (override/replace link-down-on-close 1469 * ethtool private flag) for ports with Port Disable bit set. 1470 */ 1471 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 1472 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 1473 } 1474 1475 /** 1476 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 1477 * @pi: port info structure 1478 * 1479 * If default override is enabled, initialized the user PHY cfg speed and FEC 1480 * settings using the default override mask from the NVM. 1481 * 1482 * The PHY should only be configured with the default override settings the 1483 * first time media is available. The __ICE_LINK_DEFAULT_OVERRIDE_PENDING state 1484 * is used to indicate that the user PHY cfg default override is initialized 1485 * and the PHY has not been configured with the default override settings. The 1486 * state is set here, and cleared in ice_configure_phy the first time the PHY is 1487 * configured. 1488 */ 1489 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 1490 { 1491 struct ice_link_default_override_tlv *ldo; 1492 struct ice_aqc_set_phy_cfg_data *cfg; 1493 struct ice_phy_info *phy = &pi->phy; 1494 struct ice_pf *pf = pi->hw->back; 1495 1496 ldo = &pf->link_dflt_override; 1497 1498 /* If link default override is enabled, use to mask NVM PHY capabilities 1499 * for speed and FEC default configuration. 1500 */ 1501 cfg = &phy->curr_user_phy_cfg; 1502 1503 if (ldo->phy_type_low || ldo->phy_type_high) { 1504 cfg->phy_type_low = pf->nvm_phy_type_lo & 1505 cpu_to_le64(ldo->phy_type_low); 1506 cfg->phy_type_high = pf->nvm_phy_type_hi & 1507 cpu_to_le64(ldo->phy_type_high); 1508 } 1509 cfg->link_fec_opt = ldo->fec_options; 1510 phy->curr_user_fec_req = ICE_FEC_AUTO; 1511 1512 set_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 1513 } 1514 1515 /** 1516 * ice_init_phy_user_cfg - Initialize the PHY user configuration 1517 * @pi: port info structure 1518 * 1519 * Initialize the current user PHY configuration, speed, FEC, and FC requested 1520 * mode to default. The PHY defaults are from get PHY capabilities topology 1521 * with media so call when media is first available. An error is returned if 1522 * called when media is not available. The PHY initialization completed state is 1523 * set here. 1524 * 1525 * These configurations are used when setting PHY 1526 * configuration. The user PHY configuration is updated on set PHY 1527 * configuration. Returns 0 on success, negative on failure 1528 */ 1529 static int ice_init_phy_user_cfg(struct ice_port_info *pi) 1530 { 1531 struct ice_aqc_get_phy_caps_data *pcaps; 1532 struct ice_phy_info *phy = &pi->phy; 1533 struct ice_pf *pf = pi->hw->back; 1534 enum ice_status status; 1535 struct ice_vsi *vsi; 1536 int err = 0; 1537 1538 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1539 return -EIO; 1540 1541 vsi = ice_get_main_vsi(pf); 1542 if (!vsi) 1543 return -EINVAL; 1544 1545 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1546 if (!pcaps) 1547 return -ENOMEM; 1548 1549 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1550 NULL); 1551 if (status) { 1552 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1553 err = -EIO; 1554 goto err_out; 1555 } 1556 1557 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 1558 1559 /* check if lenient mode is supported and enabled */ 1560 if (ice_fw_supports_link_override(&vsi->back->hw) && 1561 !(pcaps->module_compliance_enforcement & 1562 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 1563 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 1564 1565 /* if link default override is enabled, initialize user PHY 1566 * configuration with link default override values 1567 */ 1568 if (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN) { 1569 ice_init_phy_cfg_dflt_override(pi); 1570 goto out; 1571 } 1572 } 1573 1574 /* if link default override is not enabled, initialize PHY using 1575 * topology with media 1576 */ 1577 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 1578 pcaps->link_fec_options); 1579 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 1580 1581 out: 1582 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 1583 set_bit(__ICE_PHY_INIT_COMPLETE, pf->state); 1584 err_out: 1585 kfree(pcaps); 1586 return err; 1587 } 1588 1589 /** 1590 * ice_configure_phy - configure PHY 1591 * @vsi: VSI of PHY 1592 * 1593 * Set the PHY configuration. If the current PHY configuration is the same as 1594 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 1595 * configure the based get PHY capabilities for topology with media. 1596 */ 1597 static int ice_configure_phy(struct ice_vsi *vsi) 1598 { 1599 struct device *dev = ice_pf_to_dev(vsi->back); 1600 struct ice_aqc_get_phy_caps_data *pcaps; 1601 struct ice_aqc_set_phy_cfg_data *cfg; 1602 struct ice_port_info *pi; 1603 enum ice_status status; 1604 int err = 0; 1605 1606 pi = vsi->port_info; 1607 if (!pi) 1608 return -EINVAL; 1609 1610 /* Ensure we have media as we cannot configure a medialess port */ 1611 if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1612 return -EPERM; 1613 1614 ice_print_topo_conflict(vsi); 1615 1616 if (vsi->port_info->phy.link_info.topo_media_conflict == 1617 ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 1618 return -EPERM; 1619 1620 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 1621 return ice_force_phys_link_state(vsi, true); 1622 1623 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1624 if (!pcaps) 1625 return -ENOMEM; 1626 1627 /* Get current PHY config */ 1628 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1629 NULL); 1630 if (status) { 1631 dev_err(dev, "Failed to get PHY configuration, VSI %d error %s\n", 1632 vsi->vsi_num, ice_stat_str(status)); 1633 err = -EIO; 1634 goto done; 1635 } 1636 1637 /* If PHY enable link is configured and configuration has not changed, 1638 * there's nothing to do 1639 */ 1640 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 1641 ice_phy_caps_equals_cfg(pcaps, &pi->phy.curr_user_phy_cfg)) 1642 goto done; 1643 1644 /* Use PHY topology as baseline for configuration */ 1645 memset(pcaps, 0, sizeof(*pcaps)); 1646 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1647 NULL); 1648 if (status) { 1649 dev_err(dev, "Failed to get PHY topology, VSI %d error %s\n", 1650 vsi->vsi_num, ice_stat_str(status)); 1651 err = -EIO; 1652 goto done; 1653 } 1654 1655 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 1656 if (!cfg) { 1657 err = -ENOMEM; 1658 goto done; 1659 } 1660 1661 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 1662 1663 /* Speed - If default override pending, use curr_user_phy_cfg set in 1664 * ice_init_phy_user_cfg_ldo. 1665 */ 1666 if (test_and_clear_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, 1667 vsi->back->state)) { 1668 cfg->phy_type_low = pi->phy.curr_user_phy_cfg.phy_type_low; 1669 cfg->phy_type_high = pi->phy.curr_user_phy_cfg.phy_type_high; 1670 } else { 1671 u64 phy_low = 0, phy_high = 0; 1672 1673 ice_update_phy_type(&phy_low, &phy_high, 1674 pi->phy.curr_user_speed_req); 1675 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 1676 cfg->phy_type_high = pcaps->phy_type_high & 1677 cpu_to_le64(phy_high); 1678 } 1679 1680 /* Can't provide what was requested; use PHY capabilities */ 1681 if (!cfg->phy_type_low && !cfg->phy_type_high) { 1682 cfg->phy_type_low = pcaps->phy_type_low; 1683 cfg->phy_type_high = pcaps->phy_type_high; 1684 } 1685 1686 /* FEC */ 1687 ice_cfg_phy_fec(pi, cfg, pi->phy.curr_user_fec_req); 1688 1689 /* Can't provide what was requested; use PHY capabilities */ 1690 if (cfg->link_fec_opt != 1691 (cfg->link_fec_opt & pcaps->link_fec_options)) { 1692 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 1693 cfg->link_fec_opt = pcaps->link_fec_options; 1694 } 1695 1696 /* Flow Control - always supported; no need to check against 1697 * capabilities 1698 */ 1699 ice_cfg_phy_fc(pi, cfg, pi->phy.curr_user_fc_req); 1700 1701 /* Enable link and link update */ 1702 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 1703 1704 status = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1705 if (status) { 1706 dev_err(dev, "Failed to set phy config, VSI %d error %s\n", 1707 vsi->vsi_num, ice_stat_str(status)); 1708 err = -EIO; 1709 } 1710 1711 kfree(cfg); 1712 done: 1713 kfree(pcaps); 1714 return err; 1715 } 1716 1717 /** 1718 * ice_check_media_subtask - Check for media 1719 * @pf: pointer to PF struct 1720 * 1721 * If media is available, then initialize PHY user configuration if it is not 1722 * been, and configure the PHY if the interface is up. 1723 */ 1724 static void ice_check_media_subtask(struct ice_pf *pf) 1725 { 1726 struct ice_port_info *pi; 1727 struct ice_vsi *vsi; 1728 int err; 1729 1730 /* No need to check for media if it's already present */ 1731 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 1732 return; 1733 1734 vsi = ice_get_main_vsi(pf); 1735 if (!vsi) 1736 return; 1737 1738 /* Refresh link info and check if media is present */ 1739 pi = vsi->port_info; 1740 err = ice_update_link_info(pi); 1741 if (err) 1742 return; 1743 1744 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 1745 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) 1746 ice_init_phy_user_cfg(pi); 1747 1748 /* PHY settings are reset on media insertion, reconfigure 1749 * PHY to preserve settings. 1750 */ 1751 if (test_bit(__ICE_DOWN, vsi->state) && 1752 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 1753 return; 1754 1755 err = ice_configure_phy(vsi); 1756 if (!err) 1757 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 1758 1759 /* A Link Status Event will be generated; the event handler 1760 * will complete bringing the interface up 1761 */ 1762 } 1763 } 1764 1765 /** 1766 * ice_service_task - manage and run subtasks 1767 * @work: pointer to work_struct contained by the PF struct 1768 */ 1769 static void ice_service_task(struct work_struct *work) 1770 { 1771 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 1772 unsigned long start_time = jiffies; 1773 1774 /* subtasks */ 1775 1776 /* process reset requests first */ 1777 ice_reset_subtask(pf); 1778 1779 /* bail if a reset/recovery cycle is pending or rebuild failed */ 1780 if (ice_is_reset_in_progress(pf->state) || 1781 test_bit(__ICE_SUSPENDED, pf->state) || 1782 test_bit(__ICE_NEEDS_RESTART, pf->state)) { 1783 ice_service_task_complete(pf); 1784 return; 1785 } 1786 1787 ice_clean_adminq_subtask(pf); 1788 ice_check_media_subtask(pf); 1789 ice_check_for_hang_subtask(pf); 1790 ice_sync_fltr_subtask(pf); 1791 ice_handle_mdd_event(pf); 1792 ice_watchdog_subtask(pf); 1793 1794 if (ice_is_safe_mode(pf)) { 1795 ice_service_task_complete(pf); 1796 return; 1797 } 1798 1799 ice_process_vflr_event(pf); 1800 ice_clean_mailboxq_subtask(pf); 1801 ice_sync_arfs_fltrs(pf); 1802 /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */ 1803 ice_service_task_complete(pf); 1804 1805 /* If the tasks have taken longer than one service timer period 1806 * or there is more work to be done, reset the service timer to 1807 * schedule the service task now. 1808 */ 1809 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 1810 test_bit(__ICE_MDD_EVENT_PENDING, pf->state) || 1811 test_bit(__ICE_VFLR_EVENT_PENDING, pf->state) || 1812 test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 1813 test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1814 mod_timer(&pf->serv_tmr, jiffies); 1815 } 1816 1817 /** 1818 * ice_set_ctrlq_len - helper function to set controlq length 1819 * @hw: pointer to the HW instance 1820 */ 1821 static void ice_set_ctrlq_len(struct ice_hw *hw) 1822 { 1823 hw->adminq.num_rq_entries = ICE_AQ_LEN; 1824 hw->adminq.num_sq_entries = ICE_AQ_LEN; 1825 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 1826 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 1827 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 1828 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 1829 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1830 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1831 } 1832 1833 /** 1834 * ice_schedule_reset - schedule a reset 1835 * @pf: board private structure 1836 * @reset: reset being requested 1837 */ 1838 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 1839 { 1840 struct device *dev = ice_pf_to_dev(pf); 1841 1842 /* bail out if earlier reset has failed */ 1843 if (test_bit(__ICE_RESET_FAILED, pf->state)) { 1844 dev_dbg(dev, "earlier reset has failed\n"); 1845 return -EIO; 1846 } 1847 /* bail if reset/recovery already in progress */ 1848 if (ice_is_reset_in_progress(pf->state)) { 1849 dev_dbg(dev, "Reset already in progress\n"); 1850 return -EBUSY; 1851 } 1852 1853 switch (reset) { 1854 case ICE_RESET_PFR: 1855 set_bit(__ICE_PFR_REQ, pf->state); 1856 break; 1857 case ICE_RESET_CORER: 1858 set_bit(__ICE_CORER_REQ, pf->state); 1859 break; 1860 case ICE_RESET_GLOBR: 1861 set_bit(__ICE_GLOBR_REQ, pf->state); 1862 break; 1863 default: 1864 return -EINVAL; 1865 } 1866 1867 ice_service_task_schedule(pf); 1868 return 0; 1869 } 1870 1871 /** 1872 * ice_irq_affinity_notify - Callback for affinity changes 1873 * @notify: context as to what irq was changed 1874 * @mask: the new affinity mask 1875 * 1876 * This is a callback function used by the irq_set_affinity_notifier function 1877 * so that we may register to receive changes to the irq affinity masks. 1878 */ 1879 static void 1880 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 1881 const cpumask_t *mask) 1882 { 1883 struct ice_q_vector *q_vector = 1884 container_of(notify, struct ice_q_vector, affinity_notify); 1885 1886 cpumask_copy(&q_vector->affinity_mask, mask); 1887 } 1888 1889 /** 1890 * ice_irq_affinity_release - Callback for affinity notifier release 1891 * @ref: internal core kernel usage 1892 * 1893 * This is a callback function used by the irq_set_affinity_notifier function 1894 * to inform the current notification subscriber that they will no longer 1895 * receive notifications. 1896 */ 1897 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 1898 1899 /** 1900 * ice_vsi_ena_irq - Enable IRQ for the given VSI 1901 * @vsi: the VSI being configured 1902 */ 1903 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 1904 { 1905 struct ice_hw *hw = &vsi->back->hw; 1906 int i; 1907 1908 ice_for_each_q_vector(vsi, i) 1909 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 1910 1911 ice_flush(hw); 1912 return 0; 1913 } 1914 1915 /** 1916 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 1917 * @vsi: the VSI being configured 1918 * @basename: name for the vector 1919 */ 1920 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 1921 { 1922 int q_vectors = vsi->num_q_vectors; 1923 struct ice_pf *pf = vsi->back; 1924 int base = vsi->base_vector; 1925 struct device *dev; 1926 int rx_int_idx = 0; 1927 int tx_int_idx = 0; 1928 int vector, err; 1929 int irq_num; 1930 1931 dev = ice_pf_to_dev(pf); 1932 for (vector = 0; vector < q_vectors; vector++) { 1933 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 1934 1935 irq_num = pf->msix_entries[base + vector].vector; 1936 1937 if (q_vector->tx.ring && q_vector->rx.ring) { 1938 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1939 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 1940 tx_int_idx++; 1941 } else if (q_vector->rx.ring) { 1942 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1943 "%s-%s-%d", basename, "rx", rx_int_idx++); 1944 } else if (q_vector->tx.ring) { 1945 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1946 "%s-%s-%d", basename, "tx", tx_int_idx++); 1947 } else { 1948 /* skip this unused q_vector */ 1949 continue; 1950 } 1951 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 0, 1952 q_vector->name, q_vector); 1953 if (err) { 1954 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 1955 err); 1956 goto free_q_irqs; 1957 } 1958 1959 /* register for affinity change notifications */ 1960 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { 1961 struct irq_affinity_notify *affinity_notify; 1962 1963 affinity_notify = &q_vector->affinity_notify; 1964 affinity_notify->notify = ice_irq_affinity_notify; 1965 affinity_notify->release = ice_irq_affinity_release; 1966 irq_set_affinity_notifier(irq_num, affinity_notify); 1967 } 1968 1969 /* assign the mask for this irq */ 1970 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 1971 } 1972 1973 vsi->irqs_ready = true; 1974 return 0; 1975 1976 free_q_irqs: 1977 while (vector) { 1978 vector--; 1979 irq_num = pf->msix_entries[base + vector].vector; 1980 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) 1981 irq_set_affinity_notifier(irq_num, NULL); 1982 irq_set_affinity_hint(irq_num, NULL); 1983 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 1984 } 1985 return err; 1986 } 1987 1988 /** 1989 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 1990 * @vsi: VSI to setup Tx rings used by XDP 1991 * 1992 * Return 0 on success and negative value on error 1993 */ 1994 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 1995 { 1996 struct device *dev = ice_pf_to_dev(vsi->back); 1997 int i; 1998 1999 for (i = 0; i < vsi->num_xdp_txq; i++) { 2000 u16 xdp_q_idx = vsi->alloc_txq + i; 2001 struct ice_ring *xdp_ring; 2002 2003 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2004 2005 if (!xdp_ring) 2006 goto free_xdp_rings; 2007 2008 xdp_ring->q_index = xdp_q_idx; 2009 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2010 xdp_ring->ring_active = false; 2011 xdp_ring->vsi = vsi; 2012 xdp_ring->netdev = NULL; 2013 xdp_ring->dev = dev; 2014 xdp_ring->count = vsi->num_tx_desc; 2015 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2016 if (ice_setup_tx_ring(xdp_ring)) 2017 goto free_xdp_rings; 2018 ice_set_ring_xdp(xdp_ring); 2019 xdp_ring->xsk_umem = ice_xsk_umem(xdp_ring); 2020 } 2021 2022 return 0; 2023 2024 free_xdp_rings: 2025 for (; i >= 0; i--) 2026 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2027 ice_free_tx_ring(vsi->xdp_rings[i]); 2028 return -ENOMEM; 2029 } 2030 2031 /** 2032 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2033 * @vsi: VSI to set the bpf prog on 2034 * @prog: the bpf prog pointer 2035 */ 2036 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2037 { 2038 struct bpf_prog *old_prog; 2039 int i; 2040 2041 old_prog = xchg(&vsi->xdp_prog, prog); 2042 if (old_prog) 2043 bpf_prog_put(old_prog); 2044 2045 ice_for_each_rxq(vsi, i) 2046 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2047 } 2048 2049 /** 2050 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2051 * @vsi: VSI to bring up Tx rings used by XDP 2052 * @prog: bpf program that will be assigned to VSI 2053 * 2054 * Return 0 on success and negative value on error 2055 */ 2056 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) 2057 { 2058 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2059 int xdp_rings_rem = vsi->num_xdp_txq; 2060 struct ice_pf *pf = vsi->back; 2061 struct ice_qs_cfg xdp_qs_cfg = { 2062 .qs_mutex = &pf->avail_q_mutex, 2063 .pf_map = pf->avail_txqs, 2064 .pf_map_size = pf->max_pf_txqs, 2065 .q_count = vsi->num_xdp_txq, 2066 .scatter_count = ICE_MAX_SCATTER_TXQS, 2067 .vsi_map = vsi->txq_map, 2068 .vsi_map_offset = vsi->alloc_txq, 2069 .mapping_mode = ICE_VSI_MAP_CONTIG 2070 }; 2071 enum ice_status status; 2072 struct device *dev; 2073 int i, v_idx; 2074 2075 dev = ice_pf_to_dev(pf); 2076 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2077 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2078 if (!vsi->xdp_rings) 2079 return -ENOMEM; 2080 2081 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2082 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2083 goto err_map_xdp; 2084 2085 if (ice_xdp_alloc_setup_rings(vsi)) 2086 goto clear_xdp_rings; 2087 2088 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2089 ice_for_each_q_vector(vsi, v_idx) { 2090 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2091 int xdp_rings_per_v, q_id, q_base; 2092 2093 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2094 vsi->num_q_vectors - v_idx); 2095 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2096 2097 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2098 struct ice_ring *xdp_ring = vsi->xdp_rings[q_id]; 2099 2100 xdp_ring->q_vector = q_vector; 2101 xdp_ring->next = q_vector->tx.ring; 2102 q_vector->tx.ring = xdp_ring; 2103 } 2104 xdp_rings_rem -= xdp_rings_per_v; 2105 } 2106 2107 /* omit the scheduler update if in reset path; XDP queues will be 2108 * taken into account at the end of ice_vsi_rebuild, where 2109 * ice_cfg_vsi_lan is being called 2110 */ 2111 if (ice_is_reset_in_progress(pf->state)) 2112 return 0; 2113 2114 /* tell the Tx scheduler that right now we have 2115 * additional queues 2116 */ 2117 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2118 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2119 2120 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2121 max_txqs); 2122 if (status) { 2123 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %s\n", 2124 ice_stat_str(status)); 2125 goto clear_xdp_rings; 2126 } 2127 ice_vsi_assign_bpf_prog(vsi, prog); 2128 2129 return 0; 2130 clear_xdp_rings: 2131 for (i = 0; i < vsi->num_xdp_txq; i++) 2132 if (vsi->xdp_rings[i]) { 2133 kfree_rcu(vsi->xdp_rings[i], rcu); 2134 vsi->xdp_rings[i] = NULL; 2135 } 2136 2137 err_map_xdp: 2138 mutex_lock(&pf->avail_q_mutex); 2139 for (i = 0; i < vsi->num_xdp_txq; i++) { 2140 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2141 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2142 } 2143 mutex_unlock(&pf->avail_q_mutex); 2144 2145 devm_kfree(dev, vsi->xdp_rings); 2146 return -ENOMEM; 2147 } 2148 2149 /** 2150 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2151 * @vsi: VSI to remove XDP rings 2152 * 2153 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2154 * resources 2155 */ 2156 int ice_destroy_xdp_rings(struct ice_vsi *vsi) 2157 { 2158 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2159 struct ice_pf *pf = vsi->back; 2160 int i, v_idx; 2161 2162 /* q_vectors are freed in reset path so there's no point in detaching 2163 * rings; in case of rebuild being triggered not from reset reset bits 2164 * in pf->state won't be set, so additionally check first q_vector 2165 * against NULL 2166 */ 2167 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2168 goto free_qmap; 2169 2170 ice_for_each_q_vector(vsi, v_idx) { 2171 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2172 struct ice_ring *ring; 2173 2174 ice_for_each_ring(ring, q_vector->tx) 2175 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2176 break; 2177 2178 /* restore the value of last node prior to XDP setup */ 2179 q_vector->tx.ring = ring; 2180 } 2181 2182 free_qmap: 2183 mutex_lock(&pf->avail_q_mutex); 2184 for (i = 0; i < vsi->num_xdp_txq; i++) { 2185 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2186 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2187 } 2188 mutex_unlock(&pf->avail_q_mutex); 2189 2190 for (i = 0; i < vsi->num_xdp_txq; i++) 2191 if (vsi->xdp_rings[i]) { 2192 if (vsi->xdp_rings[i]->desc) 2193 ice_free_tx_ring(vsi->xdp_rings[i]); 2194 kfree_rcu(vsi->xdp_rings[i], rcu); 2195 vsi->xdp_rings[i] = NULL; 2196 } 2197 2198 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2199 vsi->xdp_rings = NULL; 2200 2201 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2202 return 0; 2203 2204 ice_vsi_assign_bpf_prog(vsi, NULL); 2205 2206 /* notify Tx scheduler that we destroyed XDP queues and bring 2207 * back the old number of child nodes 2208 */ 2209 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2210 max_txqs[i] = vsi->num_txq; 2211 2212 /* change number of XDP Tx queues to 0 */ 2213 vsi->num_xdp_txq = 0; 2214 2215 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2216 max_txqs); 2217 } 2218 2219 /** 2220 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2221 * @vsi: VSI to setup XDP for 2222 * @prog: XDP program 2223 * @extack: netlink extended ack 2224 */ 2225 static int 2226 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2227 struct netlink_ext_ack *extack) 2228 { 2229 int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2230 bool if_running = netif_running(vsi->netdev); 2231 int ret = 0, xdp_ring_err = 0; 2232 2233 if (frame_size > vsi->rx_buf_len) { 2234 NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP"); 2235 return -EOPNOTSUPP; 2236 } 2237 2238 /* need to stop netdev while setting up the program for Rx rings */ 2239 if (if_running && !test_and_set_bit(__ICE_DOWN, vsi->state)) { 2240 ret = ice_down(vsi); 2241 if (ret) { 2242 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 2243 return ret; 2244 } 2245 } 2246 2247 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 2248 vsi->num_xdp_txq = vsi->alloc_rxq; 2249 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); 2250 if (xdp_ring_err) 2251 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 2252 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 2253 xdp_ring_err = ice_destroy_xdp_rings(vsi); 2254 if (xdp_ring_err) 2255 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 2256 } else { 2257 ice_vsi_assign_bpf_prog(vsi, prog); 2258 } 2259 2260 if (if_running) 2261 ret = ice_up(vsi); 2262 2263 if (!ret && prog && vsi->xsk_umems) { 2264 int i; 2265 2266 ice_for_each_rxq(vsi, i) { 2267 struct ice_ring *rx_ring = vsi->rx_rings[i]; 2268 2269 if (rx_ring->xsk_umem) 2270 napi_schedule(&rx_ring->q_vector->napi); 2271 } 2272 } 2273 2274 return (ret || xdp_ring_err) ? -ENOMEM : 0; 2275 } 2276 2277 /** 2278 * ice_xdp - implements XDP handler 2279 * @dev: netdevice 2280 * @xdp: XDP command 2281 */ 2282 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2283 { 2284 struct ice_netdev_priv *np = netdev_priv(dev); 2285 struct ice_vsi *vsi = np->vsi; 2286 2287 if (vsi->type != ICE_VSI_PF) { 2288 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); 2289 return -EINVAL; 2290 } 2291 2292 switch (xdp->command) { 2293 case XDP_SETUP_PROG: 2294 return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 2295 case XDP_QUERY_PROG: 2296 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 2297 return 0; 2298 case XDP_SETUP_XSK_UMEM: 2299 return ice_xsk_umem_setup(vsi, xdp->xsk.umem, 2300 xdp->xsk.queue_id); 2301 default: 2302 return -EINVAL; 2303 } 2304 } 2305 2306 /** 2307 * ice_ena_misc_vector - enable the non-queue interrupts 2308 * @pf: board private structure 2309 */ 2310 static void ice_ena_misc_vector(struct ice_pf *pf) 2311 { 2312 struct ice_hw *hw = &pf->hw; 2313 u32 val; 2314 2315 /* Disable anti-spoof detection interrupt to prevent spurious event 2316 * interrupts during a function reset. Anti-spoof functionally is 2317 * still supported. 2318 */ 2319 val = rd32(hw, GL_MDCK_TX_TDPU); 2320 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 2321 wr32(hw, GL_MDCK_TX_TDPU, val); 2322 2323 /* clear things first */ 2324 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 2325 rd32(hw, PFINT_OICR); /* read to clear */ 2326 2327 val = (PFINT_OICR_ECC_ERR_M | 2328 PFINT_OICR_MAL_DETECT_M | 2329 PFINT_OICR_GRST_M | 2330 PFINT_OICR_PCI_EXCEPTION_M | 2331 PFINT_OICR_VFLR_M | 2332 PFINT_OICR_HMC_ERR_M | 2333 PFINT_OICR_PE_CRITERR_M); 2334 2335 wr32(hw, PFINT_OICR_ENA, val); 2336 2337 /* SW_ITR_IDX = 0, but don't change INTENA */ 2338 wr32(hw, GLINT_DYN_CTL(pf->oicr_idx), 2339 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 2340 } 2341 2342 /** 2343 * ice_misc_intr - misc interrupt handler 2344 * @irq: interrupt number 2345 * @data: pointer to a q_vector 2346 */ 2347 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 2348 { 2349 struct ice_pf *pf = (struct ice_pf *)data; 2350 struct ice_hw *hw = &pf->hw; 2351 irqreturn_t ret = IRQ_NONE; 2352 struct device *dev; 2353 u32 oicr, ena_mask; 2354 2355 dev = ice_pf_to_dev(pf); 2356 set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 2357 set_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 2358 2359 oicr = rd32(hw, PFINT_OICR); 2360 ena_mask = rd32(hw, PFINT_OICR_ENA); 2361 2362 if (oicr & PFINT_OICR_SWINT_M) { 2363 ena_mask &= ~PFINT_OICR_SWINT_M; 2364 pf->sw_int_count++; 2365 } 2366 2367 if (oicr & PFINT_OICR_MAL_DETECT_M) { 2368 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 2369 set_bit(__ICE_MDD_EVENT_PENDING, pf->state); 2370 } 2371 if (oicr & PFINT_OICR_VFLR_M) { 2372 /* disable any further VFLR event notifications */ 2373 if (test_bit(__ICE_VF_RESETS_DISABLED, pf->state)) { 2374 u32 reg = rd32(hw, PFINT_OICR_ENA); 2375 2376 reg &= ~PFINT_OICR_VFLR_M; 2377 wr32(hw, PFINT_OICR_ENA, reg); 2378 } else { 2379 ena_mask &= ~PFINT_OICR_VFLR_M; 2380 set_bit(__ICE_VFLR_EVENT_PENDING, pf->state); 2381 } 2382 } 2383 2384 if (oicr & PFINT_OICR_GRST_M) { 2385 u32 reset; 2386 2387 /* we have a reset warning */ 2388 ena_mask &= ~PFINT_OICR_GRST_M; 2389 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 2390 GLGEN_RSTAT_RESET_TYPE_S; 2391 2392 if (reset == ICE_RESET_CORER) 2393 pf->corer_count++; 2394 else if (reset == ICE_RESET_GLOBR) 2395 pf->globr_count++; 2396 else if (reset == ICE_RESET_EMPR) 2397 pf->empr_count++; 2398 else 2399 dev_dbg(dev, "Invalid reset type %d\n", reset); 2400 2401 /* If a reset cycle isn't already in progress, we set a bit in 2402 * pf->state so that the service task can start a reset/rebuild. 2403 * We also make note of which reset happened so that peer 2404 * devices/drivers can be informed. 2405 */ 2406 if (!test_and_set_bit(__ICE_RESET_OICR_RECV, pf->state)) { 2407 if (reset == ICE_RESET_CORER) 2408 set_bit(__ICE_CORER_RECV, pf->state); 2409 else if (reset == ICE_RESET_GLOBR) 2410 set_bit(__ICE_GLOBR_RECV, pf->state); 2411 else 2412 set_bit(__ICE_EMPR_RECV, pf->state); 2413 2414 /* There are couple of different bits at play here. 2415 * hw->reset_ongoing indicates whether the hardware is 2416 * in reset. This is set to true when a reset interrupt 2417 * is received and set back to false after the driver 2418 * has determined that the hardware is out of reset. 2419 * 2420 * __ICE_RESET_OICR_RECV in pf->state indicates 2421 * that a post reset rebuild is required before the 2422 * driver is operational again. This is set above. 2423 * 2424 * As this is the start of the reset/rebuild cycle, set 2425 * both to indicate that. 2426 */ 2427 hw->reset_ongoing = true; 2428 } 2429 } 2430 2431 if (oicr & PFINT_OICR_HMC_ERR_M) { 2432 ena_mask &= ~PFINT_OICR_HMC_ERR_M; 2433 dev_dbg(dev, "HMC Error interrupt - info 0x%x, data 0x%x\n", 2434 rd32(hw, PFHMC_ERRORINFO), 2435 rd32(hw, PFHMC_ERRORDATA)); 2436 } 2437 2438 /* Report any remaining unexpected interrupts */ 2439 oicr &= ena_mask; 2440 if (oicr) { 2441 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 2442 /* If a critical error is pending there is no choice but to 2443 * reset the device. 2444 */ 2445 if (oicr & (PFINT_OICR_PE_CRITERR_M | 2446 PFINT_OICR_PCI_EXCEPTION_M | 2447 PFINT_OICR_ECC_ERR_M)) { 2448 set_bit(__ICE_PFR_REQ, pf->state); 2449 ice_service_task_schedule(pf); 2450 } 2451 } 2452 ret = IRQ_HANDLED; 2453 2454 ice_service_task_schedule(pf); 2455 ice_irq_dynamic_ena(hw, NULL, NULL); 2456 2457 return ret; 2458 } 2459 2460 /** 2461 * ice_dis_ctrlq_interrupts - disable control queue interrupts 2462 * @hw: pointer to HW structure 2463 */ 2464 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 2465 { 2466 /* disable Admin queue Interrupt causes */ 2467 wr32(hw, PFINT_FW_CTL, 2468 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 2469 2470 /* disable Mailbox queue Interrupt causes */ 2471 wr32(hw, PFINT_MBX_CTL, 2472 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 2473 2474 /* disable Control queue Interrupt causes */ 2475 wr32(hw, PFINT_OICR_CTL, 2476 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 2477 2478 ice_flush(hw); 2479 } 2480 2481 /** 2482 * ice_free_irq_msix_misc - Unroll misc vector setup 2483 * @pf: board private structure 2484 */ 2485 static void ice_free_irq_msix_misc(struct ice_pf *pf) 2486 { 2487 struct ice_hw *hw = &pf->hw; 2488 2489 ice_dis_ctrlq_interrupts(hw); 2490 2491 /* disable OICR interrupt */ 2492 wr32(hw, PFINT_OICR_ENA, 0); 2493 ice_flush(hw); 2494 2495 if (pf->msix_entries) { 2496 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector); 2497 devm_free_irq(ice_pf_to_dev(pf), 2498 pf->msix_entries[pf->oicr_idx].vector, pf); 2499 } 2500 2501 pf->num_avail_sw_msix += 1; 2502 ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID); 2503 } 2504 2505 /** 2506 * ice_ena_ctrlq_interrupts - enable control queue interrupts 2507 * @hw: pointer to HW structure 2508 * @reg_idx: HW vector index to associate the control queue interrupts with 2509 */ 2510 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 2511 { 2512 u32 val; 2513 2514 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 2515 PFINT_OICR_CTL_CAUSE_ENA_M); 2516 wr32(hw, PFINT_OICR_CTL, val); 2517 2518 /* enable Admin queue Interrupt causes */ 2519 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 2520 PFINT_FW_CTL_CAUSE_ENA_M); 2521 wr32(hw, PFINT_FW_CTL, val); 2522 2523 /* enable Mailbox queue Interrupt causes */ 2524 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 2525 PFINT_MBX_CTL_CAUSE_ENA_M); 2526 wr32(hw, PFINT_MBX_CTL, val); 2527 2528 ice_flush(hw); 2529 } 2530 2531 /** 2532 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 2533 * @pf: board private structure 2534 * 2535 * This sets up the handler for MSIX 0, which is used to manage the 2536 * non-queue interrupts, e.g. AdminQ and errors. This is not used 2537 * when in MSI or Legacy interrupt mode. 2538 */ 2539 static int ice_req_irq_msix_misc(struct ice_pf *pf) 2540 { 2541 struct device *dev = ice_pf_to_dev(pf); 2542 struct ice_hw *hw = &pf->hw; 2543 int oicr_idx, err = 0; 2544 2545 if (!pf->int_name[0]) 2546 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 2547 dev_driver_string(dev), dev_name(dev)); 2548 2549 /* Do not request IRQ but do enable OICR interrupt since settings are 2550 * lost during reset. Note that this function is called only during 2551 * rebuild path and not while reset is in progress. 2552 */ 2553 if (ice_is_reset_in_progress(pf->state)) 2554 goto skip_req_irq; 2555 2556 /* reserve one vector in irq_tracker for misc interrupts */ 2557 oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2558 if (oicr_idx < 0) 2559 return oicr_idx; 2560 2561 pf->num_avail_sw_msix -= 1; 2562 pf->oicr_idx = (u16)oicr_idx; 2563 2564 err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, 2565 ice_misc_intr, 0, pf->int_name, pf); 2566 if (err) { 2567 dev_err(dev, "devm_request_irq for %s failed: %d\n", 2568 pf->int_name, err); 2569 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2570 pf->num_avail_sw_msix += 1; 2571 return err; 2572 } 2573 2574 skip_req_irq: 2575 ice_ena_misc_vector(pf); 2576 2577 ice_ena_ctrlq_interrupts(hw, pf->oicr_idx); 2578 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx), 2579 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 2580 2581 ice_flush(hw); 2582 ice_irq_dynamic_ena(hw, NULL, NULL); 2583 2584 return 0; 2585 } 2586 2587 /** 2588 * ice_napi_add - register NAPI handler for the VSI 2589 * @vsi: VSI for which NAPI handler is to be registered 2590 * 2591 * This function is only called in the driver's load path. Registering the NAPI 2592 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 2593 * reset/rebuild, etc.) 2594 */ 2595 static void ice_napi_add(struct ice_vsi *vsi) 2596 { 2597 int v_idx; 2598 2599 if (!vsi->netdev) 2600 return; 2601 2602 ice_for_each_q_vector(vsi, v_idx) 2603 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 2604 ice_napi_poll, NAPI_POLL_WEIGHT); 2605 } 2606 2607 /** 2608 * ice_set_ops - set netdev and ethtools ops for the given netdev 2609 * @netdev: netdev instance 2610 */ 2611 static void ice_set_ops(struct net_device *netdev) 2612 { 2613 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2614 2615 if (ice_is_safe_mode(pf)) { 2616 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 2617 ice_set_ethtool_safe_mode_ops(netdev); 2618 return; 2619 } 2620 2621 netdev->netdev_ops = &ice_netdev_ops; 2622 ice_set_ethtool_ops(netdev); 2623 } 2624 2625 /** 2626 * ice_set_netdev_features - set features for the given netdev 2627 * @netdev: netdev instance 2628 */ 2629 static void ice_set_netdev_features(struct net_device *netdev) 2630 { 2631 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2632 netdev_features_t csumo_features; 2633 netdev_features_t vlano_features; 2634 netdev_features_t dflt_features; 2635 netdev_features_t tso_features; 2636 2637 if (ice_is_safe_mode(pf)) { 2638 /* safe mode */ 2639 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 2640 netdev->hw_features = netdev->features; 2641 return; 2642 } 2643 2644 dflt_features = NETIF_F_SG | 2645 NETIF_F_HIGHDMA | 2646 NETIF_F_NTUPLE | 2647 NETIF_F_RXHASH; 2648 2649 csumo_features = NETIF_F_RXCSUM | 2650 NETIF_F_IP_CSUM | 2651 NETIF_F_SCTP_CRC | 2652 NETIF_F_IPV6_CSUM; 2653 2654 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 2655 NETIF_F_HW_VLAN_CTAG_TX | 2656 NETIF_F_HW_VLAN_CTAG_RX; 2657 2658 tso_features = NETIF_F_TSO | 2659 NETIF_F_TSO_ECN | 2660 NETIF_F_TSO6 | 2661 NETIF_F_GSO_GRE | 2662 NETIF_F_GSO_UDP_TUNNEL | 2663 NETIF_F_GSO_GRE_CSUM | 2664 NETIF_F_GSO_UDP_TUNNEL_CSUM | 2665 NETIF_F_GSO_PARTIAL | 2666 NETIF_F_GSO_IPXIP4 | 2667 NETIF_F_GSO_IPXIP6 | 2668 NETIF_F_GSO_UDP_L4; 2669 2670 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 2671 NETIF_F_GSO_GRE_CSUM; 2672 /* set features that user can change */ 2673 netdev->hw_features = dflt_features | csumo_features | 2674 vlano_features | tso_features; 2675 2676 /* add support for HW_CSUM on packets with MPLS header */ 2677 netdev->mpls_features = NETIF_F_HW_CSUM; 2678 2679 /* enable features */ 2680 netdev->features |= netdev->hw_features; 2681 /* encap and VLAN devices inherit default, csumo and tso features */ 2682 netdev->hw_enc_features |= dflt_features | csumo_features | 2683 tso_features; 2684 netdev->vlan_features |= dflt_features | csumo_features | 2685 tso_features; 2686 } 2687 2688 /** 2689 * ice_cfg_netdev - Allocate, configure and register a netdev 2690 * @vsi: the VSI associated with the new netdev 2691 * 2692 * Returns 0 on success, negative value on failure 2693 */ 2694 static int ice_cfg_netdev(struct ice_vsi *vsi) 2695 { 2696 struct ice_pf *pf = vsi->back; 2697 struct ice_netdev_priv *np; 2698 struct net_device *netdev; 2699 u8 mac_addr[ETH_ALEN]; 2700 int err; 2701 2702 err = ice_devlink_create_port(pf); 2703 if (err) 2704 return err; 2705 2706 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 2707 vsi->alloc_rxq); 2708 if (!netdev) { 2709 err = -ENOMEM; 2710 goto err_destroy_devlink_port; 2711 } 2712 2713 vsi->netdev = netdev; 2714 np = netdev_priv(netdev); 2715 np->vsi = vsi; 2716 2717 ice_set_netdev_features(netdev); 2718 2719 ice_set_ops(netdev); 2720 2721 if (vsi->type == ICE_VSI_PF) { 2722 SET_NETDEV_DEV(netdev, ice_pf_to_dev(pf)); 2723 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 2724 ether_addr_copy(netdev->dev_addr, mac_addr); 2725 ether_addr_copy(netdev->perm_addr, mac_addr); 2726 } 2727 2728 netdev->priv_flags |= IFF_UNICAST_FLT; 2729 2730 /* Setup netdev TC information */ 2731 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 2732 2733 /* setup watchdog timeout value to be 5 second */ 2734 netdev->watchdog_timeo = 5 * HZ; 2735 2736 netdev->min_mtu = ETH_MIN_MTU; 2737 netdev->max_mtu = ICE_MAX_MTU; 2738 2739 err = register_netdev(vsi->netdev); 2740 if (err) 2741 goto err_free_netdev; 2742 2743 devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev); 2744 2745 netif_carrier_off(vsi->netdev); 2746 2747 /* make sure transmit queues start off as stopped */ 2748 netif_tx_stop_all_queues(vsi->netdev); 2749 2750 return 0; 2751 2752 err_free_netdev: 2753 free_netdev(vsi->netdev); 2754 vsi->netdev = NULL; 2755 err_destroy_devlink_port: 2756 ice_devlink_destroy_port(pf); 2757 return err; 2758 } 2759 2760 /** 2761 * ice_fill_rss_lut - Fill the RSS lookup table with default values 2762 * @lut: Lookup table 2763 * @rss_table_size: Lookup table size 2764 * @rss_size: Range of queue number for hashing 2765 */ 2766 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 2767 { 2768 u16 i; 2769 2770 for (i = 0; i < rss_table_size; i++) 2771 lut[i] = i % rss_size; 2772 } 2773 2774 /** 2775 * ice_pf_vsi_setup - Set up a PF VSI 2776 * @pf: board private structure 2777 * @pi: pointer to the port_info instance 2778 * 2779 * Returns pointer to the successfully allocated VSI software struct 2780 * on success, otherwise returns NULL on failure. 2781 */ 2782 static struct ice_vsi * 2783 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2784 { 2785 return ice_vsi_setup(pf, pi, ICE_VSI_PF, ICE_INVAL_VFID); 2786 } 2787 2788 /** 2789 * ice_ctrl_vsi_setup - Set up a control VSI 2790 * @pf: board private structure 2791 * @pi: pointer to the port_info instance 2792 * 2793 * Returns pointer to the successfully allocated VSI software struct 2794 * on success, otherwise returns NULL on failure. 2795 */ 2796 static struct ice_vsi * 2797 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2798 { 2799 return ice_vsi_setup(pf, pi, ICE_VSI_CTRL, ICE_INVAL_VFID); 2800 } 2801 2802 /** 2803 * ice_lb_vsi_setup - Set up a loopback VSI 2804 * @pf: board private structure 2805 * @pi: pointer to the port_info instance 2806 * 2807 * Returns pointer to the successfully allocated VSI software struct 2808 * on success, otherwise returns NULL on failure. 2809 */ 2810 struct ice_vsi * 2811 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2812 { 2813 return ice_vsi_setup(pf, pi, ICE_VSI_LB, ICE_INVAL_VFID); 2814 } 2815 2816 /** 2817 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 2818 * @netdev: network interface to be adjusted 2819 * @proto: unused protocol 2820 * @vid: VLAN ID to be added 2821 * 2822 * net_device_ops implementation for adding VLAN IDs 2823 */ 2824 static int 2825 ice_vlan_rx_add_vid(struct net_device *netdev, __always_unused __be16 proto, 2826 u16 vid) 2827 { 2828 struct ice_netdev_priv *np = netdev_priv(netdev); 2829 struct ice_vsi *vsi = np->vsi; 2830 int ret; 2831 2832 if (vid >= VLAN_N_VID) { 2833 netdev_err(netdev, "VLAN id requested %d is out of range %d\n", 2834 vid, VLAN_N_VID); 2835 return -EINVAL; 2836 } 2837 2838 if (vsi->info.pvid) 2839 return -EINVAL; 2840 2841 /* VLAN 0 is added by default during load/reset */ 2842 if (!vid) 2843 return 0; 2844 2845 /* Enable VLAN pruning when a VLAN other than 0 is added */ 2846 if (!ice_vsi_is_vlan_pruning_ena(vsi)) { 2847 ret = ice_cfg_vlan_pruning(vsi, true, false); 2848 if (ret) 2849 return ret; 2850 } 2851 2852 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 2853 * packets aren't pruned by the device's internal switch on Rx 2854 */ 2855 ret = ice_vsi_add_vlan(vsi, vid, ICE_FWD_TO_VSI); 2856 if (!ret) { 2857 vsi->vlan_ena = true; 2858 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 2859 } 2860 2861 return ret; 2862 } 2863 2864 /** 2865 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 2866 * @netdev: network interface to be adjusted 2867 * @proto: unused protocol 2868 * @vid: VLAN ID to be removed 2869 * 2870 * net_device_ops implementation for removing VLAN IDs 2871 */ 2872 static int 2873 ice_vlan_rx_kill_vid(struct net_device *netdev, __always_unused __be16 proto, 2874 u16 vid) 2875 { 2876 struct ice_netdev_priv *np = netdev_priv(netdev); 2877 struct ice_vsi *vsi = np->vsi; 2878 int ret; 2879 2880 if (vsi->info.pvid) 2881 return -EINVAL; 2882 2883 /* don't allow removal of VLAN 0 */ 2884 if (!vid) 2885 return 0; 2886 2887 /* Make sure ice_vsi_kill_vlan is successful before updating VLAN 2888 * information 2889 */ 2890 ret = ice_vsi_kill_vlan(vsi, vid); 2891 if (ret) 2892 return ret; 2893 2894 /* Disable pruning when VLAN 0 is the only VLAN rule */ 2895 if (vsi->num_vlan == 1 && ice_vsi_is_vlan_pruning_ena(vsi)) 2896 ret = ice_cfg_vlan_pruning(vsi, false, false); 2897 2898 vsi->vlan_ena = false; 2899 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 2900 return ret; 2901 } 2902 2903 /** 2904 * ice_setup_pf_sw - Setup the HW switch on startup or after reset 2905 * @pf: board private structure 2906 * 2907 * Returns 0 on success, negative value on failure 2908 */ 2909 static int ice_setup_pf_sw(struct ice_pf *pf) 2910 { 2911 struct ice_vsi *vsi; 2912 int status = 0; 2913 2914 if (ice_is_reset_in_progress(pf->state)) 2915 return -EBUSY; 2916 2917 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 2918 if (!vsi) { 2919 status = -ENOMEM; 2920 goto unroll_vsi_setup; 2921 } 2922 2923 status = ice_cfg_netdev(vsi); 2924 if (status) { 2925 status = -ENODEV; 2926 goto unroll_vsi_setup; 2927 } 2928 /* netdev has to be configured before setting frame size */ 2929 ice_vsi_cfg_frame_size(vsi); 2930 2931 /* Setup DCB netlink interface */ 2932 ice_dcbnl_setup(vsi); 2933 2934 /* registering the NAPI handler requires both the queues and 2935 * netdev to be created, which are done in ice_pf_vsi_setup() 2936 * and ice_cfg_netdev() respectively 2937 */ 2938 ice_napi_add(vsi); 2939 2940 status = ice_set_cpu_rx_rmap(vsi); 2941 if (status) { 2942 dev_err(ice_pf_to_dev(pf), "Failed to set CPU Rx map VSI %d error %d\n", 2943 vsi->vsi_num, status); 2944 status = -EINVAL; 2945 goto unroll_napi_add; 2946 } 2947 status = ice_init_mac_fltr(pf); 2948 if (status) 2949 goto free_cpu_rx_map; 2950 2951 return status; 2952 2953 free_cpu_rx_map: 2954 ice_free_cpu_rx_rmap(vsi); 2955 2956 unroll_napi_add: 2957 if (vsi) { 2958 ice_napi_del(vsi); 2959 if (vsi->netdev) { 2960 if (vsi->netdev->reg_state == NETREG_REGISTERED) 2961 unregister_netdev(vsi->netdev); 2962 free_netdev(vsi->netdev); 2963 vsi->netdev = NULL; 2964 } 2965 } 2966 2967 unroll_vsi_setup: 2968 if (vsi) { 2969 ice_vsi_free_q_vectors(vsi); 2970 ice_vsi_delete(vsi); 2971 ice_vsi_put_qs(vsi); 2972 ice_vsi_clear(vsi); 2973 } 2974 return status; 2975 } 2976 2977 /** 2978 * ice_get_avail_q_count - Get count of queues in use 2979 * @pf_qmap: bitmap to get queue use count from 2980 * @lock: pointer to a mutex that protects access to pf_qmap 2981 * @size: size of the bitmap 2982 */ 2983 static u16 2984 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 2985 { 2986 unsigned long bit; 2987 u16 count = 0; 2988 2989 mutex_lock(lock); 2990 for_each_clear_bit(bit, pf_qmap, size) 2991 count++; 2992 mutex_unlock(lock); 2993 2994 return count; 2995 } 2996 2997 /** 2998 * ice_get_avail_txq_count - Get count of Tx queues in use 2999 * @pf: pointer to an ice_pf instance 3000 */ 3001 u16 ice_get_avail_txq_count(struct ice_pf *pf) 3002 { 3003 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3004 pf->max_pf_txqs); 3005 } 3006 3007 /** 3008 * ice_get_avail_rxq_count - Get count of Rx queues in use 3009 * @pf: pointer to an ice_pf instance 3010 */ 3011 u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3012 { 3013 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3014 pf->max_pf_rxqs); 3015 } 3016 3017 /** 3018 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3019 * @pf: board private structure to initialize 3020 */ 3021 static void ice_deinit_pf(struct ice_pf *pf) 3022 { 3023 ice_service_task_stop(pf); 3024 mutex_destroy(&pf->sw_mutex); 3025 mutex_destroy(&pf->tc_mutex); 3026 mutex_destroy(&pf->avail_q_mutex); 3027 3028 if (pf->avail_txqs) { 3029 bitmap_free(pf->avail_txqs); 3030 pf->avail_txqs = NULL; 3031 } 3032 3033 if (pf->avail_rxqs) { 3034 bitmap_free(pf->avail_rxqs); 3035 pf->avail_rxqs = NULL; 3036 } 3037 } 3038 3039 /** 3040 * ice_set_pf_caps - set PFs capability flags 3041 * @pf: pointer to the PF instance 3042 */ 3043 static void ice_set_pf_caps(struct ice_pf *pf) 3044 { 3045 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3046 3047 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3048 if (func_caps->common_cap.dcb) 3049 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3050 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3051 if (func_caps->common_cap.sr_iov_1_1) { 3052 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3053 pf->num_vfs_supported = min_t(int, func_caps->num_allocd_vfs, 3054 ICE_MAX_VF_COUNT); 3055 } 3056 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 3057 if (func_caps->common_cap.rss_table_size) 3058 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 3059 3060 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 3061 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 3062 u16 unused; 3063 3064 /* ctrl_vsi_idx will be set to a valid value when flow director 3065 * is setup by ice_init_fdir 3066 */ 3067 pf->ctrl_vsi_idx = ICE_NO_VSI; 3068 set_bit(ICE_FLAG_FD_ENA, pf->flags); 3069 /* force guaranteed filter pool for PF */ 3070 ice_alloc_fd_guar_item(&pf->hw, &unused, 3071 func_caps->fd_fltr_guar); 3072 /* force shared filter pool for PF */ 3073 ice_alloc_fd_shrd_item(&pf->hw, &unused, 3074 func_caps->fd_fltr_best_effort); 3075 } 3076 3077 pf->max_pf_txqs = func_caps->common_cap.num_txq; 3078 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 3079 } 3080 3081 /** 3082 * ice_init_pf - Initialize general software structures (struct ice_pf) 3083 * @pf: board private structure to initialize 3084 */ 3085 static int ice_init_pf(struct ice_pf *pf) 3086 { 3087 ice_set_pf_caps(pf); 3088 3089 mutex_init(&pf->sw_mutex); 3090 mutex_init(&pf->tc_mutex); 3091 3092 /* setup service timer and periodic service task */ 3093 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 3094 pf->serv_tmr_period = HZ; 3095 INIT_WORK(&pf->serv_task, ice_service_task); 3096 clear_bit(__ICE_SERVICE_SCHED, pf->state); 3097 3098 mutex_init(&pf->avail_q_mutex); 3099 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 3100 if (!pf->avail_txqs) 3101 return -ENOMEM; 3102 3103 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 3104 if (!pf->avail_rxqs) { 3105 devm_kfree(ice_pf_to_dev(pf), pf->avail_txqs); 3106 pf->avail_txqs = NULL; 3107 return -ENOMEM; 3108 } 3109 3110 return 0; 3111 } 3112 3113 /** 3114 * ice_ena_msix_range - Request a range of MSIX vectors from the OS 3115 * @pf: board private structure 3116 * 3117 * compute the number of MSIX vectors required (v_budget) and request from 3118 * the OS. Return the number of vectors reserved or negative on failure 3119 */ 3120 static int ice_ena_msix_range(struct ice_pf *pf) 3121 { 3122 struct device *dev = ice_pf_to_dev(pf); 3123 int v_left, v_actual, v_budget = 0; 3124 int needed, err, i; 3125 3126 v_left = pf->hw.func_caps.common_cap.num_msix_vectors; 3127 3128 /* reserve one vector for miscellaneous handler */ 3129 needed = 1; 3130 if (v_left < needed) 3131 goto no_hw_vecs_left_err; 3132 v_budget += needed; 3133 v_left -= needed; 3134 3135 /* reserve vectors for LAN traffic */ 3136 needed = min_t(int, num_online_cpus(), v_left); 3137 if (v_left < needed) 3138 goto no_hw_vecs_left_err; 3139 pf->num_lan_msix = needed; 3140 v_budget += needed; 3141 v_left -= needed; 3142 3143 /* reserve one vector for flow director */ 3144 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 3145 needed = ICE_FDIR_MSIX; 3146 if (v_left < needed) 3147 goto no_hw_vecs_left_err; 3148 v_budget += needed; 3149 v_left -= needed; 3150 } 3151 3152 pf->msix_entries = devm_kcalloc(dev, v_budget, 3153 sizeof(*pf->msix_entries), GFP_KERNEL); 3154 3155 if (!pf->msix_entries) { 3156 err = -ENOMEM; 3157 goto exit_err; 3158 } 3159 3160 for (i = 0; i < v_budget; i++) 3161 pf->msix_entries[i].entry = i; 3162 3163 /* actually reserve the vectors */ 3164 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, 3165 ICE_MIN_MSIX, v_budget); 3166 3167 if (v_actual < 0) { 3168 dev_err(dev, "unable to reserve MSI-X vectors\n"); 3169 err = v_actual; 3170 goto msix_err; 3171 } 3172 3173 if (v_actual < v_budget) { 3174 dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", 3175 v_budget, v_actual); 3176 /* 2 vectors each for LAN and RDMA (traffic + OICR), one for flow director */ 3177 #define ICE_MIN_LAN_VECS 2 3178 #define ICE_MIN_RDMA_VECS 2 3179 #define ICE_MIN_VECS (ICE_MIN_LAN_VECS + ICE_MIN_RDMA_VECS + 1) 3180 3181 if (v_actual < ICE_MIN_LAN_VECS) { 3182 /* error if we can't get minimum vectors */ 3183 pci_disable_msix(pf->pdev); 3184 err = -ERANGE; 3185 goto msix_err; 3186 } else { 3187 pf->num_lan_msix = ICE_MIN_LAN_VECS; 3188 } 3189 } 3190 3191 return v_actual; 3192 3193 msix_err: 3194 devm_kfree(dev, pf->msix_entries); 3195 goto exit_err; 3196 3197 no_hw_vecs_left_err: 3198 dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n", 3199 needed, v_left); 3200 err = -ERANGE; 3201 exit_err: 3202 pf->num_lan_msix = 0; 3203 return err; 3204 } 3205 3206 /** 3207 * ice_dis_msix - Disable MSI-X interrupt setup in OS 3208 * @pf: board private structure 3209 */ 3210 static void ice_dis_msix(struct ice_pf *pf) 3211 { 3212 pci_disable_msix(pf->pdev); 3213 devm_kfree(ice_pf_to_dev(pf), pf->msix_entries); 3214 pf->msix_entries = NULL; 3215 } 3216 3217 /** 3218 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme 3219 * @pf: board private structure 3220 */ 3221 static void ice_clear_interrupt_scheme(struct ice_pf *pf) 3222 { 3223 ice_dis_msix(pf); 3224 3225 if (pf->irq_tracker) { 3226 devm_kfree(ice_pf_to_dev(pf), pf->irq_tracker); 3227 pf->irq_tracker = NULL; 3228 } 3229 } 3230 3231 /** 3232 * ice_init_interrupt_scheme - Determine proper interrupt scheme 3233 * @pf: board private structure to initialize 3234 */ 3235 static int ice_init_interrupt_scheme(struct ice_pf *pf) 3236 { 3237 int vectors; 3238 3239 vectors = ice_ena_msix_range(pf); 3240 3241 if (vectors < 0) 3242 return vectors; 3243 3244 /* set up vector assignment tracking */ 3245 pf->irq_tracker = 3246 devm_kzalloc(ice_pf_to_dev(pf), sizeof(*pf->irq_tracker) + 3247 (sizeof(u16) * vectors), GFP_KERNEL); 3248 if (!pf->irq_tracker) { 3249 ice_dis_msix(pf); 3250 return -ENOMEM; 3251 } 3252 3253 /* populate SW interrupts pool with number of OS granted IRQs. */ 3254 pf->num_avail_sw_msix = (u16)vectors; 3255 pf->irq_tracker->num_entries = (u16)vectors; 3256 pf->irq_tracker->end = pf->irq_tracker->num_entries; 3257 3258 return 0; 3259 } 3260 3261 /** 3262 * ice_is_wol_supported - get NVM state of WoL 3263 * @pf: board private structure 3264 * 3265 * Check if WoL is supported based on the HW configuration. 3266 * Returns true if NVM supports and enables WoL for this port, false otherwise 3267 */ 3268 bool ice_is_wol_supported(struct ice_pf *pf) 3269 { 3270 struct ice_hw *hw = &pf->hw; 3271 u16 wol_ctrl; 3272 3273 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 3274 * word) indicates WoL is not supported on the corresponding PF ID. 3275 */ 3276 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 3277 return false; 3278 3279 return !(BIT(hw->pf_id) & wol_ctrl); 3280 } 3281 3282 /** 3283 * ice_vsi_recfg_qs - Change the number of queues on a VSI 3284 * @vsi: VSI being changed 3285 * @new_rx: new number of Rx queues 3286 * @new_tx: new number of Tx queues 3287 * 3288 * Only change the number of queues if new_tx, or new_rx is non-0. 3289 * 3290 * Returns 0 on success. 3291 */ 3292 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx) 3293 { 3294 struct ice_pf *pf = vsi->back; 3295 int err = 0, timeout = 50; 3296 3297 if (!new_rx && !new_tx) 3298 return -EINVAL; 3299 3300 while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) { 3301 timeout--; 3302 if (!timeout) 3303 return -EBUSY; 3304 usleep_range(1000, 2000); 3305 } 3306 3307 if (new_tx) 3308 vsi->req_txq = (u16)new_tx; 3309 if (new_rx) 3310 vsi->req_rxq = (u16)new_rx; 3311 3312 /* set for the next time the netdev is started */ 3313 if (!netif_running(vsi->netdev)) { 3314 ice_vsi_rebuild(vsi, false); 3315 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 3316 goto done; 3317 } 3318 3319 ice_vsi_close(vsi); 3320 ice_vsi_rebuild(vsi, false); 3321 ice_pf_dcb_recfg(pf); 3322 ice_vsi_open(vsi); 3323 done: 3324 clear_bit(__ICE_CFG_BUSY, pf->state); 3325 return err; 3326 } 3327 3328 /** 3329 * ice_log_pkg_init - log result of DDP package load 3330 * @hw: pointer to hardware info 3331 * @status: status of package load 3332 */ 3333 static void 3334 ice_log_pkg_init(struct ice_hw *hw, enum ice_status *status) 3335 { 3336 struct ice_pf *pf = (struct ice_pf *)hw->back; 3337 struct device *dev = ice_pf_to_dev(pf); 3338 3339 switch (*status) { 3340 case ICE_SUCCESS: 3341 /* The package download AdminQ command returned success because 3342 * this download succeeded or ICE_ERR_AQ_NO_WORK since there is 3343 * already a package loaded on the device. 3344 */ 3345 if (hw->pkg_ver.major == hw->active_pkg_ver.major && 3346 hw->pkg_ver.minor == hw->active_pkg_ver.minor && 3347 hw->pkg_ver.update == hw->active_pkg_ver.update && 3348 hw->pkg_ver.draft == hw->active_pkg_ver.draft && 3349 !memcmp(hw->pkg_name, hw->active_pkg_name, 3350 sizeof(hw->pkg_name))) { 3351 if (hw->pkg_dwnld_status == ICE_AQ_RC_EEXIST) 3352 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 3353 hw->active_pkg_name, 3354 hw->active_pkg_ver.major, 3355 hw->active_pkg_ver.minor, 3356 hw->active_pkg_ver.update, 3357 hw->active_pkg_ver.draft); 3358 else 3359 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 3360 hw->active_pkg_name, 3361 hw->active_pkg_ver.major, 3362 hw->active_pkg_ver.minor, 3363 hw->active_pkg_ver.update, 3364 hw->active_pkg_ver.draft); 3365 } else if (hw->active_pkg_ver.major != ICE_PKG_SUPP_VER_MAJ || 3366 hw->active_pkg_ver.minor != ICE_PKG_SUPP_VER_MNR) { 3367 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 3368 hw->active_pkg_name, 3369 hw->active_pkg_ver.major, 3370 hw->active_pkg_ver.minor, 3371 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3372 *status = ICE_ERR_NOT_SUPPORTED; 3373 } else if (hw->active_pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3374 hw->active_pkg_ver.minor == ICE_PKG_SUPP_VER_MNR) { 3375 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 3376 hw->active_pkg_name, 3377 hw->active_pkg_ver.major, 3378 hw->active_pkg_ver.minor, 3379 hw->active_pkg_ver.update, 3380 hw->active_pkg_ver.draft, 3381 hw->pkg_name, 3382 hw->pkg_ver.major, 3383 hw->pkg_ver.minor, 3384 hw->pkg_ver.update, 3385 hw->pkg_ver.draft); 3386 } else { 3387 dev_err(dev, "An unknown error occurred when loading the DDP package, please reboot the system. If the problem persists, update the NVM. Entering Safe Mode.\n"); 3388 *status = ICE_ERR_NOT_SUPPORTED; 3389 } 3390 break; 3391 case ICE_ERR_FW_DDP_MISMATCH: 3392 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 3393 break; 3394 case ICE_ERR_BUF_TOO_SHORT: 3395 case ICE_ERR_CFG: 3396 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 3397 break; 3398 case ICE_ERR_NOT_SUPPORTED: 3399 /* Package File version not supported */ 3400 if (hw->pkg_ver.major > ICE_PKG_SUPP_VER_MAJ || 3401 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3402 hw->pkg_ver.minor > ICE_PKG_SUPP_VER_MNR)) 3403 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 3404 else if (hw->pkg_ver.major < ICE_PKG_SUPP_VER_MAJ || 3405 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3406 hw->pkg_ver.minor < ICE_PKG_SUPP_VER_MNR)) 3407 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 3408 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3409 break; 3410 case ICE_ERR_AQ_ERROR: 3411 switch (hw->pkg_dwnld_status) { 3412 case ICE_AQ_RC_ENOSEC: 3413 case ICE_AQ_RC_EBADSIG: 3414 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 3415 return; 3416 case ICE_AQ_RC_ESVN: 3417 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 3418 return; 3419 case ICE_AQ_RC_EBADMAN: 3420 case ICE_AQ_RC_EBADBUF: 3421 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 3422 /* poll for reset to complete */ 3423 if (ice_check_reset(hw)) 3424 dev_err(dev, "Error resetting device. Please reload the driver\n"); 3425 return; 3426 default: 3427 break; 3428 } 3429 fallthrough; 3430 default: 3431 dev_err(dev, "An unknown error (%d) occurred when loading the DDP package. Entering Safe Mode.\n", 3432 *status); 3433 break; 3434 } 3435 } 3436 3437 /** 3438 * ice_load_pkg - load/reload the DDP Package file 3439 * @firmware: firmware structure when firmware requested or NULL for reload 3440 * @pf: pointer to the PF instance 3441 * 3442 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 3443 * initialize HW tables. 3444 */ 3445 static void 3446 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 3447 { 3448 enum ice_status status = ICE_ERR_PARAM; 3449 struct device *dev = ice_pf_to_dev(pf); 3450 struct ice_hw *hw = &pf->hw; 3451 3452 /* Load DDP Package */ 3453 if (firmware && !hw->pkg_copy) { 3454 status = ice_copy_and_init_pkg(hw, firmware->data, 3455 firmware->size); 3456 ice_log_pkg_init(hw, &status); 3457 } else if (!firmware && hw->pkg_copy) { 3458 /* Reload package during rebuild after CORER/GLOBR reset */ 3459 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 3460 ice_log_pkg_init(hw, &status); 3461 } else { 3462 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 3463 } 3464 3465 if (status) { 3466 /* Safe Mode */ 3467 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3468 return; 3469 } 3470 3471 /* Successful download package is the precondition for advanced 3472 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 3473 */ 3474 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3475 } 3476 3477 /** 3478 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 3479 * @pf: pointer to the PF structure 3480 * 3481 * There is no error returned here because the driver should be able to handle 3482 * 128 Byte cache lines, so we only print a warning in case issues are seen, 3483 * specifically with Tx. 3484 */ 3485 static void ice_verify_cacheline_size(struct ice_pf *pf) 3486 { 3487 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 3488 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 3489 ICE_CACHE_LINE_BYTES); 3490 } 3491 3492 /** 3493 * ice_send_version - update firmware with driver version 3494 * @pf: PF struct 3495 * 3496 * Returns ICE_SUCCESS on success, else error code 3497 */ 3498 static enum ice_status ice_send_version(struct ice_pf *pf) 3499 { 3500 struct ice_driver_ver dv; 3501 3502 dv.major_ver = 0xff; 3503 dv.minor_ver = 0xff; 3504 dv.build_ver = 0xff; 3505 dv.subbuild_ver = 0; 3506 strscpy((char *)dv.driver_string, UTS_RELEASE, 3507 sizeof(dv.driver_string)); 3508 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 3509 } 3510 3511 /** 3512 * ice_init_fdir - Initialize flow director VSI and configuration 3513 * @pf: pointer to the PF instance 3514 * 3515 * returns 0 on success, negative on error 3516 */ 3517 static int ice_init_fdir(struct ice_pf *pf) 3518 { 3519 struct device *dev = ice_pf_to_dev(pf); 3520 struct ice_vsi *ctrl_vsi; 3521 int err; 3522 3523 /* Side Band Flow Director needs to have a control VSI. 3524 * Allocate it and store it in the PF. 3525 */ 3526 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 3527 if (!ctrl_vsi) { 3528 dev_dbg(dev, "could not create control VSI\n"); 3529 return -ENOMEM; 3530 } 3531 3532 err = ice_vsi_open_ctrl(ctrl_vsi); 3533 if (err) { 3534 dev_dbg(dev, "could not open control VSI\n"); 3535 goto err_vsi_open; 3536 } 3537 3538 mutex_init(&pf->hw.fdir_fltr_lock); 3539 3540 err = ice_fdir_create_dflt_rules(pf); 3541 if (err) 3542 goto err_fdir_rule; 3543 3544 return 0; 3545 3546 err_fdir_rule: 3547 ice_fdir_release_flows(&pf->hw); 3548 ice_vsi_close(ctrl_vsi); 3549 err_vsi_open: 3550 ice_vsi_release(ctrl_vsi); 3551 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 3552 pf->vsi[pf->ctrl_vsi_idx] = NULL; 3553 pf->ctrl_vsi_idx = ICE_NO_VSI; 3554 } 3555 return err; 3556 } 3557 3558 /** 3559 * ice_get_opt_fw_name - return optional firmware file name or NULL 3560 * @pf: pointer to the PF instance 3561 */ 3562 static char *ice_get_opt_fw_name(struct ice_pf *pf) 3563 { 3564 /* Optional firmware name same as default with additional dash 3565 * followed by a EUI-64 identifier (PCIe Device Serial Number) 3566 */ 3567 struct pci_dev *pdev = pf->pdev; 3568 char *opt_fw_filename; 3569 u64 dsn; 3570 3571 /* Determine the name of the optional file using the DSN (two 3572 * dwords following the start of the DSN Capability). 3573 */ 3574 dsn = pci_get_dsn(pdev); 3575 if (!dsn) 3576 return NULL; 3577 3578 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 3579 if (!opt_fw_filename) 3580 return NULL; 3581 3582 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 3583 ICE_DDP_PKG_PATH, dsn); 3584 3585 return opt_fw_filename; 3586 } 3587 3588 /** 3589 * ice_request_fw - Device initialization routine 3590 * @pf: pointer to the PF instance 3591 */ 3592 static void ice_request_fw(struct ice_pf *pf) 3593 { 3594 char *opt_fw_filename = ice_get_opt_fw_name(pf); 3595 const struct firmware *firmware = NULL; 3596 struct device *dev = ice_pf_to_dev(pf); 3597 int err = 0; 3598 3599 /* optional device-specific DDP (if present) overrides the default DDP 3600 * package file. kernel logs a debug message if the file doesn't exist, 3601 * and warning messages for other errors. 3602 */ 3603 if (opt_fw_filename) { 3604 err = firmware_request_nowarn(&firmware, opt_fw_filename, dev); 3605 if (err) { 3606 kfree(opt_fw_filename); 3607 goto dflt_pkg_load; 3608 } 3609 3610 /* request for firmware was successful. Download to device */ 3611 ice_load_pkg(firmware, pf); 3612 kfree(opt_fw_filename); 3613 release_firmware(firmware); 3614 return; 3615 } 3616 3617 dflt_pkg_load: 3618 err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev); 3619 if (err) { 3620 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 3621 return; 3622 } 3623 3624 /* request for firmware was successful. Download to device */ 3625 ice_load_pkg(firmware, pf); 3626 release_firmware(firmware); 3627 } 3628 3629 /** 3630 * ice_print_wake_reason - show the wake up cause in the log 3631 * @pf: pointer to the PF struct 3632 */ 3633 static void ice_print_wake_reason(struct ice_pf *pf) 3634 { 3635 u32 wus = pf->wakeup_reason; 3636 const char *wake_str; 3637 3638 /* if no wake event, nothing to print */ 3639 if (!wus) 3640 return; 3641 3642 if (wus & PFPM_WUS_LNKC_M) 3643 wake_str = "Link\n"; 3644 else if (wus & PFPM_WUS_MAG_M) 3645 wake_str = "Magic Packet\n"; 3646 else if (wus & PFPM_WUS_MNG_M) 3647 wake_str = "Management\n"; 3648 else if (wus & PFPM_WUS_FW_RST_WK_M) 3649 wake_str = "Firmware Reset\n"; 3650 else 3651 wake_str = "Unknown\n"; 3652 3653 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 3654 } 3655 3656 /** 3657 * ice_probe - Device initialization routine 3658 * @pdev: PCI device information struct 3659 * @ent: entry in ice_pci_tbl 3660 * 3661 * Returns 0 on success, negative on failure 3662 */ 3663 static int 3664 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 3665 { 3666 struct device *dev = &pdev->dev; 3667 struct ice_pf *pf; 3668 struct ice_hw *hw; 3669 int err; 3670 3671 /* this driver uses devres, see 3672 * Documentation/driver-api/driver-model/devres.rst 3673 */ 3674 err = pcim_enable_device(pdev); 3675 if (err) 3676 return err; 3677 3678 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev)); 3679 if (err) { 3680 dev_err(dev, "BAR0 I/O map error %d\n", err); 3681 return err; 3682 } 3683 3684 pf = ice_allocate_pf(dev); 3685 if (!pf) 3686 return -ENOMEM; 3687 3688 /* set up for high or low DMA */ 3689 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 3690 if (err) 3691 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 3692 if (err) { 3693 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 3694 return err; 3695 } 3696 3697 pci_enable_pcie_error_reporting(pdev); 3698 pci_set_master(pdev); 3699 3700 pf->pdev = pdev; 3701 pci_set_drvdata(pdev, pf); 3702 set_bit(__ICE_DOWN, pf->state); 3703 /* Disable service task until DOWN bit is cleared */ 3704 set_bit(__ICE_SERVICE_DIS, pf->state); 3705 3706 hw = &pf->hw; 3707 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 3708 pci_save_state(pdev); 3709 3710 hw->back = pf; 3711 hw->vendor_id = pdev->vendor; 3712 hw->device_id = pdev->device; 3713 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 3714 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3715 hw->subsystem_device_id = pdev->subsystem_device; 3716 hw->bus.device = PCI_SLOT(pdev->devfn); 3717 hw->bus.func = PCI_FUNC(pdev->devfn); 3718 ice_set_ctrlq_len(hw); 3719 3720 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 3721 3722 err = ice_devlink_register(pf); 3723 if (err) { 3724 dev_err(dev, "ice_devlink_register failed: %d\n", err); 3725 goto err_exit_unroll; 3726 } 3727 3728 #ifndef CONFIG_DYNAMIC_DEBUG 3729 if (debug < -1) 3730 hw->debug_mask = debug; 3731 #endif 3732 3733 err = ice_init_hw(hw); 3734 if (err) { 3735 dev_err(dev, "ice_init_hw failed: %d\n", err); 3736 err = -EIO; 3737 goto err_exit_unroll; 3738 } 3739 3740 ice_request_fw(pf); 3741 3742 /* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be 3743 * set in pf->state, which will cause ice_is_safe_mode to return 3744 * true 3745 */ 3746 if (ice_is_safe_mode(pf)) { 3747 dev_err(dev, "Package download failed. Advanced features disabled - Device now in Safe Mode\n"); 3748 /* we already got function/device capabilities but these don't 3749 * reflect what the driver needs to do in safe mode. Instead of 3750 * adding conditional logic everywhere to ignore these 3751 * device/function capabilities, override them. 3752 */ 3753 ice_set_safe_mode_caps(hw); 3754 } 3755 3756 err = ice_init_pf(pf); 3757 if (err) { 3758 dev_err(dev, "ice_init_pf failed: %d\n", err); 3759 goto err_init_pf_unroll; 3760 } 3761 3762 ice_devlink_init_regions(pf); 3763 3764 pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; 3765 if (!pf->num_alloc_vsi) { 3766 err = -EIO; 3767 goto err_init_pf_unroll; 3768 } 3769 3770 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 3771 GFP_KERNEL); 3772 if (!pf->vsi) { 3773 err = -ENOMEM; 3774 goto err_init_pf_unroll; 3775 } 3776 3777 err = ice_init_interrupt_scheme(pf); 3778 if (err) { 3779 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 3780 err = -EIO; 3781 goto err_init_vsi_unroll; 3782 } 3783 3784 /* In case of MSIX we are going to setup the misc vector right here 3785 * to handle admin queue events etc. In case of legacy and MSI 3786 * the misc functionality and queue processing is combined in 3787 * the same vector and that gets setup at open. 3788 */ 3789 err = ice_req_irq_msix_misc(pf); 3790 if (err) { 3791 dev_err(dev, "setup of misc vector failed: %d\n", err); 3792 goto err_init_interrupt_unroll; 3793 } 3794 3795 /* create switch struct for the switch element created by FW on boot */ 3796 pf->first_sw = devm_kzalloc(dev, sizeof(*pf->first_sw), GFP_KERNEL); 3797 if (!pf->first_sw) { 3798 err = -ENOMEM; 3799 goto err_msix_misc_unroll; 3800 } 3801 3802 if (hw->evb_veb) 3803 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 3804 else 3805 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 3806 3807 pf->first_sw->pf = pf; 3808 3809 /* record the sw_id available for later use */ 3810 pf->first_sw->sw_id = hw->port_info->sw_id; 3811 3812 err = ice_setup_pf_sw(pf); 3813 if (err) { 3814 dev_err(dev, "probe failed due to setup PF switch: %d\n", err); 3815 goto err_alloc_sw_unroll; 3816 } 3817 3818 clear_bit(__ICE_SERVICE_DIS, pf->state); 3819 3820 /* tell the firmware we are up */ 3821 err = ice_send_version(pf); 3822 if (err) { 3823 dev_err(dev, "probe failed sending driver version %s. error: %d\n", 3824 UTS_RELEASE, err); 3825 goto err_alloc_sw_unroll; 3826 } 3827 3828 /* since everything is good, start the service timer */ 3829 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 3830 3831 err = ice_init_link_events(pf->hw.port_info); 3832 if (err) { 3833 dev_err(dev, "ice_init_link_events failed: %d\n", err); 3834 goto err_alloc_sw_unroll; 3835 } 3836 3837 err = ice_init_nvm_phy_type(pf->hw.port_info); 3838 if (err) { 3839 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 3840 goto err_alloc_sw_unroll; 3841 } 3842 3843 err = ice_update_link_info(pf->hw.port_info); 3844 if (err) { 3845 dev_err(dev, "ice_update_link_info failed: %d\n", err); 3846 goto err_alloc_sw_unroll; 3847 } 3848 3849 ice_init_link_dflt_override(pf->hw.port_info); 3850 3851 /* if media available, initialize PHY settings */ 3852 if (pf->hw.port_info->phy.link_info.link_info & 3853 ICE_AQ_MEDIA_AVAILABLE) { 3854 err = ice_init_phy_user_cfg(pf->hw.port_info); 3855 if (err) { 3856 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 3857 goto err_alloc_sw_unroll; 3858 } 3859 3860 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 3861 struct ice_vsi *vsi = ice_get_main_vsi(pf); 3862 3863 if (vsi) 3864 ice_configure_phy(vsi); 3865 } 3866 } else { 3867 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 3868 } 3869 3870 ice_verify_cacheline_size(pf); 3871 3872 /* Save wakeup reason register for later use */ 3873 pf->wakeup_reason = rd32(hw, PFPM_WUS); 3874 3875 /* check for a power management event */ 3876 ice_print_wake_reason(pf); 3877 3878 /* clear wake status, all bits */ 3879 wr32(hw, PFPM_WUS, U32_MAX); 3880 3881 /* Disable WoL at init, wait for user to enable */ 3882 device_set_wakeup_enable(dev, false); 3883 3884 /* If no DDP driven features have to be setup, we are done with probe */ 3885 if (ice_is_safe_mode(pf)) 3886 goto probe_done; 3887 3888 /* initialize DDP driven features */ 3889 3890 /* Note: Flow director init failure is non-fatal to load */ 3891 if (ice_init_fdir(pf)) 3892 dev_err(dev, "could not initialize flow director\n"); 3893 3894 /* Note: DCB init failure is non-fatal to load */ 3895 if (ice_init_pf_dcb(pf, false)) { 3896 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3897 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 3898 } else { 3899 ice_cfg_lldp_mib_change(&pf->hw, true); 3900 } 3901 3902 /* print PCI link speed and width */ 3903 pcie_print_link_status(pf->pdev); 3904 3905 probe_done: 3906 /* ready to go, so clear down state bit */ 3907 clear_bit(__ICE_DOWN, pf->state); 3908 return 0; 3909 3910 err_alloc_sw_unroll: 3911 ice_devlink_destroy_port(pf); 3912 set_bit(__ICE_SERVICE_DIS, pf->state); 3913 set_bit(__ICE_DOWN, pf->state); 3914 devm_kfree(dev, pf->first_sw); 3915 err_msix_misc_unroll: 3916 ice_free_irq_msix_misc(pf); 3917 err_init_interrupt_unroll: 3918 ice_clear_interrupt_scheme(pf); 3919 err_init_vsi_unroll: 3920 devm_kfree(dev, pf->vsi); 3921 err_init_pf_unroll: 3922 ice_deinit_pf(pf); 3923 ice_devlink_destroy_regions(pf); 3924 ice_deinit_hw(hw); 3925 err_exit_unroll: 3926 ice_devlink_unregister(pf); 3927 pci_disable_pcie_error_reporting(pdev); 3928 pci_disable_device(pdev); 3929 return err; 3930 } 3931 3932 /** 3933 * ice_set_wake - enable or disable Wake on LAN 3934 * @pf: pointer to the PF struct 3935 * 3936 * Simple helper for WoL control 3937 */ 3938 static void ice_set_wake(struct ice_pf *pf) 3939 { 3940 struct ice_hw *hw = &pf->hw; 3941 bool wol = pf->wol_ena; 3942 3943 /* clear wake state, otherwise new wake events won't fire */ 3944 wr32(hw, PFPM_WUS, U32_MAX); 3945 3946 /* enable / disable APM wake up, no RMW needed */ 3947 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 3948 3949 /* set magic packet filter enabled */ 3950 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 3951 } 3952 3953 /** 3954 * ice_setup_magic_mc_wake - setup device to wake on multicast magic packet 3955 * @pf: pointer to the PF struct 3956 * 3957 * Issue firmware command to enable multicast magic wake, making 3958 * sure that any locally administered address (LAA) is used for 3959 * wake, and that PF reset doesn't undo the LAA. 3960 */ 3961 static void ice_setup_mc_magic_wake(struct ice_pf *pf) 3962 { 3963 struct device *dev = ice_pf_to_dev(pf); 3964 struct ice_hw *hw = &pf->hw; 3965 enum ice_status status; 3966 u8 mac_addr[ETH_ALEN]; 3967 struct ice_vsi *vsi; 3968 u8 flags; 3969 3970 if (!pf->wol_ena) 3971 return; 3972 3973 vsi = ice_get_main_vsi(pf); 3974 if (!vsi) 3975 return; 3976 3977 /* Get current MAC address in case it's an LAA */ 3978 if (vsi->netdev) 3979 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 3980 else 3981 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 3982 3983 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 3984 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 3985 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 3986 3987 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 3988 if (status) 3989 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %s aq_err %s\n", 3990 ice_stat_str(status), 3991 ice_aq_str(hw->adminq.sq_last_status)); 3992 } 3993 3994 /** 3995 * ice_remove - Device removal routine 3996 * @pdev: PCI device information struct 3997 */ 3998 static void ice_remove(struct pci_dev *pdev) 3999 { 4000 struct ice_pf *pf = pci_get_drvdata(pdev); 4001 int i; 4002 4003 if (!pf) 4004 return; 4005 4006 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 4007 if (!ice_is_reset_in_progress(pf->state)) 4008 break; 4009 msleep(100); 4010 } 4011 4012 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 4013 set_bit(__ICE_VF_RESETS_DISABLED, pf->state); 4014 ice_free_vfs(pf); 4015 } 4016 4017 set_bit(__ICE_DOWN, pf->state); 4018 ice_service_task_stop(pf); 4019 4020 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 4021 if (!ice_is_safe_mode(pf)) 4022 ice_remove_arfs(pf); 4023 ice_setup_mc_magic_wake(pf); 4024 ice_devlink_destroy_port(pf); 4025 ice_vsi_release_all(pf); 4026 ice_set_wake(pf); 4027 ice_free_irq_msix_misc(pf); 4028 ice_for_each_vsi(pf, i) { 4029 if (!pf->vsi[i]) 4030 continue; 4031 ice_vsi_free_q_vectors(pf->vsi[i]); 4032 } 4033 ice_deinit_pf(pf); 4034 ice_devlink_destroy_regions(pf); 4035 ice_deinit_hw(&pf->hw); 4036 ice_devlink_unregister(pf); 4037 4038 /* Issue a PFR as part of the prescribed driver unload flow. Do not 4039 * do it via ice_schedule_reset() since there is no need to rebuild 4040 * and the service task is already stopped. 4041 */ 4042 ice_reset(&pf->hw, ICE_RESET_PFR); 4043 pci_wait_for_pending_transaction(pdev); 4044 ice_clear_interrupt_scheme(pf); 4045 pci_disable_pcie_error_reporting(pdev); 4046 pci_disable_device(pdev); 4047 } 4048 4049 /** 4050 * ice_shutdown - PCI callback for shutting down device 4051 * @pdev: PCI device information struct 4052 */ 4053 static void ice_shutdown(struct pci_dev *pdev) 4054 { 4055 struct ice_pf *pf = pci_get_drvdata(pdev); 4056 4057 ice_remove(pdev); 4058 4059 if (system_state == SYSTEM_POWER_OFF) { 4060 pci_wake_from_d3(pdev, pf->wol_ena); 4061 pci_set_power_state(pdev, PCI_D3hot); 4062 } 4063 } 4064 4065 #ifdef CONFIG_PM 4066 /** 4067 * ice_prepare_for_shutdown - prep for PCI shutdown 4068 * @pf: board private structure 4069 * 4070 * Inform or close all dependent features in prep for PCI device shutdown 4071 */ 4072 static void ice_prepare_for_shutdown(struct ice_pf *pf) 4073 { 4074 struct ice_hw *hw = &pf->hw; 4075 u32 v; 4076 4077 /* Notify VFs of impending reset */ 4078 if (ice_check_sq_alive(hw, &hw->mailboxq)) 4079 ice_vc_notify_reset(pf); 4080 4081 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 4082 4083 /* disable the VSIs and their queues that are not already DOWN */ 4084 ice_pf_dis_all_vsi(pf, false); 4085 4086 ice_for_each_vsi(pf, v) 4087 if (pf->vsi[v]) 4088 pf->vsi[v]->vsi_num = 0; 4089 4090 ice_shutdown_all_ctrlq(hw); 4091 } 4092 4093 /** 4094 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 4095 * @pf: board private structure to reinitialize 4096 * 4097 * This routine reinitialize interrupt scheme that was cleared during 4098 * power management suspend callback. 4099 * 4100 * This should be called during resume routine to re-allocate the q_vectors 4101 * and reacquire interrupts. 4102 */ 4103 static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 4104 { 4105 struct device *dev = ice_pf_to_dev(pf); 4106 int ret, v; 4107 4108 /* Since we clear MSIX flag during suspend, we need to 4109 * set it back during resume... 4110 */ 4111 4112 ret = ice_init_interrupt_scheme(pf); 4113 if (ret) { 4114 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 4115 return ret; 4116 } 4117 4118 /* Remap vectors and rings, after successful re-init interrupts */ 4119 ice_for_each_vsi(pf, v) { 4120 if (!pf->vsi[v]) 4121 continue; 4122 4123 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 4124 if (ret) 4125 goto err_reinit; 4126 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 4127 } 4128 4129 ret = ice_req_irq_msix_misc(pf); 4130 if (ret) { 4131 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 4132 ret); 4133 goto err_reinit; 4134 } 4135 4136 return 0; 4137 4138 err_reinit: 4139 while (v--) 4140 if (pf->vsi[v]) 4141 ice_vsi_free_q_vectors(pf->vsi[v]); 4142 4143 return ret; 4144 } 4145 4146 /** 4147 * ice_suspend 4148 * @dev: generic device information structure 4149 * 4150 * Power Management callback to quiesce the device and prepare 4151 * for D3 transition. 4152 */ 4153 static int ice_suspend(struct device *dev) 4154 { 4155 struct pci_dev *pdev = to_pci_dev(dev); 4156 struct ice_pf *pf; 4157 int disabled, v; 4158 4159 pf = pci_get_drvdata(pdev); 4160 4161 if (!ice_pf_state_is_nominal(pf)) { 4162 dev_err(dev, "Device is not ready, no need to suspend it\n"); 4163 return -EBUSY; 4164 } 4165 4166 /* Stop watchdog tasks until resume completion. 4167 * Even though it is most likely that the service task is 4168 * disabled if the device is suspended or down, the service task's 4169 * state is controlled by a different state bit, and we should 4170 * store and honor whatever state that bit is in at this point. 4171 */ 4172 disabled = ice_service_task_stop(pf); 4173 4174 /* Already suspended?, then there is nothing to do */ 4175 if (test_and_set_bit(__ICE_SUSPENDED, pf->state)) { 4176 if (!disabled) 4177 ice_service_task_restart(pf); 4178 return 0; 4179 } 4180 4181 if (test_bit(__ICE_DOWN, pf->state) || 4182 ice_is_reset_in_progress(pf->state)) { 4183 dev_err(dev, "can't suspend device in reset or already down\n"); 4184 if (!disabled) 4185 ice_service_task_restart(pf); 4186 return 0; 4187 } 4188 4189 ice_setup_mc_magic_wake(pf); 4190 4191 ice_prepare_for_shutdown(pf); 4192 4193 ice_set_wake(pf); 4194 4195 /* Free vectors, clear the interrupt scheme and release IRQs 4196 * for proper hibernation, especially with large number of CPUs. 4197 * Otherwise hibernation might fail when mapping all the vectors back 4198 * to CPU0. 4199 */ 4200 ice_free_irq_msix_misc(pf); 4201 ice_for_each_vsi(pf, v) { 4202 if (!pf->vsi[v]) 4203 continue; 4204 ice_vsi_free_q_vectors(pf->vsi[v]); 4205 } 4206 ice_clear_interrupt_scheme(pf); 4207 4208 pci_wake_from_d3(pdev, pf->wol_ena); 4209 pci_set_power_state(pdev, PCI_D3hot); 4210 return 0; 4211 } 4212 4213 /** 4214 * ice_resume - PM callback for waking up from D3 4215 * @dev: generic device information structure 4216 */ 4217 static int ice_resume(struct device *dev) 4218 { 4219 struct pci_dev *pdev = to_pci_dev(dev); 4220 enum ice_reset_req reset_type; 4221 struct ice_pf *pf; 4222 struct ice_hw *hw; 4223 int ret; 4224 4225 pci_set_power_state(pdev, PCI_D0); 4226 pci_restore_state(pdev); 4227 pci_save_state(pdev); 4228 4229 if (!pci_device_is_present(pdev)) 4230 return -ENODEV; 4231 4232 ret = pci_enable_device_mem(pdev); 4233 if (ret) { 4234 dev_err(dev, "Cannot enable device after suspend\n"); 4235 return ret; 4236 } 4237 4238 pf = pci_get_drvdata(pdev); 4239 hw = &pf->hw; 4240 4241 pf->wakeup_reason = rd32(hw, PFPM_WUS); 4242 ice_print_wake_reason(pf); 4243 4244 /* We cleared the interrupt scheme when we suspended, so we need to 4245 * restore it now to resume device functionality. 4246 */ 4247 ret = ice_reinit_interrupt_scheme(pf); 4248 if (ret) 4249 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 4250 4251 clear_bit(__ICE_DOWN, pf->state); 4252 /* Now perform PF reset and rebuild */ 4253 reset_type = ICE_RESET_PFR; 4254 /* re-enable service task for reset, but allow reset to schedule it */ 4255 clear_bit(__ICE_SERVICE_DIS, pf->state); 4256 4257 if (ice_schedule_reset(pf, reset_type)) 4258 dev_err(dev, "Reset during resume failed.\n"); 4259 4260 clear_bit(__ICE_SUSPENDED, pf->state); 4261 ice_service_task_restart(pf); 4262 4263 /* Restart the service task */ 4264 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4265 4266 return 0; 4267 } 4268 #endif /* CONFIG_PM */ 4269 4270 /** 4271 * ice_pci_err_detected - warning that PCI error has been detected 4272 * @pdev: PCI device information struct 4273 * @err: the type of PCI error 4274 * 4275 * Called to warn that something happened on the PCI bus and the error handling 4276 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 4277 */ 4278 static pci_ers_result_t 4279 ice_pci_err_detected(struct pci_dev *pdev, enum pci_channel_state err) 4280 { 4281 struct ice_pf *pf = pci_get_drvdata(pdev); 4282 4283 if (!pf) { 4284 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 4285 __func__, err); 4286 return PCI_ERS_RESULT_DISCONNECT; 4287 } 4288 4289 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4290 ice_service_task_stop(pf); 4291 4292 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4293 set_bit(__ICE_PFR_REQ, pf->state); 4294 ice_prepare_for_reset(pf); 4295 } 4296 } 4297 4298 return PCI_ERS_RESULT_NEED_RESET; 4299 } 4300 4301 /** 4302 * ice_pci_err_slot_reset - a PCI slot reset has just happened 4303 * @pdev: PCI device information struct 4304 * 4305 * Called to determine if the driver can recover from the PCI slot reset by 4306 * using a register read to determine if the device is recoverable. 4307 */ 4308 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 4309 { 4310 struct ice_pf *pf = pci_get_drvdata(pdev); 4311 pci_ers_result_t result; 4312 int err; 4313 u32 reg; 4314 4315 err = pci_enable_device_mem(pdev); 4316 if (err) { 4317 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 4318 err); 4319 result = PCI_ERS_RESULT_DISCONNECT; 4320 } else { 4321 pci_set_master(pdev); 4322 pci_restore_state(pdev); 4323 pci_save_state(pdev); 4324 pci_wake_from_d3(pdev, false); 4325 4326 /* Check for life */ 4327 reg = rd32(&pf->hw, GLGEN_RTRIG); 4328 if (!reg) 4329 result = PCI_ERS_RESULT_RECOVERED; 4330 else 4331 result = PCI_ERS_RESULT_DISCONNECT; 4332 } 4333 4334 err = pci_aer_clear_nonfatal_status(pdev); 4335 if (err) 4336 dev_dbg(&pdev->dev, "pci_aer_clear_nonfatal_status() failed, error %d\n", 4337 err); 4338 /* non-fatal, continue */ 4339 4340 return result; 4341 } 4342 4343 /** 4344 * ice_pci_err_resume - restart operations after PCI error recovery 4345 * @pdev: PCI device information struct 4346 * 4347 * Called to allow the driver to bring things back up after PCI error and/or 4348 * reset recovery have finished 4349 */ 4350 static void ice_pci_err_resume(struct pci_dev *pdev) 4351 { 4352 struct ice_pf *pf = pci_get_drvdata(pdev); 4353 4354 if (!pf) { 4355 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 4356 __func__); 4357 return; 4358 } 4359 4360 if (test_bit(__ICE_SUSPENDED, pf->state)) { 4361 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 4362 __func__); 4363 return; 4364 } 4365 4366 ice_do_reset(pf, ICE_RESET_PFR); 4367 ice_service_task_restart(pf); 4368 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4369 } 4370 4371 /** 4372 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 4373 * @pdev: PCI device information struct 4374 */ 4375 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 4376 { 4377 struct ice_pf *pf = pci_get_drvdata(pdev); 4378 4379 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4380 ice_service_task_stop(pf); 4381 4382 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4383 set_bit(__ICE_PFR_REQ, pf->state); 4384 ice_prepare_for_reset(pf); 4385 } 4386 } 4387 } 4388 4389 /** 4390 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 4391 * @pdev: PCI device information struct 4392 */ 4393 static void ice_pci_err_reset_done(struct pci_dev *pdev) 4394 { 4395 ice_pci_err_resume(pdev); 4396 } 4397 4398 /* ice_pci_tbl - PCI Device ID Table 4399 * 4400 * Wildcard entries (PCI_ANY_ID) should come last 4401 * Last entry must be all 0s 4402 * 4403 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 4404 * Class, Class Mask, private data (not used) } 4405 */ 4406 static const struct pci_device_id ice_pci_tbl[] = { 4407 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 4408 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 4409 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 4410 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 }, 4411 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 }, 4412 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 }, 4413 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 }, 4414 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 }, 4415 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 }, 4416 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 }, 4417 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 }, 4418 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 }, 4419 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 }, 4420 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 }, 4421 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 }, 4422 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 }, 4423 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 }, 4424 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 }, 4425 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 }, 4426 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 }, 4427 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 }, 4428 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 }, 4429 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 }, 4430 /* required last entry */ 4431 { 0, } 4432 }; 4433 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 4434 4435 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 4436 4437 static const struct pci_error_handlers ice_pci_err_handler = { 4438 .error_detected = ice_pci_err_detected, 4439 .slot_reset = ice_pci_err_slot_reset, 4440 .reset_prepare = ice_pci_err_reset_prepare, 4441 .reset_done = ice_pci_err_reset_done, 4442 .resume = ice_pci_err_resume 4443 }; 4444 4445 static struct pci_driver ice_driver = { 4446 .name = KBUILD_MODNAME, 4447 .id_table = ice_pci_tbl, 4448 .probe = ice_probe, 4449 .remove = ice_remove, 4450 #ifdef CONFIG_PM 4451 .driver.pm = &ice_pm_ops, 4452 #endif /* CONFIG_PM */ 4453 .shutdown = ice_shutdown, 4454 .sriov_configure = ice_sriov_configure, 4455 .err_handler = &ice_pci_err_handler 4456 }; 4457 4458 /** 4459 * ice_module_init - Driver registration routine 4460 * 4461 * ice_module_init is the first routine called when the driver is 4462 * loaded. All it does is register with the PCI subsystem. 4463 */ 4464 static int __init ice_module_init(void) 4465 { 4466 int status; 4467 4468 pr_info("%s\n", ice_driver_string); 4469 pr_info("%s\n", ice_copyright); 4470 4471 ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME); 4472 if (!ice_wq) { 4473 pr_err("Failed to create workqueue\n"); 4474 return -ENOMEM; 4475 } 4476 4477 status = pci_register_driver(&ice_driver); 4478 if (status) { 4479 pr_err("failed to register PCI driver, err %d\n", status); 4480 destroy_workqueue(ice_wq); 4481 } 4482 4483 return status; 4484 } 4485 module_init(ice_module_init); 4486 4487 /** 4488 * ice_module_exit - Driver exit cleanup routine 4489 * 4490 * ice_module_exit is called just before the driver is removed 4491 * from memory. 4492 */ 4493 static void __exit ice_module_exit(void) 4494 { 4495 pci_unregister_driver(&ice_driver); 4496 destroy_workqueue(ice_wq); 4497 pr_info("module unloaded\n"); 4498 } 4499 module_exit(ice_module_exit); 4500 4501 /** 4502 * ice_set_mac_address - NDO callback to set MAC address 4503 * @netdev: network interface device structure 4504 * @pi: pointer to an address structure 4505 * 4506 * Returns 0 on success, negative on failure 4507 */ 4508 static int ice_set_mac_address(struct net_device *netdev, void *pi) 4509 { 4510 struct ice_netdev_priv *np = netdev_priv(netdev); 4511 struct ice_vsi *vsi = np->vsi; 4512 struct ice_pf *pf = vsi->back; 4513 struct ice_hw *hw = &pf->hw; 4514 struct sockaddr *addr = pi; 4515 enum ice_status status; 4516 u8 flags = 0; 4517 int err = 0; 4518 u8 *mac; 4519 4520 mac = (u8 *)addr->sa_data; 4521 4522 if (!is_valid_ether_addr(mac)) 4523 return -EADDRNOTAVAIL; 4524 4525 if (ether_addr_equal(netdev->dev_addr, mac)) { 4526 netdev_warn(netdev, "already using mac %pM\n", mac); 4527 return 0; 4528 } 4529 4530 if (test_bit(__ICE_DOWN, pf->state) || 4531 ice_is_reset_in_progress(pf->state)) { 4532 netdev_err(netdev, "can't set mac %pM. device not ready\n", 4533 mac); 4534 return -EBUSY; 4535 } 4536 4537 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 4538 status = ice_fltr_remove_mac(vsi, netdev->dev_addr, ICE_FWD_TO_VSI); 4539 if (status && status != ICE_ERR_DOES_NOT_EXIST) { 4540 err = -EADDRNOTAVAIL; 4541 goto err_update_filters; 4542 } 4543 4544 /* Add filter for new MAC. If filter exists, just return success */ 4545 status = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 4546 if (status == ICE_ERR_ALREADY_EXISTS) { 4547 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 4548 return 0; 4549 } 4550 4551 /* error if the new filter addition failed */ 4552 if (status) 4553 err = -EADDRNOTAVAIL; 4554 4555 err_update_filters: 4556 if (err) { 4557 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 4558 mac); 4559 return err; 4560 } 4561 4562 /* change the netdev's MAC address */ 4563 memcpy(netdev->dev_addr, mac, netdev->addr_len); 4564 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 4565 netdev->dev_addr); 4566 4567 /* write new MAC address to the firmware */ 4568 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 4569 status = ice_aq_manage_mac_write(hw, mac, flags, NULL); 4570 if (status) { 4571 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %s\n", 4572 mac, ice_stat_str(status)); 4573 } 4574 return 0; 4575 } 4576 4577 /** 4578 * ice_set_rx_mode - NDO callback to set the netdev filters 4579 * @netdev: network interface device structure 4580 */ 4581 static void ice_set_rx_mode(struct net_device *netdev) 4582 { 4583 struct ice_netdev_priv *np = netdev_priv(netdev); 4584 struct ice_vsi *vsi = np->vsi; 4585 4586 if (!vsi) 4587 return; 4588 4589 /* Set the flags to synchronize filters 4590 * ndo_set_rx_mode may be triggered even without a change in netdev 4591 * flags 4592 */ 4593 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 4594 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 4595 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 4596 4597 /* schedule our worker thread which will take care of 4598 * applying the new filter changes 4599 */ 4600 ice_service_task_schedule(vsi->back); 4601 } 4602 4603 /** 4604 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 4605 * @netdev: network interface device structure 4606 * @queue_index: Queue ID 4607 * @maxrate: maximum bandwidth in Mbps 4608 */ 4609 static int 4610 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 4611 { 4612 struct ice_netdev_priv *np = netdev_priv(netdev); 4613 struct ice_vsi *vsi = np->vsi; 4614 enum ice_status status; 4615 u16 q_handle; 4616 u8 tc; 4617 4618 /* Validate maxrate requested is within permitted range */ 4619 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 4620 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 4621 maxrate, queue_index); 4622 return -EINVAL; 4623 } 4624 4625 q_handle = vsi->tx_rings[queue_index]->q_handle; 4626 tc = ice_dcb_get_tc(vsi, queue_index); 4627 4628 /* Set BW back to default, when user set maxrate to 0 */ 4629 if (!maxrate) 4630 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 4631 q_handle, ICE_MAX_BW); 4632 else 4633 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 4634 q_handle, ICE_MAX_BW, maxrate * 1000); 4635 if (status) { 4636 netdev_err(netdev, "Unable to set Tx max rate, error %s\n", 4637 ice_stat_str(status)); 4638 return -EIO; 4639 } 4640 4641 return 0; 4642 } 4643 4644 /** 4645 * ice_fdb_add - add an entry to the hardware database 4646 * @ndm: the input from the stack 4647 * @tb: pointer to array of nladdr (unused) 4648 * @dev: the net device pointer 4649 * @addr: the MAC address entry being added 4650 * @vid: VLAN ID 4651 * @flags: instructions from stack about fdb operation 4652 * @extack: netlink extended ack 4653 */ 4654 static int 4655 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 4656 struct net_device *dev, const unsigned char *addr, u16 vid, 4657 u16 flags, struct netlink_ext_ack __always_unused *extack) 4658 { 4659 int err; 4660 4661 if (vid) { 4662 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 4663 return -EINVAL; 4664 } 4665 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 4666 netdev_err(dev, "FDB only supports static addresses\n"); 4667 return -EINVAL; 4668 } 4669 4670 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 4671 err = dev_uc_add_excl(dev, addr); 4672 else if (is_multicast_ether_addr(addr)) 4673 err = dev_mc_add_excl(dev, addr); 4674 else 4675 err = -EINVAL; 4676 4677 /* Only return duplicate errors if NLM_F_EXCL is set */ 4678 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 4679 err = 0; 4680 4681 return err; 4682 } 4683 4684 /** 4685 * ice_fdb_del - delete an entry from the hardware database 4686 * @ndm: the input from the stack 4687 * @tb: pointer to array of nladdr (unused) 4688 * @dev: the net device pointer 4689 * @addr: the MAC address entry being added 4690 * @vid: VLAN ID 4691 */ 4692 static int 4693 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 4694 struct net_device *dev, const unsigned char *addr, 4695 __always_unused u16 vid) 4696 { 4697 int err; 4698 4699 if (ndm->ndm_state & NUD_PERMANENT) { 4700 netdev_err(dev, "FDB only supports static addresses\n"); 4701 return -EINVAL; 4702 } 4703 4704 if (is_unicast_ether_addr(addr)) 4705 err = dev_uc_del(dev, addr); 4706 else if (is_multicast_ether_addr(addr)) 4707 err = dev_mc_del(dev, addr); 4708 else 4709 err = -EINVAL; 4710 4711 return err; 4712 } 4713 4714 /** 4715 * ice_set_features - set the netdev feature flags 4716 * @netdev: ptr to the netdev being adjusted 4717 * @features: the feature set that the stack is suggesting 4718 */ 4719 static int 4720 ice_set_features(struct net_device *netdev, netdev_features_t features) 4721 { 4722 struct ice_netdev_priv *np = netdev_priv(netdev); 4723 struct ice_vsi *vsi = np->vsi; 4724 struct ice_pf *pf = vsi->back; 4725 int ret = 0; 4726 4727 /* Don't set any netdev advanced features with device in Safe Mode */ 4728 if (ice_is_safe_mode(vsi->back)) { 4729 dev_err(ice_pf_to_dev(vsi->back), "Device is in Safe Mode - not enabling advanced netdev features\n"); 4730 return ret; 4731 } 4732 4733 /* Do not change setting during reset */ 4734 if (ice_is_reset_in_progress(pf->state)) { 4735 dev_err(ice_pf_to_dev(vsi->back), "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 4736 return -EBUSY; 4737 } 4738 4739 /* Multiple features can be changed in one call so keep features in 4740 * separate if/else statements to guarantee each feature is checked 4741 */ 4742 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 4743 ret = ice_vsi_manage_rss_lut(vsi, true); 4744 else if (!(features & NETIF_F_RXHASH) && 4745 netdev->features & NETIF_F_RXHASH) 4746 ret = ice_vsi_manage_rss_lut(vsi, false); 4747 4748 if ((features & NETIF_F_HW_VLAN_CTAG_RX) && 4749 !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 4750 ret = ice_vsi_manage_vlan_stripping(vsi, true); 4751 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && 4752 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 4753 ret = ice_vsi_manage_vlan_stripping(vsi, false); 4754 4755 if ((features & NETIF_F_HW_VLAN_CTAG_TX) && 4756 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 4757 ret = ice_vsi_manage_vlan_insertion(vsi); 4758 else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) && 4759 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 4760 ret = ice_vsi_manage_vlan_insertion(vsi); 4761 4762 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && 4763 !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 4764 ret = ice_cfg_vlan_pruning(vsi, true, false); 4765 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && 4766 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 4767 ret = ice_cfg_vlan_pruning(vsi, false, false); 4768 4769 if ((features & NETIF_F_NTUPLE) && 4770 !(netdev->features & NETIF_F_NTUPLE)) { 4771 ice_vsi_manage_fdir(vsi, true); 4772 ice_init_arfs(vsi); 4773 } else if (!(features & NETIF_F_NTUPLE) && 4774 (netdev->features & NETIF_F_NTUPLE)) { 4775 ice_vsi_manage_fdir(vsi, false); 4776 ice_clear_arfs(vsi); 4777 } 4778 4779 return ret; 4780 } 4781 4782 /** 4783 * ice_vsi_vlan_setup - Setup VLAN offload properties on a VSI 4784 * @vsi: VSI to setup VLAN properties for 4785 */ 4786 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 4787 { 4788 int ret = 0; 4789 4790 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 4791 ret = ice_vsi_manage_vlan_stripping(vsi, true); 4792 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX) 4793 ret = ice_vsi_manage_vlan_insertion(vsi); 4794 4795 return ret; 4796 } 4797 4798 /** 4799 * ice_vsi_cfg - Setup the VSI 4800 * @vsi: the VSI being configured 4801 * 4802 * Return 0 on success and negative value on error 4803 */ 4804 int ice_vsi_cfg(struct ice_vsi *vsi) 4805 { 4806 int err; 4807 4808 if (vsi->netdev) { 4809 ice_set_rx_mode(vsi->netdev); 4810 4811 err = ice_vsi_vlan_setup(vsi); 4812 4813 if (err) 4814 return err; 4815 } 4816 ice_vsi_cfg_dcb_rings(vsi); 4817 4818 err = ice_vsi_cfg_lan_txqs(vsi); 4819 if (!err && ice_is_xdp_ena_vsi(vsi)) 4820 err = ice_vsi_cfg_xdp_txqs(vsi); 4821 if (!err) 4822 err = ice_vsi_cfg_rxqs(vsi); 4823 4824 return err; 4825 } 4826 4827 /** 4828 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4829 * @vsi: the VSI being configured 4830 */ 4831 static void ice_napi_enable_all(struct ice_vsi *vsi) 4832 { 4833 int q_idx; 4834 4835 if (!vsi->netdev) 4836 return; 4837 4838 ice_for_each_q_vector(vsi, q_idx) { 4839 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 4840 4841 if (q_vector->rx.ring || q_vector->tx.ring) 4842 napi_enable(&q_vector->napi); 4843 } 4844 } 4845 4846 /** 4847 * ice_up_complete - Finish the last steps of bringing up a connection 4848 * @vsi: The VSI being configured 4849 * 4850 * Return 0 on success and negative value on error 4851 */ 4852 static int ice_up_complete(struct ice_vsi *vsi) 4853 { 4854 struct ice_pf *pf = vsi->back; 4855 int err; 4856 4857 ice_vsi_cfg_msix(vsi); 4858 4859 /* Enable only Rx rings, Tx rings were enabled by the FW when the 4860 * Tx queue group list was configured and the context bits were 4861 * programmed using ice_vsi_cfg_txqs 4862 */ 4863 err = ice_vsi_start_all_rx_rings(vsi); 4864 if (err) 4865 return err; 4866 4867 clear_bit(__ICE_DOWN, vsi->state); 4868 ice_napi_enable_all(vsi); 4869 ice_vsi_ena_irq(vsi); 4870 4871 if (vsi->port_info && 4872 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 4873 vsi->netdev) { 4874 ice_print_link_msg(vsi, true); 4875 netif_tx_start_all_queues(vsi->netdev); 4876 netif_carrier_on(vsi->netdev); 4877 } 4878 4879 ice_service_task_schedule(pf); 4880 4881 return 0; 4882 } 4883 4884 /** 4885 * ice_up - Bring the connection back up after being down 4886 * @vsi: VSI being configured 4887 */ 4888 int ice_up(struct ice_vsi *vsi) 4889 { 4890 int err; 4891 4892 err = ice_vsi_cfg(vsi); 4893 if (!err) 4894 err = ice_up_complete(vsi); 4895 4896 return err; 4897 } 4898 4899 /** 4900 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 4901 * @ring: Tx or Rx ring to read stats from 4902 * @pkts: packets stats counter 4903 * @bytes: bytes stats counter 4904 * 4905 * This function fetches stats from the ring considering the atomic operations 4906 * that needs to be performed to read u64 values in 32 bit machine. 4907 */ 4908 static void 4909 ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts, u64 *bytes) 4910 { 4911 unsigned int start; 4912 *pkts = 0; 4913 *bytes = 0; 4914 4915 if (!ring) 4916 return; 4917 do { 4918 start = u64_stats_fetch_begin_irq(&ring->syncp); 4919 *pkts = ring->stats.pkts; 4920 *bytes = ring->stats.bytes; 4921 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4922 } 4923 4924 /** 4925 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 4926 * @vsi: the VSI to be updated 4927 * @rings: rings to work on 4928 * @count: number of rings 4929 */ 4930 static void 4931 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, struct ice_ring **rings, 4932 u16 count) 4933 { 4934 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 4935 u16 i; 4936 4937 for (i = 0; i < count; i++) { 4938 struct ice_ring *ring; 4939 u64 pkts, bytes; 4940 4941 ring = READ_ONCE(rings[i]); 4942 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 4943 vsi_stats->tx_packets += pkts; 4944 vsi_stats->tx_bytes += bytes; 4945 vsi->tx_restart += ring->tx_stats.restart_q; 4946 vsi->tx_busy += ring->tx_stats.tx_busy; 4947 vsi->tx_linearize += ring->tx_stats.tx_linearize; 4948 } 4949 } 4950 4951 /** 4952 * ice_update_vsi_ring_stats - Update VSI stats counters 4953 * @vsi: the VSI to be updated 4954 */ 4955 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 4956 { 4957 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 4958 struct ice_ring *ring; 4959 u64 pkts, bytes; 4960 int i; 4961 4962 /* reset netdev stats */ 4963 vsi_stats->tx_packets = 0; 4964 vsi_stats->tx_bytes = 0; 4965 vsi_stats->rx_packets = 0; 4966 vsi_stats->rx_bytes = 0; 4967 4968 /* reset non-netdev (extended) stats */ 4969 vsi->tx_restart = 0; 4970 vsi->tx_busy = 0; 4971 vsi->tx_linearize = 0; 4972 vsi->rx_buf_failed = 0; 4973 vsi->rx_page_failed = 0; 4974 4975 rcu_read_lock(); 4976 4977 /* update Tx rings counters */ 4978 ice_update_vsi_tx_ring_stats(vsi, vsi->tx_rings, vsi->num_txq); 4979 4980 /* update Rx rings counters */ 4981 ice_for_each_rxq(vsi, i) { 4982 ring = READ_ONCE(vsi->rx_rings[i]); 4983 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 4984 vsi_stats->rx_packets += pkts; 4985 vsi_stats->rx_bytes += bytes; 4986 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed; 4987 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; 4988 } 4989 4990 /* update XDP Tx rings counters */ 4991 if (ice_is_xdp_ena_vsi(vsi)) 4992 ice_update_vsi_tx_ring_stats(vsi, vsi->xdp_rings, 4993 vsi->num_xdp_txq); 4994 4995 rcu_read_unlock(); 4996 } 4997 4998 /** 4999 * ice_update_vsi_stats - Update VSI stats counters 5000 * @vsi: the VSI to be updated 5001 */ 5002 void ice_update_vsi_stats(struct ice_vsi *vsi) 5003 { 5004 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 5005 struct ice_eth_stats *cur_es = &vsi->eth_stats; 5006 struct ice_pf *pf = vsi->back; 5007 5008 if (test_bit(__ICE_DOWN, vsi->state) || 5009 test_bit(__ICE_CFG_BUSY, pf->state)) 5010 return; 5011 5012 /* get stats as recorded by Tx/Rx rings */ 5013 ice_update_vsi_ring_stats(vsi); 5014 5015 /* get VSI stats as recorded by the hardware */ 5016 ice_update_eth_stats(vsi); 5017 5018 cur_ns->tx_errors = cur_es->tx_errors; 5019 cur_ns->rx_dropped = cur_es->rx_discards; 5020 cur_ns->tx_dropped = cur_es->tx_discards; 5021 cur_ns->multicast = cur_es->rx_multicast; 5022 5023 /* update some more netdev stats if this is main VSI */ 5024 if (vsi->type == ICE_VSI_PF) { 5025 cur_ns->rx_crc_errors = pf->stats.crc_errors; 5026 cur_ns->rx_errors = pf->stats.crc_errors + 5027 pf->stats.illegal_bytes + 5028 pf->stats.rx_len_errors + 5029 pf->stats.rx_undersize + 5030 pf->hw_csum_rx_error + 5031 pf->stats.rx_jabber + 5032 pf->stats.rx_fragments + 5033 pf->stats.rx_oversize; 5034 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 5035 /* record drops from the port level */ 5036 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 5037 } 5038 } 5039 5040 /** 5041 * ice_update_pf_stats - Update PF port stats counters 5042 * @pf: PF whose stats needs to be updated 5043 */ 5044 void ice_update_pf_stats(struct ice_pf *pf) 5045 { 5046 struct ice_hw_port_stats *prev_ps, *cur_ps; 5047 struct ice_hw *hw = &pf->hw; 5048 u16 fd_ctr_base; 5049 u8 port; 5050 5051 port = hw->port_info->lport; 5052 prev_ps = &pf->stats_prev; 5053 cur_ps = &pf->stats; 5054 5055 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 5056 &prev_ps->eth.rx_bytes, 5057 &cur_ps->eth.rx_bytes); 5058 5059 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 5060 &prev_ps->eth.rx_unicast, 5061 &cur_ps->eth.rx_unicast); 5062 5063 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 5064 &prev_ps->eth.rx_multicast, 5065 &cur_ps->eth.rx_multicast); 5066 5067 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 5068 &prev_ps->eth.rx_broadcast, 5069 &cur_ps->eth.rx_broadcast); 5070 5071 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 5072 &prev_ps->eth.rx_discards, 5073 &cur_ps->eth.rx_discards); 5074 5075 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 5076 &prev_ps->eth.tx_bytes, 5077 &cur_ps->eth.tx_bytes); 5078 5079 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 5080 &prev_ps->eth.tx_unicast, 5081 &cur_ps->eth.tx_unicast); 5082 5083 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 5084 &prev_ps->eth.tx_multicast, 5085 &cur_ps->eth.tx_multicast); 5086 5087 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 5088 &prev_ps->eth.tx_broadcast, 5089 &cur_ps->eth.tx_broadcast); 5090 5091 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 5092 &prev_ps->tx_dropped_link_down, 5093 &cur_ps->tx_dropped_link_down); 5094 5095 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 5096 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 5097 5098 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 5099 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 5100 5101 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 5102 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 5103 5104 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 5105 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 5106 5107 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 5108 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 5109 5110 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 5111 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 5112 5113 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 5114 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 5115 5116 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 5117 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 5118 5119 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 5120 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 5121 5122 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 5123 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 5124 5125 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 5126 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 5127 5128 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 5129 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 5130 5131 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 5132 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 5133 5134 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 5135 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 5136 5137 fd_ctr_base = hw->fd_ctr_base; 5138 5139 ice_stat_update40(hw, 5140 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 5141 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 5142 &cur_ps->fd_sb_match); 5143 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 5144 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 5145 5146 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 5147 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 5148 5149 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 5150 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 5151 5152 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 5153 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 5154 5155 ice_update_dcb_stats(pf); 5156 5157 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 5158 &prev_ps->crc_errors, &cur_ps->crc_errors); 5159 5160 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 5161 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 5162 5163 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 5164 &prev_ps->mac_local_faults, 5165 &cur_ps->mac_local_faults); 5166 5167 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 5168 &prev_ps->mac_remote_faults, 5169 &cur_ps->mac_remote_faults); 5170 5171 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 5172 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 5173 5174 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 5175 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 5176 5177 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 5178 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 5179 5180 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 5181 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 5182 5183 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 5184 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 5185 5186 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 5187 5188 pf->stat_prev_loaded = true; 5189 } 5190 5191 /** 5192 * ice_get_stats64 - get statistics for network device structure 5193 * @netdev: network interface device structure 5194 * @stats: main device statistics structure 5195 */ 5196 static 5197 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 5198 { 5199 struct ice_netdev_priv *np = netdev_priv(netdev); 5200 struct rtnl_link_stats64 *vsi_stats; 5201 struct ice_vsi *vsi = np->vsi; 5202 5203 vsi_stats = &vsi->net_stats; 5204 5205 if (!vsi->num_txq || !vsi->num_rxq) 5206 return; 5207 5208 /* netdev packet/byte stats come from ring counter. These are obtained 5209 * by summing up ring counters (done by ice_update_vsi_ring_stats). 5210 * But, only call the update routine and read the registers if VSI is 5211 * not down. 5212 */ 5213 if (!test_bit(__ICE_DOWN, vsi->state)) 5214 ice_update_vsi_ring_stats(vsi); 5215 stats->tx_packets = vsi_stats->tx_packets; 5216 stats->tx_bytes = vsi_stats->tx_bytes; 5217 stats->rx_packets = vsi_stats->rx_packets; 5218 stats->rx_bytes = vsi_stats->rx_bytes; 5219 5220 /* The rest of the stats can be read from the hardware but instead we 5221 * just return values that the watchdog task has already obtained from 5222 * the hardware. 5223 */ 5224 stats->multicast = vsi_stats->multicast; 5225 stats->tx_errors = vsi_stats->tx_errors; 5226 stats->tx_dropped = vsi_stats->tx_dropped; 5227 stats->rx_errors = vsi_stats->rx_errors; 5228 stats->rx_dropped = vsi_stats->rx_dropped; 5229 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 5230 stats->rx_length_errors = vsi_stats->rx_length_errors; 5231 } 5232 5233 /** 5234 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5235 * @vsi: VSI having NAPI disabled 5236 */ 5237 static void ice_napi_disable_all(struct ice_vsi *vsi) 5238 { 5239 int q_idx; 5240 5241 if (!vsi->netdev) 5242 return; 5243 5244 ice_for_each_q_vector(vsi, q_idx) { 5245 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 5246 5247 if (q_vector->rx.ring || q_vector->tx.ring) 5248 napi_disable(&q_vector->napi); 5249 } 5250 } 5251 5252 /** 5253 * ice_down - Shutdown the connection 5254 * @vsi: The VSI being stopped 5255 */ 5256 int ice_down(struct ice_vsi *vsi) 5257 { 5258 int i, tx_err, rx_err, link_err = 0; 5259 5260 /* Caller of this function is expected to set the 5261 * vsi->state __ICE_DOWN bit 5262 */ 5263 if (vsi->netdev) { 5264 netif_carrier_off(vsi->netdev); 5265 netif_tx_disable(vsi->netdev); 5266 } 5267 5268 ice_vsi_dis_irq(vsi); 5269 5270 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 5271 if (tx_err) 5272 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 5273 vsi->vsi_num, tx_err); 5274 if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { 5275 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 5276 if (tx_err) 5277 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 5278 vsi->vsi_num, tx_err); 5279 } 5280 5281 rx_err = ice_vsi_stop_all_rx_rings(vsi); 5282 if (rx_err) 5283 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 5284 vsi->vsi_num, rx_err); 5285 5286 ice_napi_disable_all(vsi); 5287 5288 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 5289 link_err = ice_force_phys_link_state(vsi, false); 5290 if (link_err) 5291 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 5292 vsi->vsi_num, link_err); 5293 } 5294 5295 ice_for_each_txq(vsi, i) 5296 ice_clean_tx_ring(vsi->tx_rings[i]); 5297 5298 ice_for_each_rxq(vsi, i) 5299 ice_clean_rx_ring(vsi->rx_rings[i]); 5300 5301 if (tx_err || rx_err || link_err) { 5302 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 5303 vsi->vsi_num, vsi->vsw->sw_id); 5304 return -EIO; 5305 } 5306 5307 return 0; 5308 } 5309 5310 /** 5311 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 5312 * @vsi: VSI having resources allocated 5313 * 5314 * Return 0 on success, negative on failure 5315 */ 5316 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 5317 { 5318 int i, err = 0; 5319 5320 if (!vsi->num_txq) { 5321 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 5322 vsi->vsi_num); 5323 return -EINVAL; 5324 } 5325 5326 ice_for_each_txq(vsi, i) { 5327 struct ice_ring *ring = vsi->tx_rings[i]; 5328 5329 if (!ring) 5330 return -EINVAL; 5331 5332 ring->netdev = vsi->netdev; 5333 err = ice_setup_tx_ring(ring); 5334 if (err) 5335 break; 5336 } 5337 5338 return err; 5339 } 5340 5341 /** 5342 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 5343 * @vsi: VSI having resources allocated 5344 * 5345 * Return 0 on success, negative on failure 5346 */ 5347 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 5348 { 5349 int i, err = 0; 5350 5351 if (!vsi->num_rxq) { 5352 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 5353 vsi->vsi_num); 5354 return -EINVAL; 5355 } 5356 5357 ice_for_each_rxq(vsi, i) { 5358 struct ice_ring *ring = vsi->rx_rings[i]; 5359 5360 if (!ring) 5361 return -EINVAL; 5362 5363 ring->netdev = vsi->netdev; 5364 err = ice_setup_rx_ring(ring); 5365 if (err) 5366 break; 5367 } 5368 5369 return err; 5370 } 5371 5372 /** 5373 * ice_vsi_open_ctrl - open control VSI for use 5374 * @vsi: the VSI to open 5375 * 5376 * Initialization of the Control VSI 5377 * 5378 * Returns 0 on success, negative value on error 5379 */ 5380 int ice_vsi_open_ctrl(struct ice_vsi *vsi) 5381 { 5382 char int_name[ICE_INT_NAME_STR_LEN]; 5383 struct ice_pf *pf = vsi->back; 5384 struct device *dev; 5385 int err; 5386 5387 dev = ice_pf_to_dev(pf); 5388 /* allocate descriptors */ 5389 err = ice_vsi_setup_tx_rings(vsi); 5390 if (err) 5391 goto err_setup_tx; 5392 5393 err = ice_vsi_setup_rx_rings(vsi); 5394 if (err) 5395 goto err_setup_rx; 5396 5397 err = ice_vsi_cfg(vsi); 5398 if (err) 5399 goto err_setup_rx; 5400 5401 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 5402 dev_driver_string(dev), dev_name(dev)); 5403 err = ice_vsi_req_irq_msix(vsi, int_name); 5404 if (err) 5405 goto err_setup_rx; 5406 5407 ice_vsi_cfg_msix(vsi); 5408 5409 err = ice_vsi_start_all_rx_rings(vsi); 5410 if (err) 5411 goto err_up_complete; 5412 5413 clear_bit(__ICE_DOWN, vsi->state); 5414 ice_vsi_ena_irq(vsi); 5415 5416 return 0; 5417 5418 err_up_complete: 5419 ice_down(vsi); 5420 err_setup_rx: 5421 ice_vsi_free_rx_rings(vsi); 5422 err_setup_tx: 5423 ice_vsi_free_tx_rings(vsi); 5424 5425 return err; 5426 } 5427 5428 /** 5429 * ice_vsi_open - Called when a network interface is made active 5430 * @vsi: the VSI to open 5431 * 5432 * Initialization of the VSI 5433 * 5434 * Returns 0 on success, negative value on error 5435 */ 5436 static int ice_vsi_open(struct ice_vsi *vsi) 5437 { 5438 char int_name[ICE_INT_NAME_STR_LEN]; 5439 struct ice_pf *pf = vsi->back; 5440 int err; 5441 5442 /* allocate descriptors */ 5443 err = ice_vsi_setup_tx_rings(vsi); 5444 if (err) 5445 goto err_setup_tx; 5446 5447 err = ice_vsi_setup_rx_rings(vsi); 5448 if (err) 5449 goto err_setup_rx; 5450 5451 err = ice_vsi_cfg(vsi); 5452 if (err) 5453 goto err_setup_rx; 5454 5455 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5456 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 5457 err = ice_vsi_req_irq_msix(vsi, int_name); 5458 if (err) 5459 goto err_setup_rx; 5460 5461 /* Notify the stack of the actual queue counts. */ 5462 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 5463 if (err) 5464 goto err_set_qs; 5465 5466 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 5467 if (err) 5468 goto err_set_qs; 5469 5470 err = ice_up_complete(vsi); 5471 if (err) 5472 goto err_up_complete; 5473 5474 return 0; 5475 5476 err_up_complete: 5477 ice_down(vsi); 5478 err_set_qs: 5479 ice_vsi_free_irq(vsi); 5480 err_setup_rx: 5481 ice_vsi_free_rx_rings(vsi); 5482 err_setup_tx: 5483 ice_vsi_free_tx_rings(vsi); 5484 5485 return err; 5486 } 5487 5488 /** 5489 * ice_vsi_release_all - Delete all VSIs 5490 * @pf: PF from which all VSIs are being removed 5491 */ 5492 static void ice_vsi_release_all(struct ice_pf *pf) 5493 { 5494 int err, i; 5495 5496 if (!pf->vsi) 5497 return; 5498 5499 ice_for_each_vsi(pf, i) { 5500 if (!pf->vsi[i]) 5501 continue; 5502 5503 err = ice_vsi_release(pf->vsi[i]); 5504 if (err) 5505 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 5506 i, err, pf->vsi[i]->vsi_num); 5507 } 5508 } 5509 5510 /** 5511 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 5512 * @pf: pointer to the PF instance 5513 * @type: VSI type to rebuild 5514 * 5515 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 5516 */ 5517 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 5518 { 5519 struct device *dev = ice_pf_to_dev(pf); 5520 enum ice_status status; 5521 int i, err; 5522 5523 ice_for_each_vsi(pf, i) { 5524 struct ice_vsi *vsi = pf->vsi[i]; 5525 5526 if (!vsi || vsi->type != type) 5527 continue; 5528 5529 /* rebuild the VSI */ 5530 err = ice_vsi_rebuild(vsi, true); 5531 if (err) { 5532 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 5533 err, vsi->idx, ice_vsi_type_str(type)); 5534 return err; 5535 } 5536 5537 /* replay filters for the VSI */ 5538 status = ice_replay_vsi(&pf->hw, vsi->idx); 5539 if (status) { 5540 dev_err(dev, "replay VSI failed, status %s, VSI index %d, type %s\n", 5541 ice_stat_str(status), vsi->idx, 5542 ice_vsi_type_str(type)); 5543 return -EIO; 5544 } 5545 5546 /* Re-map HW VSI number, using VSI handle that has been 5547 * previously validated in ice_replay_vsi() call above 5548 */ 5549 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 5550 5551 /* enable the VSI */ 5552 err = ice_ena_vsi(vsi, false); 5553 if (err) { 5554 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 5555 err, vsi->idx, ice_vsi_type_str(type)); 5556 return err; 5557 } 5558 5559 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 5560 ice_vsi_type_str(type)); 5561 } 5562 5563 return 0; 5564 } 5565 5566 /** 5567 * ice_update_pf_netdev_link - Update PF netdev link status 5568 * @pf: pointer to the PF instance 5569 */ 5570 static void ice_update_pf_netdev_link(struct ice_pf *pf) 5571 { 5572 bool link_up; 5573 int i; 5574 5575 ice_for_each_vsi(pf, i) { 5576 struct ice_vsi *vsi = pf->vsi[i]; 5577 5578 if (!vsi || vsi->type != ICE_VSI_PF) 5579 return; 5580 5581 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 5582 if (link_up) { 5583 netif_carrier_on(pf->vsi[i]->netdev); 5584 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 5585 } else { 5586 netif_carrier_off(pf->vsi[i]->netdev); 5587 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 5588 } 5589 } 5590 } 5591 5592 /** 5593 * ice_rebuild - rebuild after reset 5594 * @pf: PF to rebuild 5595 * @reset_type: type of reset 5596 * 5597 * Do not rebuild VF VSI in this flow because that is already handled via 5598 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 5599 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 5600 * to reset/rebuild all the VF VSI twice. 5601 */ 5602 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 5603 { 5604 struct device *dev = ice_pf_to_dev(pf); 5605 struct ice_hw *hw = &pf->hw; 5606 enum ice_status ret; 5607 int err; 5608 5609 if (test_bit(__ICE_DOWN, pf->state)) 5610 goto clear_recovery; 5611 5612 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 5613 5614 ret = ice_init_all_ctrlq(hw); 5615 if (ret) { 5616 dev_err(dev, "control queues init failed %s\n", 5617 ice_stat_str(ret)); 5618 goto err_init_ctrlq; 5619 } 5620 5621 /* if DDP was previously loaded successfully */ 5622 if (!ice_is_safe_mode(pf)) { 5623 /* reload the SW DB of filter tables */ 5624 if (reset_type == ICE_RESET_PFR) 5625 ice_fill_blk_tbls(hw); 5626 else 5627 /* Reload DDP Package after CORER/GLOBR reset */ 5628 ice_load_pkg(NULL, pf); 5629 } 5630 5631 ret = ice_clear_pf_cfg(hw); 5632 if (ret) { 5633 dev_err(dev, "clear PF configuration failed %s\n", 5634 ice_stat_str(ret)); 5635 goto err_init_ctrlq; 5636 } 5637 5638 if (pf->first_sw->dflt_vsi_ena) 5639 dev_info(dev, "Clearing default VSI, re-enable after reset completes\n"); 5640 /* clear the default VSI configuration if it exists */ 5641 pf->first_sw->dflt_vsi = NULL; 5642 pf->first_sw->dflt_vsi_ena = false; 5643 5644 ice_clear_pxe_mode(hw); 5645 5646 ret = ice_get_caps(hw); 5647 if (ret) { 5648 dev_err(dev, "ice_get_caps failed %s\n", ice_stat_str(ret)); 5649 goto err_init_ctrlq; 5650 } 5651 5652 ret = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 5653 if (ret) { 5654 dev_err(dev, "set_mac_cfg failed %s\n", ice_stat_str(ret)); 5655 goto err_init_ctrlq; 5656 } 5657 5658 err = ice_sched_init_port(hw->port_info); 5659 if (err) 5660 goto err_sched_init_port; 5661 5662 err = ice_update_link_info(hw->port_info); 5663 if (err) 5664 dev_err(dev, "Get link status error %d\n", err); 5665 5666 /* start misc vector */ 5667 err = ice_req_irq_msix_misc(pf); 5668 if (err) { 5669 dev_err(dev, "misc vector setup failed: %d\n", err); 5670 goto err_sched_init_port; 5671 } 5672 5673 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 5674 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 5675 if (!rd32(hw, PFQF_FD_SIZE)) { 5676 u16 unused, guar, b_effort; 5677 5678 guar = hw->func_caps.fd_fltr_guar; 5679 b_effort = hw->func_caps.fd_fltr_best_effort; 5680 5681 /* force guaranteed filter pool for PF */ 5682 ice_alloc_fd_guar_item(hw, &unused, guar); 5683 /* force shared filter pool for PF */ 5684 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 5685 } 5686 } 5687 5688 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 5689 ice_dcb_rebuild(pf); 5690 5691 /* rebuild PF VSI */ 5692 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 5693 if (err) { 5694 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 5695 goto err_vsi_rebuild; 5696 } 5697 5698 /* If Flow Director is active */ 5699 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 5700 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 5701 if (err) { 5702 dev_err(dev, "control VSI rebuild failed: %d\n", err); 5703 goto err_vsi_rebuild; 5704 } 5705 5706 /* replay HW Flow Director recipes */ 5707 if (hw->fdir_prof) 5708 ice_fdir_replay_flows(hw); 5709 5710 /* replay Flow Director filters */ 5711 ice_fdir_replay_fltrs(pf); 5712 5713 ice_rebuild_arfs(pf); 5714 } 5715 5716 ice_update_pf_netdev_link(pf); 5717 5718 /* tell the firmware we are up */ 5719 ret = ice_send_version(pf); 5720 if (ret) { 5721 dev_err(dev, "Rebuild failed due to error sending driver version: %s\n", 5722 ice_stat_str(ret)); 5723 goto err_vsi_rebuild; 5724 } 5725 5726 ice_replay_post(hw); 5727 5728 /* if we get here, reset flow is successful */ 5729 clear_bit(__ICE_RESET_FAILED, pf->state); 5730 return; 5731 5732 err_vsi_rebuild: 5733 err_sched_init_port: 5734 ice_sched_cleanup_all(hw); 5735 err_init_ctrlq: 5736 ice_shutdown_all_ctrlq(hw); 5737 set_bit(__ICE_RESET_FAILED, pf->state); 5738 clear_recovery: 5739 /* set this bit in PF state to control service task scheduling */ 5740 set_bit(__ICE_NEEDS_RESTART, pf->state); 5741 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 5742 } 5743 5744 /** 5745 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 5746 * @vsi: Pointer to VSI structure 5747 */ 5748 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 5749 { 5750 if (PAGE_SIZE >= 8192 || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 5751 return ICE_RXBUF_2048 - XDP_PACKET_HEADROOM; 5752 else 5753 return ICE_RXBUF_3072; 5754 } 5755 5756 /** 5757 * ice_change_mtu - NDO callback to change the MTU 5758 * @netdev: network interface device structure 5759 * @new_mtu: new value for maximum frame size 5760 * 5761 * Returns 0 on success, negative on failure 5762 */ 5763 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 5764 { 5765 struct ice_netdev_priv *np = netdev_priv(netdev); 5766 struct ice_vsi *vsi = np->vsi; 5767 struct ice_pf *pf = vsi->back; 5768 u8 count = 0; 5769 5770 if (new_mtu == (int)netdev->mtu) { 5771 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 5772 return 0; 5773 } 5774 5775 if (ice_is_xdp_ena_vsi(vsi)) { 5776 int frame_size = ice_max_xdp_frame_size(vsi); 5777 5778 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 5779 netdev_err(netdev, "max MTU for XDP usage is %d\n", 5780 frame_size - ICE_ETH_PKT_HDR_PAD); 5781 return -EINVAL; 5782 } 5783 } 5784 5785 if (new_mtu < (int)netdev->min_mtu) { 5786 netdev_err(netdev, "new MTU invalid. min_mtu is %d\n", 5787 netdev->min_mtu); 5788 return -EINVAL; 5789 } else if (new_mtu > (int)netdev->max_mtu) { 5790 netdev_err(netdev, "new MTU invalid. max_mtu is %d\n", 5791 netdev->min_mtu); 5792 return -EINVAL; 5793 } 5794 /* if a reset is in progress, wait for some time for it to complete */ 5795 do { 5796 if (ice_is_reset_in_progress(pf->state)) { 5797 count++; 5798 usleep_range(1000, 2000); 5799 } else { 5800 break; 5801 } 5802 5803 } while (count < 100); 5804 5805 if (count == 100) { 5806 netdev_err(netdev, "can't change MTU. Device is busy\n"); 5807 return -EBUSY; 5808 } 5809 5810 netdev->mtu = (unsigned int)new_mtu; 5811 5812 /* if VSI is up, bring it down and then back up */ 5813 if (!test_and_set_bit(__ICE_DOWN, vsi->state)) { 5814 int err; 5815 5816 err = ice_down(vsi); 5817 if (err) { 5818 netdev_err(netdev, "change MTU if_up err %d\n", err); 5819 return err; 5820 } 5821 5822 err = ice_up(vsi); 5823 if (err) { 5824 netdev_err(netdev, "change MTU if_up err %d\n", err); 5825 return err; 5826 } 5827 } 5828 5829 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 5830 return 0; 5831 } 5832 5833 /** 5834 * ice_aq_str - convert AQ err code to a string 5835 * @aq_err: the AQ error code to convert 5836 */ 5837 const char *ice_aq_str(enum ice_aq_err aq_err) 5838 { 5839 switch (aq_err) { 5840 case ICE_AQ_RC_OK: 5841 return "OK"; 5842 case ICE_AQ_RC_EPERM: 5843 return "ICE_AQ_RC_EPERM"; 5844 case ICE_AQ_RC_ENOENT: 5845 return "ICE_AQ_RC_ENOENT"; 5846 case ICE_AQ_RC_ENOMEM: 5847 return "ICE_AQ_RC_ENOMEM"; 5848 case ICE_AQ_RC_EBUSY: 5849 return "ICE_AQ_RC_EBUSY"; 5850 case ICE_AQ_RC_EEXIST: 5851 return "ICE_AQ_RC_EEXIST"; 5852 case ICE_AQ_RC_EINVAL: 5853 return "ICE_AQ_RC_EINVAL"; 5854 case ICE_AQ_RC_ENOSPC: 5855 return "ICE_AQ_RC_ENOSPC"; 5856 case ICE_AQ_RC_ENOSYS: 5857 return "ICE_AQ_RC_ENOSYS"; 5858 case ICE_AQ_RC_EMODE: 5859 return "ICE_AQ_RC_EMODE"; 5860 case ICE_AQ_RC_ENOSEC: 5861 return "ICE_AQ_RC_ENOSEC"; 5862 case ICE_AQ_RC_EBADSIG: 5863 return "ICE_AQ_RC_EBADSIG"; 5864 case ICE_AQ_RC_ESVN: 5865 return "ICE_AQ_RC_ESVN"; 5866 case ICE_AQ_RC_EBADMAN: 5867 return "ICE_AQ_RC_EBADMAN"; 5868 case ICE_AQ_RC_EBADBUF: 5869 return "ICE_AQ_RC_EBADBUF"; 5870 } 5871 5872 return "ICE_AQ_RC_UNKNOWN"; 5873 } 5874 5875 /** 5876 * ice_stat_str - convert status err code to a string 5877 * @stat_err: the status error code to convert 5878 */ 5879 const char *ice_stat_str(enum ice_status stat_err) 5880 { 5881 switch (stat_err) { 5882 case ICE_SUCCESS: 5883 return "OK"; 5884 case ICE_ERR_PARAM: 5885 return "ICE_ERR_PARAM"; 5886 case ICE_ERR_NOT_IMPL: 5887 return "ICE_ERR_NOT_IMPL"; 5888 case ICE_ERR_NOT_READY: 5889 return "ICE_ERR_NOT_READY"; 5890 case ICE_ERR_NOT_SUPPORTED: 5891 return "ICE_ERR_NOT_SUPPORTED"; 5892 case ICE_ERR_BAD_PTR: 5893 return "ICE_ERR_BAD_PTR"; 5894 case ICE_ERR_INVAL_SIZE: 5895 return "ICE_ERR_INVAL_SIZE"; 5896 case ICE_ERR_DEVICE_NOT_SUPPORTED: 5897 return "ICE_ERR_DEVICE_NOT_SUPPORTED"; 5898 case ICE_ERR_RESET_FAILED: 5899 return "ICE_ERR_RESET_FAILED"; 5900 case ICE_ERR_FW_API_VER: 5901 return "ICE_ERR_FW_API_VER"; 5902 case ICE_ERR_NO_MEMORY: 5903 return "ICE_ERR_NO_MEMORY"; 5904 case ICE_ERR_CFG: 5905 return "ICE_ERR_CFG"; 5906 case ICE_ERR_OUT_OF_RANGE: 5907 return "ICE_ERR_OUT_OF_RANGE"; 5908 case ICE_ERR_ALREADY_EXISTS: 5909 return "ICE_ERR_ALREADY_EXISTS"; 5910 case ICE_ERR_NVM_CHECKSUM: 5911 return "ICE_ERR_NVM_CHECKSUM"; 5912 case ICE_ERR_BUF_TOO_SHORT: 5913 return "ICE_ERR_BUF_TOO_SHORT"; 5914 case ICE_ERR_NVM_BLANK_MODE: 5915 return "ICE_ERR_NVM_BLANK_MODE"; 5916 case ICE_ERR_IN_USE: 5917 return "ICE_ERR_IN_USE"; 5918 case ICE_ERR_MAX_LIMIT: 5919 return "ICE_ERR_MAX_LIMIT"; 5920 case ICE_ERR_RESET_ONGOING: 5921 return "ICE_ERR_RESET_ONGOING"; 5922 case ICE_ERR_HW_TABLE: 5923 return "ICE_ERR_HW_TABLE"; 5924 case ICE_ERR_DOES_NOT_EXIST: 5925 return "ICE_ERR_DOES_NOT_EXIST"; 5926 case ICE_ERR_FW_DDP_MISMATCH: 5927 return "ICE_ERR_FW_DDP_MISMATCH"; 5928 case ICE_ERR_AQ_ERROR: 5929 return "ICE_ERR_AQ_ERROR"; 5930 case ICE_ERR_AQ_TIMEOUT: 5931 return "ICE_ERR_AQ_TIMEOUT"; 5932 case ICE_ERR_AQ_FULL: 5933 return "ICE_ERR_AQ_FULL"; 5934 case ICE_ERR_AQ_NO_WORK: 5935 return "ICE_ERR_AQ_NO_WORK"; 5936 case ICE_ERR_AQ_EMPTY: 5937 return "ICE_ERR_AQ_EMPTY"; 5938 case ICE_ERR_AQ_FW_CRITICAL: 5939 return "ICE_ERR_AQ_FW_CRITICAL"; 5940 } 5941 5942 return "ICE_ERR_UNKNOWN"; 5943 } 5944 5945 /** 5946 * ice_set_rss - Set RSS keys and lut 5947 * @vsi: Pointer to VSI structure 5948 * @seed: RSS hash seed 5949 * @lut: Lookup table 5950 * @lut_size: Lookup table size 5951 * 5952 * Returns 0 on success, negative on failure 5953 */ 5954 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 5955 { 5956 struct ice_pf *pf = vsi->back; 5957 struct ice_hw *hw = &pf->hw; 5958 enum ice_status status; 5959 struct device *dev; 5960 5961 dev = ice_pf_to_dev(pf); 5962 if (seed) { 5963 struct ice_aqc_get_set_rss_keys *buf = 5964 (struct ice_aqc_get_set_rss_keys *)seed; 5965 5966 status = ice_aq_set_rss_key(hw, vsi->idx, buf); 5967 5968 if (status) { 5969 dev_err(dev, "Cannot set RSS key, err %s aq_err %s\n", 5970 ice_stat_str(status), 5971 ice_aq_str(hw->adminq.sq_last_status)); 5972 return -EIO; 5973 } 5974 } 5975 5976 if (lut) { 5977 status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 5978 lut, lut_size); 5979 if (status) { 5980 dev_err(dev, "Cannot set RSS lut, err %s aq_err %s\n", 5981 ice_stat_str(status), 5982 ice_aq_str(hw->adminq.sq_last_status)); 5983 return -EIO; 5984 } 5985 } 5986 5987 return 0; 5988 } 5989 5990 /** 5991 * ice_get_rss - Get RSS keys and lut 5992 * @vsi: Pointer to VSI structure 5993 * @seed: Buffer to store the keys 5994 * @lut: Buffer to store the lookup table entries 5995 * @lut_size: Size of buffer to store the lookup table entries 5996 * 5997 * Returns 0 on success, negative on failure 5998 */ 5999 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 6000 { 6001 struct ice_pf *pf = vsi->back; 6002 struct ice_hw *hw = &pf->hw; 6003 enum ice_status status; 6004 struct device *dev; 6005 6006 dev = ice_pf_to_dev(pf); 6007 if (seed) { 6008 struct ice_aqc_get_set_rss_keys *buf = 6009 (struct ice_aqc_get_set_rss_keys *)seed; 6010 6011 status = ice_aq_get_rss_key(hw, vsi->idx, buf); 6012 if (status) { 6013 dev_err(dev, "Cannot get RSS key, err %s aq_err %s\n", 6014 ice_stat_str(status), 6015 ice_aq_str(hw->adminq.sq_last_status)); 6016 return -EIO; 6017 } 6018 } 6019 6020 if (lut) { 6021 status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 6022 lut, lut_size); 6023 if (status) { 6024 dev_err(dev, "Cannot get RSS lut, err %s aq_err %s\n", 6025 ice_stat_str(status), 6026 ice_aq_str(hw->adminq.sq_last_status)); 6027 return -EIO; 6028 } 6029 } 6030 6031 return 0; 6032 } 6033 6034 /** 6035 * ice_bridge_getlink - Get the hardware bridge mode 6036 * @skb: skb buff 6037 * @pid: process ID 6038 * @seq: RTNL message seq 6039 * @dev: the netdev being configured 6040 * @filter_mask: filter mask passed in 6041 * @nlflags: netlink flags passed in 6042 * 6043 * Return the bridge mode (VEB/VEPA) 6044 */ 6045 static int 6046 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 6047 struct net_device *dev, u32 filter_mask, int nlflags) 6048 { 6049 struct ice_netdev_priv *np = netdev_priv(dev); 6050 struct ice_vsi *vsi = np->vsi; 6051 struct ice_pf *pf = vsi->back; 6052 u16 bmode; 6053 6054 bmode = pf->first_sw->bridge_mode; 6055 6056 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 6057 filter_mask, NULL); 6058 } 6059 6060 /** 6061 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 6062 * @vsi: Pointer to VSI structure 6063 * @bmode: Hardware bridge mode (VEB/VEPA) 6064 * 6065 * Returns 0 on success, negative on failure 6066 */ 6067 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 6068 { 6069 struct ice_aqc_vsi_props *vsi_props; 6070 struct ice_hw *hw = &vsi->back->hw; 6071 struct ice_vsi_ctx *ctxt; 6072 enum ice_status status; 6073 int ret = 0; 6074 6075 vsi_props = &vsi->info; 6076 6077 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 6078 if (!ctxt) 6079 return -ENOMEM; 6080 6081 ctxt->info = vsi->info; 6082 6083 if (bmode == BRIDGE_MODE_VEB) 6084 /* change from VEPA to VEB mode */ 6085 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6086 else 6087 /* change from VEB to VEPA mode */ 6088 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6089 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 6090 6091 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 6092 if (status) { 6093 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %s aq_err %s\n", 6094 bmode, ice_stat_str(status), 6095 ice_aq_str(hw->adminq.sq_last_status)); 6096 ret = -EIO; 6097 goto out; 6098 } 6099 /* Update sw flags for book keeping */ 6100 vsi_props->sw_flags = ctxt->info.sw_flags; 6101 6102 out: 6103 kfree(ctxt); 6104 return ret; 6105 } 6106 6107 /** 6108 * ice_bridge_setlink - Set the hardware bridge mode 6109 * @dev: the netdev being configured 6110 * @nlh: RTNL message 6111 * @flags: bridge setlink flags 6112 * @extack: netlink extended ack 6113 * 6114 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 6115 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 6116 * not already set for all VSIs connected to this switch. And also update the 6117 * unicast switch filter rules for the corresponding switch of the netdev. 6118 */ 6119 static int 6120 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 6121 u16 __always_unused flags, 6122 struct netlink_ext_ack __always_unused *extack) 6123 { 6124 struct ice_netdev_priv *np = netdev_priv(dev); 6125 struct ice_pf *pf = np->vsi->back; 6126 struct nlattr *attr, *br_spec; 6127 struct ice_hw *hw = &pf->hw; 6128 enum ice_status status; 6129 struct ice_sw *pf_sw; 6130 int rem, v, err = 0; 6131 6132 pf_sw = pf->first_sw; 6133 /* find the attribute in the netlink message */ 6134 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 6135 6136 nla_for_each_nested(attr, br_spec, rem) { 6137 __u16 mode; 6138 6139 if (nla_type(attr) != IFLA_BRIDGE_MODE) 6140 continue; 6141 mode = nla_get_u16(attr); 6142 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 6143 return -EINVAL; 6144 /* Continue if bridge mode is not being flipped */ 6145 if (mode == pf_sw->bridge_mode) 6146 continue; 6147 /* Iterates through the PF VSI list and update the loopback 6148 * mode of the VSI 6149 */ 6150 ice_for_each_vsi(pf, v) { 6151 if (!pf->vsi[v]) 6152 continue; 6153 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 6154 if (err) 6155 return err; 6156 } 6157 6158 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 6159 /* Update the unicast switch filter rules for the corresponding 6160 * switch of the netdev 6161 */ 6162 status = ice_update_sw_rule_bridge_mode(hw); 6163 if (status) { 6164 netdev_err(dev, "switch rule update failed, mode = %d err %s aq_err %s\n", 6165 mode, ice_stat_str(status), 6166 ice_aq_str(hw->adminq.sq_last_status)); 6167 /* revert hw->evb_veb */ 6168 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 6169 return -EIO; 6170 } 6171 6172 pf_sw->bridge_mode = mode; 6173 } 6174 6175 return 0; 6176 } 6177 6178 /** 6179 * ice_tx_timeout - Respond to a Tx Hang 6180 * @netdev: network interface device structure 6181 * @txqueue: Tx queue 6182 */ 6183 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 6184 { 6185 struct ice_netdev_priv *np = netdev_priv(netdev); 6186 struct ice_ring *tx_ring = NULL; 6187 struct ice_vsi *vsi = np->vsi; 6188 struct ice_pf *pf = vsi->back; 6189 u32 i; 6190 6191 pf->tx_timeout_count++; 6192 6193 /* Check if PFC is enabled for the TC to which the queue belongs 6194 * to. If yes then Tx timeout is not caused by a hung queue, no 6195 * need to reset and rebuild 6196 */ 6197 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 6198 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 6199 txqueue); 6200 return; 6201 } 6202 6203 /* now that we have an index, find the tx_ring struct */ 6204 for (i = 0; i < vsi->num_txq; i++) 6205 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 6206 if (txqueue == vsi->tx_rings[i]->q_index) { 6207 tx_ring = vsi->tx_rings[i]; 6208 break; 6209 } 6210 6211 /* Reset recovery level if enough time has elapsed after last timeout. 6212 * Also ensure no new reset action happens before next timeout period. 6213 */ 6214 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 6215 pf->tx_timeout_recovery_level = 1; 6216 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 6217 netdev->watchdog_timeo))) 6218 return; 6219 6220 if (tx_ring) { 6221 struct ice_hw *hw = &pf->hw; 6222 u32 head, val = 0; 6223 6224 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) & 6225 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 6226 /* Read interrupt register */ 6227 val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 6228 6229 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 6230 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 6231 head, tx_ring->next_to_use, val); 6232 } 6233 6234 pf->tx_timeout_last_recovery = jiffies; 6235 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 6236 pf->tx_timeout_recovery_level, txqueue); 6237 6238 switch (pf->tx_timeout_recovery_level) { 6239 case 1: 6240 set_bit(__ICE_PFR_REQ, pf->state); 6241 break; 6242 case 2: 6243 set_bit(__ICE_CORER_REQ, pf->state); 6244 break; 6245 case 3: 6246 set_bit(__ICE_GLOBR_REQ, pf->state); 6247 break; 6248 default: 6249 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 6250 set_bit(__ICE_DOWN, pf->state); 6251 set_bit(__ICE_NEEDS_RESTART, vsi->state); 6252 set_bit(__ICE_SERVICE_DIS, pf->state); 6253 break; 6254 } 6255 6256 ice_service_task_schedule(pf); 6257 pf->tx_timeout_recovery_level++; 6258 } 6259 6260 /** 6261 * ice_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 6262 * @netdev: This physical port's netdev 6263 * @ti: Tunnel endpoint information 6264 */ 6265 static void 6266 ice_udp_tunnel_add(struct net_device *netdev, struct udp_tunnel_info *ti) 6267 { 6268 struct ice_netdev_priv *np = netdev_priv(netdev); 6269 struct ice_vsi *vsi = np->vsi; 6270 struct ice_pf *pf = vsi->back; 6271 enum ice_tunnel_type tnl_type; 6272 u16 port = ntohs(ti->port); 6273 enum ice_status status; 6274 6275 switch (ti->type) { 6276 case UDP_TUNNEL_TYPE_VXLAN: 6277 tnl_type = TNL_VXLAN; 6278 break; 6279 case UDP_TUNNEL_TYPE_GENEVE: 6280 tnl_type = TNL_GENEVE; 6281 break; 6282 default: 6283 netdev_err(netdev, "Unknown tunnel type\n"); 6284 return; 6285 } 6286 6287 status = ice_create_tunnel(&pf->hw, tnl_type, port); 6288 if (status == ICE_ERR_OUT_OF_RANGE) 6289 netdev_info(netdev, "Max tunneled UDP ports reached, port %d not added\n", 6290 port); 6291 else if (status) 6292 netdev_err(netdev, "Error adding UDP tunnel - %s\n", 6293 ice_stat_str(status)); 6294 } 6295 6296 /** 6297 * ice_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 6298 * @netdev: This physical port's netdev 6299 * @ti: Tunnel endpoint information 6300 */ 6301 static void 6302 ice_udp_tunnel_del(struct net_device *netdev, struct udp_tunnel_info *ti) 6303 { 6304 struct ice_netdev_priv *np = netdev_priv(netdev); 6305 struct ice_vsi *vsi = np->vsi; 6306 struct ice_pf *pf = vsi->back; 6307 u16 port = ntohs(ti->port); 6308 enum ice_status status; 6309 bool retval; 6310 6311 retval = ice_tunnel_port_in_use(&pf->hw, port, NULL); 6312 if (!retval) { 6313 netdev_info(netdev, "port %d not found in UDP tunnels list\n", 6314 port); 6315 return; 6316 } 6317 6318 status = ice_destroy_tunnel(&pf->hw, port, false); 6319 if (status) 6320 netdev_err(netdev, "error deleting port %d from UDP tunnels list\n", 6321 port); 6322 } 6323 6324 /** 6325 * ice_open - Called when a network interface becomes active 6326 * @netdev: network interface device structure 6327 * 6328 * The open entry point is called when a network interface is made 6329 * active by the system (IFF_UP). At this point all resources needed 6330 * for transmit and receive operations are allocated, the interrupt 6331 * handler is registered with the OS, the netdev watchdog is enabled, 6332 * and the stack is notified that the interface is ready. 6333 * 6334 * Returns 0 on success, negative value on failure 6335 */ 6336 int ice_open(struct net_device *netdev) 6337 { 6338 struct ice_netdev_priv *np = netdev_priv(netdev); 6339 struct ice_vsi *vsi = np->vsi; 6340 struct ice_pf *pf = vsi->back; 6341 struct ice_port_info *pi; 6342 int err; 6343 6344 if (test_bit(__ICE_NEEDS_RESTART, pf->state)) { 6345 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 6346 return -EIO; 6347 } 6348 6349 if (test_bit(__ICE_DOWN, pf->state)) { 6350 netdev_err(netdev, "device is not ready yet\n"); 6351 return -EBUSY; 6352 } 6353 6354 netif_carrier_off(netdev); 6355 6356 pi = vsi->port_info; 6357 err = ice_update_link_info(pi); 6358 if (err) { 6359 netdev_err(netdev, "Failed to get link info, error %d\n", 6360 err); 6361 return err; 6362 } 6363 6364 /* Set PHY if there is media, otherwise, turn off PHY */ 6365 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 6366 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6367 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) { 6368 err = ice_init_phy_user_cfg(pi); 6369 if (err) { 6370 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 6371 err); 6372 return err; 6373 } 6374 } 6375 6376 err = ice_configure_phy(vsi); 6377 if (err) { 6378 netdev_err(netdev, "Failed to set physical link up, error %d\n", 6379 err); 6380 return err; 6381 } 6382 } else { 6383 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6384 err = ice_aq_set_link_restart_an(pi, false, NULL); 6385 if (err) { 6386 netdev_err(netdev, "Failed to set PHY state, VSI %d error %d\n", 6387 vsi->vsi_num, err); 6388 return err; 6389 } 6390 } 6391 6392 err = ice_vsi_open(vsi); 6393 if (err) 6394 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 6395 vsi->vsi_num, vsi->vsw->sw_id); 6396 6397 /* Update existing tunnels information */ 6398 udp_tunnel_get_rx_info(netdev); 6399 6400 return err; 6401 } 6402 6403 /** 6404 * ice_stop - Disables a network interface 6405 * @netdev: network interface device structure 6406 * 6407 * The stop entry point is called when an interface is de-activated by the OS, 6408 * and the netdevice enters the DOWN state. The hardware is still under the 6409 * driver's control, but the netdev interface is disabled. 6410 * 6411 * Returns success only - not allowed to fail 6412 */ 6413 int ice_stop(struct net_device *netdev) 6414 { 6415 struct ice_netdev_priv *np = netdev_priv(netdev); 6416 struct ice_vsi *vsi = np->vsi; 6417 6418 ice_vsi_close(vsi); 6419 6420 return 0; 6421 } 6422 6423 /** 6424 * ice_features_check - Validate encapsulated packet conforms to limits 6425 * @skb: skb buffer 6426 * @netdev: This port's netdev 6427 * @features: Offload features that the stack believes apply 6428 */ 6429 static netdev_features_t 6430 ice_features_check(struct sk_buff *skb, 6431 struct net_device __always_unused *netdev, 6432 netdev_features_t features) 6433 { 6434 size_t len; 6435 6436 /* No point in doing any of this if neither checksum nor GSO are 6437 * being requested for this frame. We can rule out both by just 6438 * checking for CHECKSUM_PARTIAL 6439 */ 6440 if (skb->ip_summed != CHECKSUM_PARTIAL) 6441 return features; 6442 6443 /* We cannot support GSO if the MSS is going to be less than 6444 * 64 bytes. If it is then we need to drop support for GSO. 6445 */ 6446 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 6447 features &= ~NETIF_F_GSO_MASK; 6448 6449 len = skb_network_header(skb) - skb->data; 6450 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 6451 goto out_rm_features; 6452 6453 len = skb_transport_header(skb) - skb_network_header(skb); 6454 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6455 goto out_rm_features; 6456 6457 if (skb->encapsulation) { 6458 len = skb_inner_network_header(skb) - skb_transport_header(skb); 6459 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 6460 goto out_rm_features; 6461 6462 len = skb_inner_transport_header(skb) - 6463 skb_inner_network_header(skb); 6464 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6465 goto out_rm_features; 6466 } 6467 6468 return features; 6469 out_rm_features: 6470 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 6471 } 6472 6473 static const struct net_device_ops ice_netdev_safe_mode_ops = { 6474 .ndo_open = ice_open, 6475 .ndo_stop = ice_stop, 6476 .ndo_start_xmit = ice_start_xmit, 6477 .ndo_set_mac_address = ice_set_mac_address, 6478 .ndo_validate_addr = eth_validate_addr, 6479 .ndo_change_mtu = ice_change_mtu, 6480 .ndo_get_stats64 = ice_get_stats64, 6481 .ndo_tx_timeout = ice_tx_timeout, 6482 }; 6483 6484 static const struct net_device_ops ice_netdev_ops = { 6485 .ndo_open = ice_open, 6486 .ndo_stop = ice_stop, 6487 .ndo_start_xmit = ice_start_xmit, 6488 .ndo_features_check = ice_features_check, 6489 .ndo_set_rx_mode = ice_set_rx_mode, 6490 .ndo_set_mac_address = ice_set_mac_address, 6491 .ndo_validate_addr = eth_validate_addr, 6492 .ndo_change_mtu = ice_change_mtu, 6493 .ndo_get_stats64 = ice_get_stats64, 6494 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 6495 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 6496 .ndo_set_vf_mac = ice_set_vf_mac, 6497 .ndo_get_vf_config = ice_get_vf_cfg, 6498 .ndo_set_vf_trust = ice_set_vf_trust, 6499 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 6500 .ndo_set_vf_link_state = ice_set_vf_link_state, 6501 .ndo_get_vf_stats = ice_get_vf_stats, 6502 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 6503 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 6504 .ndo_set_features = ice_set_features, 6505 .ndo_bridge_getlink = ice_bridge_getlink, 6506 .ndo_bridge_setlink = ice_bridge_setlink, 6507 .ndo_fdb_add = ice_fdb_add, 6508 .ndo_fdb_del = ice_fdb_del, 6509 #ifdef CONFIG_RFS_ACCEL 6510 .ndo_rx_flow_steer = ice_rx_flow_steer, 6511 #endif 6512 .ndo_tx_timeout = ice_tx_timeout, 6513 .ndo_bpf = ice_xdp, 6514 .ndo_xdp_xmit = ice_xdp_xmit, 6515 .ndo_xsk_wakeup = ice_xsk_wakeup, 6516 .ndo_udp_tunnel_add = ice_udp_tunnel_add, 6517 .ndo_udp_tunnel_del = ice_udp_tunnel_del, 6518 }; 6519