1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <generated/utsrelease.h> 9 #include "ice.h" 10 #include "ice_base.h" 11 #include "ice_lib.h" 12 #include "ice_fltr.h" 13 #include "ice_dcb_lib.h" 14 #include "ice_dcb_nl.h" 15 #include "ice_devlink.h" 16 17 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 18 static const char ice_driver_string[] = DRV_SUMMARY; 19 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 20 21 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 22 #define ICE_DDP_PKG_PATH "intel/ice/ddp/" 23 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 24 25 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 26 MODULE_DESCRIPTION(DRV_SUMMARY); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 29 30 static int debug = -1; 31 module_param(debug, int, 0644); 32 #ifndef CONFIG_DYNAMIC_DEBUG 33 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 34 #else 35 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 36 #endif /* !CONFIG_DYNAMIC_DEBUG */ 37 38 static struct workqueue_struct *ice_wq; 39 static const struct net_device_ops ice_netdev_safe_mode_ops; 40 static const struct net_device_ops ice_netdev_ops; 41 static int ice_vsi_open(struct ice_vsi *vsi); 42 43 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 44 45 static void ice_vsi_release_all(struct ice_pf *pf); 46 47 /** 48 * ice_get_tx_pending - returns number of Tx descriptors not processed 49 * @ring: the ring of descriptors 50 */ 51 static u16 ice_get_tx_pending(struct ice_ring *ring) 52 { 53 u16 head, tail; 54 55 head = ring->next_to_clean; 56 tail = ring->next_to_use; 57 58 if (head != tail) 59 return (head < tail) ? 60 tail - head : (tail + ring->count - head); 61 return 0; 62 } 63 64 /** 65 * ice_check_for_hang_subtask - check for and recover hung queues 66 * @pf: pointer to PF struct 67 */ 68 static void ice_check_for_hang_subtask(struct ice_pf *pf) 69 { 70 struct ice_vsi *vsi = NULL; 71 struct ice_hw *hw; 72 unsigned int i; 73 int packets; 74 u32 v; 75 76 ice_for_each_vsi(pf, v) 77 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 78 vsi = pf->vsi[v]; 79 break; 80 } 81 82 if (!vsi || test_bit(__ICE_DOWN, vsi->state)) 83 return; 84 85 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 86 return; 87 88 hw = &vsi->back->hw; 89 90 for (i = 0; i < vsi->num_txq; i++) { 91 struct ice_ring *tx_ring = vsi->tx_rings[i]; 92 93 if (tx_ring && tx_ring->desc) { 94 /* If packet counter has not changed the queue is 95 * likely stalled, so force an interrupt for this 96 * queue. 97 * 98 * prev_pkt would be negative if there was no 99 * pending work. 100 */ 101 packets = tx_ring->stats.pkts & INT_MAX; 102 if (tx_ring->tx_stats.prev_pkt == packets) { 103 /* Trigger sw interrupt to revive the queue */ 104 ice_trigger_sw_intr(hw, tx_ring->q_vector); 105 continue; 106 } 107 108 /* Memory barrier between read of packet count and call 109 * to ice_get_tx_pending() 110 */ 111 smp_rmb(); 112 tx_ring->tx_stats.prev_pkt = 113 ice_get_tx_pending(tx_ring) ? packets : -1; 114 } 115 } 116 } 117 118 /** 119 * ice_init_mac_fltr - Set initial MAC filters 120 * @pf: board private structure 121 * 122 * Set initial set of MAC filters for PF VSI; configure filters for permanent 123 * address and broadcast address. If an error is encountered, netdevice will be 124 * unregistered. 125 */ 126 static int ice_init_mac_fltr(struct ice_pf *pf) 127 { 128 enum ice_status status; 129 struct ice_vsi *vsi; 130 u8 *perm_addr; 131 132 vsi = ice_get_main_vsi(pf); 133 if (!vsi) 134 return -EINVAL; 135 136 perm_addr = vsi->port_info->mac.perm_addr; 137 status = ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 138 if (!status) 139 return 0; 140 141 /* We aren't useful with no MAC filters, so unregister if we 142 * had an error 143 */ 144 if (vsi->netdev->reg_state == NETREG_REGISTERED) { 145 dev_err(ice_pf_to_dev(pf), "Could not add MAC filters error %s. Unregistering device\n", 146 ice_stat_str(status)); 147 unregister_netdev(vsi->netdev); 148 free_netdev(vsi->netdev); 149 vsi->netdev = NULL; 150 } 151 152 return -EIO; 153 } 154 155 /** 156 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 157 * @netdev: the net device on which the sync is happening 158 * @addr: MAC address to sync 159 * 160 * This is a callback function which is called by the in kernel device sync 161 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 162 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 163 * MAC filters from the hardware. 164 */ 165 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 166 { 167 struct ice_netdev_priv *np = netdev_priv(netdev); 168 struct ice_vsi *vsi = np->vsi; 169 170 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 171 ICE_FWD_TO_VSI)) 172 return -EINVAL; 173 174 return 0; 175 } 176 177 /** 178 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 179 * @netdev: the net device on which the unsync is happening 180 * @addr: MAC address to unsync 181 * 182 * This is a callback function which is called by the in kernel device unsync 183 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 184 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 185 * delete the MAC filters from the hardware. 186 */ 187 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 188 { 189 struct ice_netdev_priv *np = netdev_priv(netdev); 190 struct ice_vsi *vsi = np->vsi; 191 192 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 193 ICE_FWD_TO_VSI)) 194 return -EINVAL; 195 196 return 0; 197 } 198 199 /** 200 * ice_vsi_fltr_changed - check if filter state changed 201 * @vsi: VSI to be checked 202 * 203 * returns true if filter state has changed, false otherwise. 204 */ 205 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 206 { 207 return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) || 208 test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) || 209 test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 210 } 211 212 /** 213 * ice_cfg_promisc - Enable or disable promiscuous mode for a given PF 214 * @vsi: the VSI being configured 215 * @promisc_m: mask of promiscuous config bits 216 * @set_promisc: enable or disable promisc flag request 217 * 218 */ 219 static int ice_cfg_promisc(struct ice_vsi *vsi, u8 promisc_m, bool set_promisc) 220 { 221 struct ice_hw *hw = &vsi->back->hw; 222 enum ice_status status = 0; 223 224 if (vsi->type != ICE_VSI_PF) 225 return 0; 226 227 if (vsi->vlan_ena) { 228 status = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_m, 229 set_promisc); 230 } else { 231 if (set_promisc) 232 status = ice_set_vsi_promisc(hw, vsi->idx, promisc_m, 233 0); 234 else 235 status = ice_clear_vsi_promisc(hw, vsi->idx, promisc_m, 236 0); 237 } 238 239 if (status) 240 return -EIO; 241 242 return 0; 243 } 244 245 /** 246 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 247 * @vsi: ptr to the VSI 248 * 249 * Push any outstanding VSI filter changes through the AdminQ. 250 */ 251 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 252 { 253 struct device *dev = ice_pf_to_dev(vsi->back); 254 struct net_device *netdev = vsi->netdev; 255 bool promisc_forced_on = false; 256 struct ice_pf *pf = vsi->back; 257 struct ice_hw *hw = &pf->hw; 258 enum ice_status status = 0; 259 u32 changed_flags = 0; 260 u8 promisc_m; 261 int err = 0; 262 263 if (!vsi->netdev) 264 return -EINVAL; 265 266 while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) 267 usleep_range(1000, 2000); 268 269 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 270 vsi->current_netdev_flags = vsi->netdev->flags; 271 272 INIT_LIST_HEAD(&vsi->tmp_sync_list); 273 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 274 275 if (ice_vsi_fltr_changed(vsi)) { 276 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 277 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 278 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 279 280 /* grab the netdev's addr_list_lock */ 281 netif_addr_lock_bh(netdev); 282 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 283 ice_add_mac_to_unsync_list); 284 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 285 ice_add_mac_to_unsync_list); 286 /* our temp lists are populated. release lock */ 287 netif_addr_unlock_bh(netdev); 288 } 289 290 /* Remove MAC addresses in the unsync list */ 291 status = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 292 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 293 if (status) { 294 netdev_err(netdev, "Failed to delete MAC filters\n"); 295 /* if we failed because of alloc failures, just bail */ 296 if (status == ICE_ERR_NO_MEMORY) { 297 err = -ENOMEM; 298 goto out; 299 } 300 } 301 302 /* Add MAC addresses in the sync list */ 303 status = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 304 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 305 /* If filter is added successfully or already exists, do not go into 306 * 'if' condition and report it as error. Instead continue processing 307 * rest of the function. 308 */ 309 if (status && status != ICE_ERR_ALREADY_EXISTS) { 310 netdev_err(netdev, "Failed to add MAC filters\n"); 311 /* If there is no more space for new umac filters, VSI 312 * should go into promiscuous mode. There should be some 313 * space reserved for promiscuous filters. 314 */ 315 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 316 !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC, 317 vsi->state)) { 318 promisc_forced_on = true; 319 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 320 vsi->vsi_num); 321 } else { 322 err = -EIO; 323 goto out; 324 } 325 } 326 /* check for changes in promiscuous modes */ 327 if (changed_flags & IFF_ALLMULTI) { 328 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 329 if (vsi->vlan_ena) 330 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 331 else 332 promisc_m = ICE_MCAST_PROMISC_BITS; 333 334 err = ice_cfg_promisc(vsi, promisc_m, true); 335 if (err) { 336 netdev_err(netdev, "Error setting Multicast promiscuous mode on VSI %i\n", 337 vsi->vsi_num); 338 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 339 goto out_promisc; 340 } 341 } else { 342 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 343 if (vsi->vlan_ena) 344 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 345 else 346 promisc_m = ICE_MCAST_PROMISC_BITS; 347 348 err = ice_cfg_promisc(vsi, promisc_m, false); 349 if (err) { 350 netdev_err(netdev, "Error clearing Multicast promiscuous mode on VSI %i\n", 351 vsi->vsi_num); 352 vsi->current_netdev_flags |= IFF_ALLMULTI; 353 goto out_promisc; 354 } 355 } 356 } 357 358 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 359 test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) { 360 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 361 if (vsi->current_netdev_flags & IFF_PROMISC) { 362 /* Apply Rx filter rule to get traffic from wire */ 363 if (!ice_is_dflt_vsi_in_use(pf->first_sw)) { 364 err = ice_set_dflt_vsi(pf->first_sw, vsi); 365 if (err && err != -EEXIST) { 366 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 367 err, vsi->vsi_num); 368 vsi->current_netdev_flags &= 369 ~IFF_PROMISC; 370 goto out_promisc; 371 } 372 } 373 } else { 374 /* Clear Rx filter to remove traffic from wire */ 375 if (ice_is_vsi_dflt_vsi(pf->first_sw, vsi)) { 376 err = ice_clear_dflt_vsi(pf->first_sw); 377 if (err) { 378 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 379 err, vsi->vsi_num); 380 vsi->current_netdev_flags |= 381 IFF_PROMISC; 382 goto out_promisc; 383 } 384 } 385 } 386 } 387 goto exit; 388 389 out_promisc: 390 set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 391 goto exit; 392 out: 393 /* if something went wrong then set the changed flag so we try again */ 394 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 395 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 396 exit: 397 clear_bit(__ICE_CFG_BUSY, vsi->state); 398 return err; 399 } 400 401 /** 402 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 403 * @pf: board private structure 404 */ 405 static void ice_sync_fltr_subtask(struct ice_pf *pf) 406 { 407 int v; 408 409 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 410 return; 411 412 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 413 414 ice_for_each_vsi(pf, v) 415 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 416 ice_vsi_sync_fltr(pf->vsi[v])) { 417 /* come back and try again later */ 418 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 419 break; 420 } 421 } 422 423 /** 424 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 425 * @pf: the PF 426 * @locked: is the rtnl_lock already held 427 */ 428 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 429 { 430 int v; 431 432 ice_for_each_vsi(pf, v) 433 if (pf->vsi[v]) 434 ice_dis_vsi(pf->vsi[v], locked); 435 } 436 437 /** 438 * ice_prepare_for_reset - prep for the core to reset 439 * @pf: board private structure 440 * 441 * Inform or close all dependent features in prep for reset. 442 */ 443 static void 444 ice_prepare_for_reset(struct ice_pf *pf) 445 { 446 struct ice_hw *hw = &pf->hw; 447 unsigned int i; 448 449 /* already prepared for reset */ 450 if (test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) 451 return; 452 453 /* Notify VFs of impending reset */ 454 if (ice_check_sq_alive(hw, &hw->mailboxq)) 455 ice_vc_notify_reset(pf); 456 457 /* Disable VFs until reset is completed */ 458 ice_for_each_vf(pf, i) 459 ice_set_vf_state_qs_dis(&pf->vf[i]); 460 461 /* clear SW filtering DB */ 462 ice_clear_hw_tbls(hw); 463 /* disable the VSIs and their queues that are not already DOWN */ 464 ice_pf_dis_all_vsi(pf, false); 465 466 if (hw->port_info) 467 ice_sched_clear_port(hw->port_info); 468 469 ice_shutdown_all_ctrlq(hw); 470 471 set_bit(__ICE_PREPARED_FOR_RESET, pf->state); 472 } 473 474 /** 475 * ice_do_reset - Initiate one of many types of resets 476 * @pf: board private structure 477 * @reset_type: reset type requested 478 * before this function was called. 479 */ 480 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 481 { 482 struct device *dev = ice_pf_to_dev(pf); 483 struct ice_hw *hw = &pf->hw; 484 485 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 486 WARN_ON(in_interrupt()); 487 488 ice_prepare_for_reset(pf); 489 490 /* trigger the reset */ 491 if (ice_reset(hw, reset_type)) { 492 dev_err(dev, "reset %d failed\n", reset_type); 493 set_bit(__ICE_RESET_FAILED, pf->state); 494 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 495 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 496 clear_bit(__ICE_PFR_REQ, pf->state); 497 clear_bit(__ICE_CORER_REQ, pf->state); 498 clear_bit(__ICE_GLOBR_REQ, pf->state); 499 return; 500 } 501 502 /* PFR is a bit of a special case because it doesn't result in an OICR 503 * interrupt. So for PFR, rebuild after the reset and clear the reset- 504 * associated state bits. 505 */ 506 if (reset_type == ICE_RESET_PFR) { 507 pf->pfr_count++; 508 ice_rebuild(pf, reset_type); 509 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 510 clear_bit(__ICE_PFR_REQ, pf->state); 511 ice_reset_all_vfs(pf, true); 512 } 513 } 514 515 /** 516 * ice_reset_subtask - Set up for resetting the device and driver 517 * @pf: board private structure 518 */ 519 static void ice_reset_subtask(struct ice_pf *pf) 520 { 521 enum ice_reset_req reset_type = ICE_RESET_INVAL; 522 523 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 524 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 525 * of reset is pending and sets bits in pf->state indicating the reset 526 * type and __ICE_RESET_OICR_RECV. So, if the latter bit is set 527 * prepare for pending reset if not already (for PF software-initiated 528 * global resets the software should already be prepared for it as 529 * indicated by __ICE_PREPARED_FOR_RESET; for global resets initiated 530 * by firmware or software on other PFs, that bit is not set so prepare 531 * for the reset now), poll for reset done, rebuild and return. 532 */ 533 if (test_bit(__ICE_RESET_OICR_RECV, pf->state)) { 534 /* Perform the largest reset requested */ 535 if (test_and_clear_bit(__ICE_CORER_RECV, pf->state)) 536 reset_type = ICE_RESET_CORER; 537 if (test_and_clear_bit(__ICE_GLOBR_RECV, pf->state)) 538 reset_type = ICE_RESET_GLOBR; 539 if (test_and_clear_bit(__ICE_EMPR_RECV, pf->state)) 540 reset_type = ICE_RESET_EMPR; 541 /* return if no valid reset type requested */ 542 if (reset_type == ICE_RESET_INVAL) 543 return; 544 ice_prepare_for_reset(pf); 545 546 /* make sure we are ready to rebuild */ 547 if (ice_check_reset(&pf->hw)) { 548 set_bit(__ICE_RESET_FAILED, pf->state); 549 } else { 550 /* done with reset. start rebuild */ 551 pf->hw.reset_ongoing = false; 552 ice_rebuild(pf, reset_type); 553 /* clear bit to resume normal operations, but 554 * ICE_NEEDS_RESTART bit is set in case rebuild failed 555 */ 556 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 557 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 558 clear_bit(__ICE_PFR_REQ, pf->state); 559 clear_bit(__ICE_CORER_REQ, pf->state); 560 clear_bit(__ICE_GLOBR_REQ, pf->state); 561 ice_reset_all_vfs(pf, true); 562 } 563 564 return; 565 } 566 567 /* No pending resets to finish processing. Check for new resets */ 568 if (test_bit(__ICE_PFR_REQ, pf->state)) 569 reset_type = ICE_RESET_PFR; 570 if (test_bit(__ICE_CORER_REQ, pf->state)) 571 reset_type = ICE_RESET_CORER; 572 if (test_bit(__ICE_GLOBR_REQ, pf->state)) 573 reset_type = ICE_RESET_GLOBR; 574 /* If no valid reset type requested just return */ 575 if (reset_type == ICE_RESET_INVAL) 576 return; 577 578 /* reset if not already down or busy */ 579 if (!test_bit(__ICE_DOWN, pf->state) && 580 !test_bit(__ICE_CFG_BUSY, pf->state)) { 581 ice_do_reset(pf, reset_type); 582 } 583 } 584 585 /** 586 * ice_print_topo_conflict - print topology conflict message 587 * @vsi: the VSI whose topology status is being checked 588 */ 589 static void ice_print_topo_conflict(struct ice_vsi *vsi) 590 { 591 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 592 case ICE_AQ_LINK_TOPO_CONFLICT: 593 case ICE_AQ_LINK_MEDIA_CONFLICT: 594 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 595 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 596 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 597 netdev_info(vsi->netdev, "Possible mis-configuration of the Ethernet port detected, please use the Intel(R) Ethernet Port Configuration Tool application to address the issue.\n"); 598 break; 599 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 600 netdev_info(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 601 break; 602 default: 603 break; 604 } 605 } 606 607 /** 608 * ice_print_link_msg - print link up or down message 609 * @vsi: the VSI whose link status is being queried 610 * @isup: boolean for if the link is now up or down 611 */ 612 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 613 { 614 struct ice_aqc_get_phy_caps_data *caps; 615 const char *an_advertised; 616 enum ice_status status; 617 const char *fec_req; 618 const char *speed; 619 const char *fec; 620 const char *fc; 621 const char *an; 622 623 if (!vsi) 624 return; 625 626 if (vsi->current_isup == isup) 627 return; 628 629 vsi->current_isup = isup; 630 631 if (!isup) { 632 netdev_info(vsi->netdev, "NIC Link is Down\n"); 633 return; 634 } 635 636 switch (vsi->port_info->phy.link_info.link_speed) { 637 case ICE_AQ_LINK_SPEED_100GB: 638 speed = "100 G"; 639 break; 640 case ICE_AQ_LINK_SPEED_50GB: 641 speed = "50 G"; 642 break; 643 case ICE_AQ_LINK_SPEED_40GB: 644 speed = "40 G"; 645 break; 646 case ICE_AQ_LINK_SPEED_25GB: 647 speed = "25 G"; 648 break; 649 case ICE_AQ_LINK_SPEED_20GB: 650 speed = "20 G"; 651 break; 652 case ICE_AQ_LINK_SPEED_10GB: 653 speed = "10 G"; 654 break; 655 case ICE_AQ_LINK_SPEED_5GB: 656 speed = "5 G"; 657 break; 658 case ICE_AQ_LINK_SPEED_2500MB: 659 speed = "2.5 G"; 660 break; 661 case ICE_AQ_LINK_SPEED_1000MB: 662 speed = "1 G"; 663 break; 664 case ICE_AQ_LINK_SPEED_100MB: 665 speed = "100 M"; 666 break; 667 default: 668 speed = "Unknown"; 669 break; 670 } 671 672 switch (vsi->port_info->fc.current_mode) { 673 case ICE_FC_FULL: 674 fc = "Rx/Tx"; 675 break; 676 case ICE_FC_TX_PAUSE: 677 fc = "Tx"; 678 break; 679 case ICE_FC_RX_PAUSE: 680 fc = "Rx"; 681 break; 682 case ICE_FC_NONE: 683 fc = "None"; 684 break; 685 default: 686 fc = "Unknown"; 687 break; 688 } 689 690 /* Get FEC mode based on negotiated link info */ 691 switch (vsi->port_info->phy.link_info.fec_info) { 692 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 693 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 694 fec = "RS-FEC"; 695 break; 696 case ICE_AQ_LINK_25G_KR_FEC_EN: 697 fec = "FC-FEC/BASE-R"; 698 break; 699 default: 700 fec = "NONE"; 701 break; 702 } 703 704 /* check if autoneg completed, might be false due to not supported */ 705 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 706 an = "True"; 707 else 708 an = "False"; 709 710 /* Get FEC mode requested based on PHY caps last SW configuration */ 711 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 712 if (!caps) { 713 fec_req = "Unknown"; 714 an_advertised = "Unknown"; 715 goto done; 716 } 717 718 status = ice_aq_get_phy_caps(vsi->port_info, false, 719 ICE_AQC_REPORT_SW_CFG, caps, NULL); 720 if (status) 721 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 722 723 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 724 725 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 726 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 727 fec_req = "RS-FEC"; 728 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 729 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 730 fec_req = "FC-FEC/BASE-R"; 731 else 732 fec_req = "NONE"; 733 734 kfree(caps); 735 736 done: 737 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 738 speed, fec_req, fec, an_advertised, an, fc); 739 ice_print_topo_conflict(vsi); 740 } 741 742 /** 743 * ice_vsi_link_event - update the VSI's netdev 744 * @vsi: the VSI on which the link event occurred 745 * @link_up: whether or not the VSI needs to be set up or down 746 */ 747 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 748 { 749 if (!vsi) 750 return; 751 752 if (test_bit(__ICE_DOWN, vsi->state) || !vsi->netdev) 753 return; 754 755 if (vsi->type == ICE_VSI_PF) { 756 if (link_up == netif_carrier_ok(vsi->netdev)) 757 return; 758 759 if (link_up) { 760 netif_carrier_on(vsi->netdev); 761 netif_tx_wake_all_queues(vsi->netdev); 762 } else { 763 netif_carrier_off(vsi->netdev); 764 netif_tx_stop_all_queues(vsi->netdev); 765 } 766 } 767 } 768 769 /** 770 * ice_link_event - process the link event 771 * @pf: PF that the link event is associated with 772 * @pi: port_info for the port that the link event is associated with 773 * @link_up: true if the physical link is up and false if it is down 774 * @link_speed: current link speed received from the link event 775 * 776 * Returns 0 on success and negative on failure 777 */ 778 static int 779 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 780 u16 link_speed) 781 { 782 struct device *dev = ice_pf_to_dev(pf); 783 struct ice_phy_info *phy_info; 784 struct ice_vsi *vsi; 785 u16 old_link_speed; 786 bool old_link; 787 int result; 788 789 phy_info = &pi->phy; 790 phy_info->link_info_old = phy_info->link_info; 791 792 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 793 old_link_speed = phy_info->link_info_old.link_speed; 794 795 /* update the link info structures and re-enable link events, 796 * don't bail on failure due to other book keeping needed 797 */ 798 result = ice_update_link_info(pi); 799 if (result) 800 dev_dbg(dev, "Failed to update link status and re-enable link events for port %d\n", 801 pi->lport); 802 803 vsi = ice_get_main_vsi(pf); 804 if (!vsi || !vsi->port_info) 805 return -EINVAL; 806 807 /* turn off PHY if media was removed */ 808 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 809 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 810 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 811 812 result = ice_aq_set_link_restart_an(pi, false, NULL); 813 if (result) { 814 dev_dbg(dev, "Failed to set link down, VSI %d error %d\n", 815 vsi->vsi_num, result); 816 return result; 817 } 818 } 819 820 /* if the old link up/down and speed is the same as the new */ 821 if (link_up == old_link && link_speed == old_link_speed) 822 return result; 823 824 ice_dcb_rebuild(pf); 825 ice_vsi_link_event(vsi, link_up); 826 ice_print_link_msg(vsi, link_up); 827 828 ice_vc_notify_link_state(pf); 829 830 return result; 831 } 832 833 /** 834 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 835 * @pf: board private structure 836 */ 837 static void ice_watchdog_subtask(struct ice_pf *pf) 838 { 839 int i; 840 841 /* if interface is down do nothing */ 842 if (test_bit(__ICE_DOWN, pf->state) || 843 test_bit(__ICE_CFG_BUSY, pf->state)) 844 return; 845 846 /* make sure we don't do these things too often */ 847 if (time_before(jiffies, 848 pf->serv_tmr_prev + pf->serv_tmr_period)) 849 return; 850 851 pf->serv_tmr_prev = jiffies; 852 853 /* Update the stats for active netdevs so the network stack 854 * can look at updated numbers whenever it cares to 855 */ 856 ice_update_pf_stats(pf); 857 ice_for_each_vsi(pf, i) 858 if (pf->vsi[i] && pf->vsi[i]->netdev) 859 ice_update_vsi_stats(pf->vsi[i]); 860 } 861 862 /** 863 * ice_init_link_events - enable/initialize link events 864 * @pi: pointer to the port_info instance 865 * 866 * Returns -EIO on failure, 0 on success 867 */ 868 static int ice_init_link_events(struct ice_port_info *pi) 869 { 870 u16 mask; 871 872 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 873 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL)); 874 875 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 876 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 877 pi->lport); 878 return -EIO; 879 } 880 881 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 882 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 883 pi->lport); 884 return -EIO; 885 } 886 887 return 0; 888 } 889 890 /** 891 * ice_handle_link_event - handle link event via ARQ 892 * @pf: PF that the link event is associated with 893 * @event: event structure containing link status info 894 */ 895 static int 896 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 897 { 898 struct ice_aqc_get_link_status_data *link_data; 899 struct ice_port_info *port_info; 900 int status; 901 902 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 903 port_info = pf->hw.port_info; 904 if (!port_info) 905 return -EINVAL; 906 907 status = ice_link_event(pf, port_info, 908 !!(link_data->link_info & ICE_AQ_LINK_UP), 909 le16_to_cpu(link_data->link_speed)); 910 if (status) 911 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 912 status); 913 914 return status; 915 } 916 917 /** 918 * __ice_clean_ctrlq - helper function to clean controlq rings 919 * @pf: ptr to struct ice_pf 920 * @q_type: specific Control queue type 921 */ 922 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 923 { 924 struct device *dev = ice_pf_to_dev(pf); 925 struct ice_rq_event_info event; 926 struct ice_hw *hw = &pf->hw; 927 struct ice_ctl_q_info *cq; 928 u16 pending, i = 0; 929 const char *qtype; 930 u32 oldval, val; 931 932 /* Do not clean control queue if/when PF reset fails */ 933 if (test_bit(__ICE_RESET_FAILED, pf->state)) 934 return 0; 935 936 switch (q_type) { 937 case ICE_CTL_Q_ADMIN: 938 cq = &hw->adminq; 939 qtype = "Admin"; 940 break; 941 case ICE_CTL_Q_MAILBOX: 942 cq = &hw->mailboxq; 943 qtype = "Mailbox"; 944 break; 945 default: 946 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 947 return 0; 948 } 949 950 /* check for error indications - PF_xx_AxQLEN register layout for 951 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 952 */ 953 val = rd32(hw, cq->rq.len); 954 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 955 PF_FW_ARQLEN_ARQCRIT_M)) { 956 oldval = val; 957 if (val & PF_FW_ARQLEN_ARQVFE_M) 958 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 959 qtype); 960 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 961 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 962 qtype); 963 } 964 if (val & PF_FW_ARQLEN_ARQCRIT_M) 965 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 966 qtype); 967 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 968 PF_FW_ARQLEN_ARQCRIT_M); 969 if (oldval != val) 970 wr32(hw, cq->rq.len, val); 971 } 972 973 val = rd32(hw, cq->sq.len); 974 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 975 PF_FW_ATQLEN_ATQCRIT_M)) { 976 oldval = val; 977 if (val & PF_FW_ATQLEN_ATQVFE_M) 978 dev_dbg(dev, "%s Send Queue VF Error detected\n", 979 qtype); 980 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 981 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 982 qtype); 983 } 984 if (val & PF_FW_ATQLEN_ATQCRIT_M) 985 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 986 qtype); 987 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 988 PF_FW_ATQLEN_ATQCRIT_M); 989 if (oldval != val) 990 wr32(hw, cq->sq.len, val); 991 } 992 993 event.buf_len = cq->rq_buf_size; 994 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 995 if (!event.msg_buf) 996 return 0; 997 998 do { 999 enum ice_status ret; 1000 u16 opcode; 1001 1002 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1003 if (ret == ICE_ERR_AQ_NO_WORK) 1004 break; 1005 if (ret) { 1006 dev_err(dev, "%s Receive Queue event error %s\n", qtype, 1007 ice_stat_str(ret)); 1008 break; 1009 } 1010 1011 opcode = le16_to_cpu(event.desc.opcode); 1012 1013 switch (opcode) { 1014 case ice_aqc_opc_get_link_status: 1015 if (ice_handle_link_event(pf, &event)) 1016 dev_err(dev, "Could not handle link event\n"); 1017 break; 1018 case ice_aqc_opc_event_lan_overflow: 1019 ice_vf_lan_overflow_event(pf, &event); 1020 break; 1021 case ice_mbx_opc_send_msg_to_pf: 1022 ice_vc_process_vf_msg(pf, &event); 1023 break; 1024 case ice_aqc_opc_fw_logging: 1025 ice_output_fw_log(hw, &event.desc, event.msg_buf); 1026 break; 1027 case ice_aqc_opc_lldp_set_mib_change: 1028 ice_dcb_process_lldp_set_mib_change(pf, &event); 1029 break; 1030 default: 1031 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1032 qtype, opcode); 1033 break; 1034 } 1035 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1036 1037 kfree(event.msg_buf); 1038 1039 return pending && (i == ICE_DFLT_IRQ_WORK); 1040 } 1041 1042 /** 1043 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1044 * @hw: pointer to hardware info 1045 * @cq: control queue information 1046 * 1047 * returns true if there are pending messages in a queue, false if there aren't 1048 */ 1049 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1050 { 1051 u16 ntu; 1052 1053 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1054 return cq->rq.next_to_clean != ntu; 1055 } 1056 1057 /** 1058 * ice_clean_adminq_subtask - clean the AdminQ rings 1059 * @pf: board private structure 1060 */ 1061 static void ice_clean_adminq_subtask(struct ice_pf *pf) 1062 { 1063 struct ice_hw *hw = &pf->hw; 1064 1065 if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1066 return; 1067 1068 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1069 return; 1070 1071 clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 1072 1073 /* There might be a situation where new messages arrive to a control 1074 * queue between processing the last message and clearing the 1075 * EVENT_PENDING bit. So before exiting, check queue head again (using 1076 * ice_ctrlq_pending) and process new messages if any. 1077 */ 1078 if (ice_ctrlq_pending(hw, &hw->adminq)) 1079 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1080 1081 ice_flush(hw); 1082 } 1083 1084 /** 1085 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1086 * @pf: board private structure 1087 */ 1088 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1089 { 1090 struct ice_hw *hw = &pf->hw; 1091 1092 if (!test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1093 return; 1094 1095 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1096 return; 1097 1098 clear_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1099 1100 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1101 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1102 1103 ice_flush(hw); 1104 } 1105 1106 /** 1107 * ice_service_task_schedule - schedule the service task to wake up 1108 * @pf: board private structure 1109 * 1110 * If not already scheduled, this puts the task into the work queue. 1111 */ 1112 void ice_service_task_schedule(struct ice_pf *pf) 1113 { 1114 if (!test_bit(__ICE_SERVICE_DIS, pf->state) && 1115 !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) && 1116 !test_bit(__ICE_NEEDS_RESTART, pf->state)) 1117 queue_work(ice_wq, &pf->serv_task); 1118 } 1119 1120 /** 1121 * ice_service_task_complete - finish up the service task 1122 * @pf: board private structure 1123 */ 1124 static void ice_service_task_complete(struct ice_pf *pf) 1125 { 1126 WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state)); 1127 1128 /* force memory (pf->state) to sync before next service task */ 1129 smp_mb__before_atomic(); 1130 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1131 } 1132 1133 /** 1134 * ice_service_task_stop - stop service task and cancel works 1135 * @pf: board private structure 1136 * 1137 * Return 0 if the __ICE_SERVICE_DIS bit was not already set, 1138 * 1 otherwise. 1139 */ 1140 static int ice_service_task_stop(struct ice_pf *pf) 1141 { 1142 int ret; 1143 1144 ret = test_and_set_bit(__ICE_SERVICE_DIS, pf->state); 1145 1146 if (pf->serv_tmr.function) 1147 del_timer_sync(&pf->serv_tmr); 1148 if (pf->serv_task.func) 1149 cancel_work_sync(&pf->serv_task); 1150 1151 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1152 return ret; 1153 } 1154 1155 /** 1156 * ice_service_task_restart - restart service task and schedule works 1157 * @pf: board private structure 1158 * 1159 * This function is needed for suspend and resume works (e.g WoL scenario) 1160 */ 1161 static void ice_service_task_restart(struct ice_pf *pf) 1162 { 1163 clear_bit(__ICE_SERVICE_DIS, pf->state); 1164 ice_service_task_schedule(pf); 1165 } 1166 1167 /** 1168 * ice_service_timer - timer callback to schedule service task 1169 * @t: pointer to timer_list 1170 */ 1171 static void ice_service_timer(struct timer_list *t) 1172 { 1173 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1174 1175 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1176 ice_service_task_schedule(pf); 1177 } 1178 1179 /** 1180 * ice_handle_mdd_event - handle malicious driver detect event 1181 * @pf: pointer to the PF structure 1182 * 1183 * Called from service task. OICR interrupt handler indicates MDD event. 1184 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1185 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1186 * disable the queue, the PF can be configured to reset the VF using ethtool 1187 * private flag mdd-auto-reset-vf. 1188 */ 1189 static void ice_handle_mdd_event(struct ice_pf *pf) 1190 { 1191 struct device *dev = ice_pf_to_dev(pf); 1192 struct ice_hw *hw = &pf->hw; 1193 unsigned int i; 1194 u32 reg; 1195 1196 if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state)) { 1197 /* Since the VF MDD event logging is rate limited, check if 1198 * there are pending MDD events. 1199 */ 1200 ice_print_vfs_mdd_events(pf); 1201 return; 1202 } 1203 1204 /* find what triggered an MDD event */ 1205 reg = rd32(hw, GL_MDET_TX_PQM); 1206 if (reg & GL_MDET_TX_PQM_VALID_M) { 1207 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1208 GL_MDET_TX_PQM_PF_NUM_S; 1209 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1210 GL_MDET_TX_PQM_VF_NUM_S; 1211 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1212 GL_MDET_TX_PQM_MAL_TYPE_S; 1213 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1214 GL_MDET_TX_PQM_QNUM_S); 1215 1216 if (netif_msg_tx_err(pf)) 1217 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1218 event, queue, pf_num, vf_num); 1219 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1220 } 1221 1222 reg = rd32(hw, GL_MDET_TX_TCLAN); 1223 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1224 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1225 GL_MDET_TX_TCLAN_PF_NUM_S; 1226 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1227 GL_MDET_TX_TCLAN_VF_NUM_S; 1228 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1229 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1230 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1231 GL_MDET_TX_TCLAN_QNUM_S); 1232 1233 if (netif_msg_tx_err(pf)) 1234 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1235 event, queue, pf_num, vf_num); 1236 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1237 } 1238 1239 reg = rd32(hw, GL_MDET_RX); 1240 if (reg & GL_MDET_RX_VALID_M) { 1241 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1242 GL_MDET_RX_PF_NUM_S; 1243 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1244 GL_MDET_RX_VF_NUM_S; 1245 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1246 GL_MDET_RX_MAL_TYPE_S; 1247 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1248 GL_MDET_RX_QNUM_S); 1249 1250 if (netif_msg_rx_err(pf)) 1251 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1252 event, queue, pf_num, vf_num); 1253 wr32(hw, GL_MDET_RX, 0xffffffff); 1254 } 1255 1256 /* check to see if this PF caused an MDD event */ 1257 reg = rd32(hw, PF_MDET_TX_PQM); 1258 if (reg & PF_MDET_TX_PQM_VALID_M) { 1259 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1260 if (netif_msg_tx_err(pf)) 1261 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1262 } 1263 1264 reg = rd32(hw, PF_MDET_TX_TCLAN); 1265 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1266 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1267 if (netif_msg_tx_err(pf)) 1268 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1269 } 1270 1271 reg = rd32(hw, PF_MDET_RX); 1272 if (reg & PF_MDET_RX_VALID_M) { 1273 wr32(hw, PF_MDET_RX, 0xFFFF); 1274 if (netif_msg_rx_err(pf)) 1275 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1276 } 1277 1278 /* Check to see if one of the VFs caused an MDD event, and then 1279 * increment counters and set print pending 1280 */ 1281 ice_for_each_vf(pf, i) { 1282 struct ice_vf *vf = &pf->vf[i]; 1283 1284 reg = rd32(hw, VP_MDET_TX_PQM(i)); 1285 if (reg & VP_MDET_TX_PQM_VALID_M) { 1286 wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF); 1287 vf->mdd_tx_events.count++; 1288 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1289 if (netif_msg_tx_err(pf)) 1290 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1291 i); 1292 } 1293 1294 reg = rd32(hw, VP_MDET_TX_TCLAN(i)); 1295 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1296 wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF); 1297 vf->mdd_tx_events.count++; 1298 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1299 if (netif_msg_tx_err(pf)) 1300 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1301 i); 1302 } 1303 1304 reg = rd32(hw, VP_MDET_TX_TDPU(i)); 1305 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1306 wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF); 1307 vf->mdd_tx_events.count++; 1308 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1309 if (netif_msg_tx_err(pf)) 1310 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1311 i); 1312 } 1313 1314 reg = rd32(hw, VP_MDET_RX(i)); 1315 if (reg & VP_MDET_RX_VALID_M) { 1316 wr32(hw, VP_MDET_RX(i), 0xFFFF); 1317 vf->mdd_rx_events.count++; 1318 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1319 if (netif_msg_rx_err(pf)) 1320 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1321 i); 1322 1323 /* Since the queue is disabled on VF Rx MDD events, the 1324 * PF can be configured to reset the VF through ethtool 1325 * private flag mdd-auto-reset-vf. 1326 */ 1327 if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 1328 /* VF MDD event counters will be cleared by 1329 * reset, so print the event prior to reset. 1330 */ 1331 ice_print_vf_rx_mdd_event(vf); 1332 ice_reset_vf(&pf->vf[i], false); 1333 } 1334 } 1335 } 1336 1337 ice_print_vfs_mdd_events(pf); 1338 } 1339 1340 /** 1341 * ice_force_phys_link_state - Force the physical link state 1342 * @vsi: VSI to force the physical link state to up/down 1343 * @link_up: true/false indicates to set the physical link to up/down 1344 * 1345 * Force the physical link state by getting the current PHY capabilities from 1346 * hardware and setting the PHY config based on the determined capabilities. If 1347 * link changes a link event will be triggered because both the Enable Automatic 1348 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1349 * 1350 * Returns 0 on success, negative on failure 1351 */ 1352 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1353 { 1354 struct ice_aqc_get_phy_caps_data *pcaps; 1355 struct ice_aqc_set_phy_cfg_data *cfg; 1356 struct ice_port_info *pi; 1357 struct device *dev; 1358 int retcode; 1359 1360 if (!vsi || !vsi->port_info || !vsi->back) 1361 return -EINVAL; 1362 if (vsi->type != ICE_VSI_PF) 1363 return 0; 1364 1365 dev = ice_pf_to_dev(vsi->back); 1366 1367 pi = vsi->port_info; 1368 1369 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1370 if (!pcaps) 1371 return -ENOMEM; 1372 1373 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1374 NULL); 1375 if (retcode) { 1376 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1377 vsi->vsi_num, retcode); 1378 retcode = -EIO; 1379 goto out; 1380 } 1381 1382 /* No change in link */ 1383 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1384 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1385 goto out; 1386 1387 /* Use the current user PHY configuration. The current user PHY 1388 * configuration is initialized during probe from PHY capabilities 1389 * software mode, and updated on set PHY configuration. 1390 */ 1391 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1392 if (!cfg) { 1393 retcode = -ENOMEM; 1394 goto out; 1395 } 1396 1397 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1398 if (link_up) 1399 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1400 else 1401 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1402 1403 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1404 if (retcode) { 1405 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1406 vsi->vsi_num, retcode); 1407 retcode = -EIO; 1408 } 1409 1410 kfree(cfg); 1411 out: 1412 kfree(pcaps); 1413 return retcode; 1414 } 1415 1416 /** 1417 * ice_init_nvm_phy_type - Initialize the NVM PHY type 1418 * @pi: port info structure 1419 * 1420 * Initialize nvm_phy_type_[low|high] for link lenient mode support 1421 */ 1422 static int ice_init_nvm_phy_type(struct ice_port_info *pi) 1423 { 1424 struct ice_aqc_get_phy_caps_data *pcaps; 1425 struct ice_pf *pf = pi->hw->back; 1426 enum ice_status status; 1427 int err = 0; 1428 1429 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1430 if (!pcaps) 1431 return -ENOMEM; 1432 1433 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_NVM_CAP, pcaps, 1434 NULL); 1435 1436 if (status) { 1437 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1438 err = -EIO; 1439 goto out; 1440 } 1441 1442 pf->nvm_phy_type_hi = pcaps->phy_type_high; 1443 pf->nvm_phy_type_lo = pcaps->phy_type_low; 1444 1445 out: 1446 kfree(pcaps); 1447 return err; 1448 } 1449 1450 /** 1451 * ice_init_link_dflt_override - Initialize link default override 1452 * @pi: port info structure 1453 * 1454 * Initialize link default override and PHY total port shutdown during probe 1455 */ 1456 static void ice_init_link_dflt_override(struct ice_port_info *pi) 1457 { 1458 struct ice_link_default_override_tlv *ldo; 1459 struct ice_pf *pf = pi->hw->back; 1460 1461 ldo = &pf->link_dflt_override; 1462 if (ice_get_link_default_override(ldo, pi)) 1463 return; 1464 1465 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 1466 return; 1467 1468 /* Enable Total Port Shutdown (override/replace link-down-on-close 1469 * ethtool private flag) for ports with Port Disable bit set. 1470 */ 1471 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 1472 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 1473 } 1474 1475 /** 1476 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 1477 * @pi: port info structure 1478 * 1479 * If default override is enabled, initialized the user PHY cfg speed and FEC 1480 * settings using the default override mask from the NVM. 1481 * 1482 * The PHY should only be configured with the default override settings the 1483 * first time media is available. The __ICE_LINK_DEFAULT_OVERRIDE_PENDING state 1484 * is used to indicate that the user PHY cfg default override is initialized 1485 * and the PHY has not been configured with the default override settings. The 1486 * state is set here, and cleared in ice_configure_phy the first time the PHY is 1487 * configured. 1488 */ 1489 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 1490 { 1491 struct ice_link_default_override_tlv *ldo; 1492 struct ice_aqc_set_phy_cfg_data *cfg; 1493 struct ice_phy_info *phy = &pi->phy; 1494 struct ice_pf *pf = pi->hw->back; 1495 1496 ldo = &pf->link_dflt_override; 1497 1498 /* If link default override is enabled, use to mask NVM PHY capabilities 1499 * for speed and FEC default configuration. 1500 */ 1501 cfg = &phy->curr_user_phy_cfg; 1502 1503 if (ldo->phy_type_low || ldo->phy_type_high) { 1504 cfg->phy_type_low = pf->nvm_phy_type_lo & 1505 cpu_to_le64(ldo->phy_type_low); 1506 cfg->phy_type_high = pf->nvm_phy_type_hi & 1507 cpu_to_le64(ldo->phy_type_high); 1508 } 1509 cfg->link_fec_opt = ldo->fec_options; 1510 phy->curr_user_fec_req = ICE_FEC_AUTO; 1511 1512 set_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 1513 } 1514 1515 /** 1516 * ice_init_phy_user_cfg - Initialize the PHY user configuration 1517 * @pi: port info structure 1518 * 1519 * Initialize the current user PHY configuration, speed, FEC, and FC requested 1520 * mode to default. The PHY defaults are from get PHY capabilities topology 1521 * with media so call when media is first available. An error is returned if 1522 * called when media is not available. The PHY initialization completed state is 1523 * set here. 1524 * 1525 * These configurations are used when setting PHY 1526 * configuration. The user PHY configuration is updated on set PHY 1527 * configuration. Returns 0 on success, negative on failure 1528 */ 1529 static int ice_init_phy_user_cfg(struct ice_port_info *pi) 1530 { 1531 struct ice_aqc_get_phy_caps_data *pcaps; 1532 struct ice_phy_info *phy = &pi->phy; 1533 struct ice_pf *pf = pi->hw->back; 1534 enum ice_status status; 1535 struct ice_vsi *vsi; 1536 int err = 0; 1537 1538 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1539 return -EIO; 1540 1541 vsi = ice_get_main_vsi(pf); 1542 if (!vsi) 1543 return -EINVAL; 1544 1545 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1546 if (!pcaps) 1547 return -ENOMEM; 1548 1549 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1550 NULL); 1551 if (status) { 1552 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1553 err = -EIO; 1554 goto err_out; 1555 } 1556 1557 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 1558 1559 /* check if lenient mode is supported and enabled */ 1560 if (ice_fw_supports_link_override(&vsi->back->hw) && 1561 !(pcaps->module_compliance_enforcement & 1562 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 1563 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 1564 1565 /* if link default override is enabled, initialize user PHY 1566 * configuration with link default override values 1567 */ 1568 if (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN) { 1569 ice_init_phy_cfg_dflt_override(pi); 1570 goto out; 1571 } 1572 } 1573 1574 /* if link default override is not enabled, initialize PHY using 1575 * topology with media 1576 */ 1577 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 1578 pcaps->link_fec_options); 1579 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 1580 1581 out: 1582 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 1583 set_bit(__ICE_PHY_INIT_COMPLETE, pf->state); 1584 err_out: 1585 kfree(pcaps); 1586 return err; 1587 } 1588 1589 /** 1590 * ice_configure_phy - configure PHY 1591 * @vsi: VSI of PHY 1592 * 1593 * Set the PHY configuration. If the current PHY configuration is the same as 1594 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 1595 * configure the based get PHY capabilities for topology with media. 1596 */ 1597 static int ice_configure_phy(struct ice_vsi *vsi) 1598 { 1599 struct device *dev = ice_pf_to_dev(vsi->back); 1600 struct ice_aqc_get_phy_caps_data *pcaps; 1601 struct ice_aqc_set_phy_cfg_data *cfg; 1602 struct ice_port_info *pi; 1603 enum ice_status status; 1604 int err = 0; 1605 1606 pi = vsi->port_info; 1607 if (!pi) 1608 return -EINVAL; 1609 1610 /* Ensure we have media as we cannot configure a medialess port */ 1611 if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1612 return -EPERM; 1613 1614 ice_print_topo_conflict(vsi); 1615 1616 if (vsi->port_info->phy.link_info.topo_media_conflict == 1617 ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 1618 return -EPERM; 1619 1620 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 1621 return ice_force_phys_link_state(vsi, true); 1622 1623 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1624 if (!pcaps) 1625 return -ENOMEM; 1626 1627 /* Get current PHY config */ 1628 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1629 NULL); 1630 if (status) { 1631 dev_err(dev, "Failed to get PHY configuration, VSI %d error %s\n", 1632 vsi->vsi_num, ice_stat_str(status)); 1633 err = -EIO; 1634 goto done; 1635 } 1636 1637 /* If PHY enable link is configured and configuration has not changed, 1638 * there's nothing to do 1639 */ 1640 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 1641 ice_phy_caps_equals_cfg(pcaps, &pi->phy.curr_user_phy_cfg)) 1642 goto done; 1643 1644 /* Use PHY topology as baseline for configuration */ 1645 memset(pcaps, 0, sizeof(*pcaps)); 1646 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1647 NULL); 1648 if (status) { 1649 dev_err(dev, "Failed to get PHY topology, VSI %d error %s\n", 1650 vsi->vsi_num, ice_stat_str(status)); 1651 err = -EIO; 1652 goto done; 1653 } 1654 1655 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 1656 if (!cfg) { 1657 err = -ENOMEM; 1658 goto done; 1659 } 1660 1661 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 1662 1663 /* Speed - If default override pending, use curr_user_phy_cfg set in 1664 * ice_init_phy_user_cfg_ldo. 1665 */ 1666 if (test_and_clear_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, 1667 vsi->back->state)) { 1668 cfg->phy_type_low = pi->phy.curr_user_phy_cfg.phy_type_low; 1669 cfg->phy_type_high = pi->phy.curr_user_phy_cfg.phy_type_high; 1670 } else { 1671 u64 phy_low = 0, phy_high = 0; 1672 1673 ice_update_phy_type(&phy_low, &phy_high, 1674 pi->phy.curr_user_speed_req); 1675 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 1676 cfg->phy_type_high = pcaps->phy_type_high & 1677 cpu_to_le64(phy_high); 1678 } 1679 1680 /* Can't provide what was requested; use PHY capabilities */ 1681 if (!cfg->phy_type_low && !cfg->phy_type_high) { 1682 cfg->phy_type_low = pcaps->phy_type_low; 1683 cfg->phy_type_high = pcaps->phy_type_high; 1684 } 1685 1686 /* FEC */ 1687 ice_cfg_phy_fec(pi, cfg, pi->phy.curr_user_fec_req); 1688 1689 /* Can't provide what was requested; use PHY capabilities */ 1690 if (cfg->link_fec_opt != 1691 (cfg->link_fec_opt & pcaps->link_fec_options)) { 1692 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 1693 cfg->link_fec_opt = pcaps->link_fec_options; 1694 } 1695 1696 /* Flow Control - always supported; no need to check against 1697 * capabilities 1698 */ 1699 ice_cfg_phy_fc(pi, cfg, pi->phy.curr_user_fc_req); 1700 1701 /* Enable link and link update */ 1702 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 1703 1704 status = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1705 if (status) { 1706 dev_err(dev, "Failed to set phy config, VSI %d error %s\n", 1707 vsi->vsi_num, ice_stat_str(status)); 1708 err = -EIO; 1709 } 1710 1711 kfree(cfg); 1712 done: 1713 kfree(pcaps); 1714 return err; 1715 } 1716 1717 /** 1718 * ice_check_media_subtask - Check for media 1719 * @pf: pointer to PF struct 1720 * 1721 * If media is available, then initialize PHY user configuration if it is not 1722 * been, and configure the PHY if the interface is up. 1723 */ 1724 static void ice_check_media_subtask(struct ice_pf *pf) 1725 { 1726 struct ice_port_info *pi; 1727 struct ice_vsi *vsi; 1728 int err; 1729 1730 /* No need to check for media if it's already present */ 1731 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 1732 return; 1733 1734 vsi = ice_get_main_vsi(pf); 1735 if (!vsi) 1736 return; 1737 1738 /* Refresh link info and check if media is present */ 1739 pi = vsi->port_info; 1740 err = ice_update_link_info(pi); 1741 if (err) 1742 return; 1743 1744 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 1745 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) 1746 ice_init_phy_user_cfg(pi); 1747 1748 /* PHY settings are reset on media insertion, reconfigure 1749 * PHY to preserve settings. 1750 */ 1751 if (test_bit(__ICE_DOWN, vsi->state) && 1752 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 1753 return; 1754 1755 err = ice_configure_phy(vsi); 1756 if (!err) 1757 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 1758 1759 /* A Link Status Event will be generated; the event handler 1760 * will complete bringing the interface up 1761 */ 1762 } 1763 } 1764 1765 /** 1766 * ice_service_task - manage and run subtasks 1767 * @work: pointer to work_struct contained by the PF struct 1768 */ 1769 static void ice_service_task(struct work_struct *work) 1770 { 1771 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 1772 unsigned long start_time = jiffies; 1773 1774 /* subtasks */ 1775 1776 /* process reset requests first */ 1777 ice_reset_subtask(pf); 1778 1779 /* bail if a reset/recovery cycle is pending or rebuild failed */ 1780 if (ice_is_reset_in_progress(pf->state) || 1781 test_bit(__ICE_SUSPENDED, pf->state) || 1782 test_bit(__ICE_NEEDS_RESTART, pf->state)) { 1783 ice_service_task_complete(pf); 1784 return; 1785 } 1786 1787 ice_clean_adminq_subtask(pf); 1788 ice_check_media_subtask(pf); 1789 ice_check_for_hang_subtask(pf); 1790 ice_sync_fltr_subtask(pf); 1791 ice_handle_mdd_event(pf); 1792 ice_watchdog_subtask(pf); 1793 1794 if (ice_is_safe_mode(pf)) { 1795 ice_service_task_complete(pf); 1796 return; 1797 } 1798 1799 ice_process_vflr_event(pf); 1800 ice_clean_mailboxq_subtask(pf); 1801 ice_sync_arfs_fltrs(pf); 1802 /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */ 1803 ice_service_task_complete(pf); 1804 1805 /* If the tasks have taken longer than one service timer period 1806 * or there is more work to be done, reset the service timer to 1807 * schedule the service task now. 1808 */ 1809 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 1810 test_bit(__ICE_MDD_EVENT_PENDING, pf->state) || 1811 test_bit(__ICE_VFLR_EVENT_PENDING, pf->state) || 1812 test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 1813 test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1814 mod_timer(&pf->serv_tmr, jiffies); 1815 } 1816 1817 /** 1818 * ice_set_ctrlq_len - helper function to set controlq length 1819 * @hw: pointer to the HW instance 1820 */ 1821 static void ice_set_ctrlq_len(struct ice_hw *hw) 1822 { 1823 hw->adminq.num_rq_entries = ICE_AQ_LEN; 1824 hw->adminq.num_sq_entries = ICE_AQ_LEN; 1825 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 1826 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 1827 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 1828 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 1829 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1830 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1831 } 1832 1833 /** 1834 * ice_schedule_reset - schedule a reset 1835 * @pf: board private structure 1836 * @reset: reset being requested 1837 */ 1838 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 1839 { 1840 struct device *dev = ice_pf_to_dev(pf); 1841 1842 /* bail out if earlier reset has failed */ 1843 if (test_bit(__ICE_RESET_FAILED, pf->state)) { 1844 dev_dbg(dev, "earlier reset has failed\n"); 1845 return -EIO; 1846 } 1847 /* bail if reset/recovery already in progress */ 1848 if (ice_is_reset_in_progress(pf->state)) { 1849 dev_dbg(dev, "Reset already in progress\n"); 1850 return -EBUSY; 1851 } 1852 1853 switch (reset) { 1854 case ICE_RESET_PFR: 1855 set_bit(__ICE_PFR_REQ, pf->state); 1856 break; 1857 case ICE_RESET_CORER: 1858 set_bit(__ICE_CORER_REQ, pf->state); 1859 break; 1860 case ICE_RESET_GLOBR: 1861 set_bit(__ICE_GLOBR_REQ, pf->state); 1862 break; 1863 default: 1864 return -EINVAL; 1865 } 1866 1867 ice_service_task_schedule(pf); 1868 return 0; 1869 } 1870 1871 /** 1872 * ice_irq_affinity_notify - Callback for affinity changes 1873 * @notify: context as to what irq was changed 1874 * @mask: the new affinity mask 1875 * 1876 * This is a callback function used by the irq_set_affinity_notifier function 1877 * so that we may register to receive changes to the irq affinity masks. 1878 */ 1879 static void 1880 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 1881 const cpumask_t *mask) 1882 { 1883 struct ice_q_vector *q_vector = 1884 container_of(notify, struct ice_q_vector, affinity_notify); 1885 1886 cpumask_copy(&q_vector->affinity_mask, mask); 1887 } 1888 1889 /** 1890 * ice_irq_affinity_release - Callback for affinity notifier release 1891 * @ref: internal core kernel usage 1892 * 1893 * This is a callback function used by the irq_set_affinity_notifier function 1894 * to inform the current notification subscriber that they will no longer 1895 * receive notifications. 1896 */ 1897 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 1898 1899 /** 1900 * ice_vsi_ena_irq - Enable IRQ for the given VSI 1901 * @vsi: the VSI being configured 1902 */ 1903 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 1904 { 1905 struct ice_hw *hw = &vsi->back->hw; 1906 int i; 1907 1908 ice_for_each_q_vector(vsi, i) 1909 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 1910 1911 ice_flush(hw); 1912 return 0; 1913 } 1914 1915 /** 1916 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 1917 * @vsi: the VSI being configured 1918 * @basename: name for the vector 1919 */ 1920 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 1921 { 1922 int q_vectors = vsi->num_q_vectors; 1923 struct ice_pf *pf = vsi->back; 1924 int base = vsi->base_vector; 1925 struct device *dev; 1926 int rx_int_idx = 0; 1927 int tx_int_idx = 0; 1928 int vector, err; 1929 int irq_num; 1930 1931 dev = ice_pf_to_dev(pf); 1932 for (vector = 0; vector < q_vectors; vector++) { 1933 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 1934 1935 irq_num = pf->msix_entries[base + vector].vector; 1936 1937 if (q_vector->tx.ring && q_vector->rx.ring) { 1938 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1939 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 1940 tx_int_idx++; 1941 } else if (q_vector->rx.ring) { 1942 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1943 "%s-%s-%d", basename, "rx", rx_int_idx++); 1944 } else if (q_vector->tx.ring) { 1945 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1946 "%s-%s-%d", basename, "tx", tx_int_idx++); 1947 } else { 1948 /* skip this unused q_vector */ 1949 continue; 1950 } 1951 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 0, 1952 q_vector->name, q_vector); 1953 if (err) { 1954 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 1955 err); 1956 goto free_q_irqs; 1957 } 1958 1959 /* register for affinity change notifications */ 1960 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { 1961 struct irq_affinity_notify *affinity_notify; 1962 1963 affinity_notify = &q_vector->affinity_notify; 1964 affinity_notify->notify = ice_irq_affinity_notify; 1965 affinity_notify->release = ice_irq_affinity_release; 1966 irq_set_affinity_notifier(irq_num, affinity_notify); 1967 } 1968 1969 /* assign the mask for this irq */ 1970 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 1971 } 1972 1973 vsi->irqs_ready = true; 1974 return 0; 1975 1976 free_q_irqs: 1977 while (vector) { 1978 vector--; 1979 irq_num = pf->msix_entries[base + vector].vector; 1980 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) 1981 irq_set_affinity_notifier(irq_num, NULL); 1982 irq_set_affinity_hint(irq_num, NULL); 1983 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 1984 } 1985 return err; 1986 } 1987 1988 /** 1989 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 1990 * @vsi: VSI to setup Tx rings used by XDP 1991 * 1992 * Return 0 on success and negative value on error 1993 */ 1994 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 1995 { 1996 struct device *dev = ice_pf_to_dev(vsi->back); 1997 int i; 1998 1999 for (i = 0; i < vsi->num_xdp_txq; i++) { 2000 u16 xdp_q_idx = vsi->alloc_txq + i; 2001 struct ice_ring *xdp_ring; 2002 2003 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2004 2005 if (!xdp_ring) 2006 goto free_xdp_rings; 2007 2008 xdp_ring->q_index = xdp_q_idx; 2009 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2010 xdp_ring->ring_active = false; 2011 xdp_ring->vsi = vsi; 2012 xdp_ring->netdev = NULL; 2013 xdp_ring->dev = dev; 2014 xdp_ring->count = vsi->num_tx_desc; 2015 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2016 if (ice_setup_tx_ring(xdp_ring)) 2017 goto free_xdp_rings; 2018 ice_set_ring_xdp(xdp_ring); 2019 xdp_ring->xsk_umem = ice_xsk_umem(xdp_ring); 2020 } 2021 2022 return 0; 2023 2024 free_xdp_rings: 2025 for (; i >= 0; i--) 2026 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2027 ice_free_tx_ring(vsi->xdp_rings[i]); 2028 return -ENOMEM; 2029 } 2030 2031 /** 2032 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2033 * @vsi: VSI to set the bpf prog on 2034 * @prog: the bpf prog pointer 2035 */ 2036 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2037 { 2038 struct bpf_prog *old_prog; 2039 int i; 2040 2041 old_prog = xchg(&vsi->xdp_prog, prog); 2042 if (old_prog) 2043 bpf_prog_put(old_prog); 2044 2045 ice_for_each_rxq(vsi, i) 2046 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2047 } 2048 2049 /** 2050 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2051 * @vsi: VSI to bring up Tx rings used by XDP 2052 * @prog: bpf program that will be assigned to VSI 2053 * 2054 * Return 0 on success and negative value on error 2055 */ 2056 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) 2057 { 2058 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2059 int xdp_rings_rem = vsi->num_xdp_txq; 2060 struct ice_pf *pf = vsi->back; 2061 struct ice_qs_cfg xdp_qs_cfg = { 2062 .qs_mutex = &pf->avail_q_mutex, 2063 .pf_map = pf->avail_txqs, 2064 .pf_map_size = pf->max_pf_txqs, 2065 .q_count = vsi->num_xdp_txq, 2066 .scatter_count = ICE_MAX_SCATTER_TXQS, 2067 .vsi_map = vsi->txq_map, 2068 .vsi_map_offset = vsi->alloc_txq, 2069 .mapping_mode = ICE_VSI_MAP_CONTIG 2070 }; 2071 enum ice_status status; 2072 struct device *dev; 2073 int i, v_idx; 2074 2075 dev = ice_pf_to_dev(pf); 2076 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2077 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2078 if (!vsi->xdp_rings) 2079 return -ENOMEM; 2080 2081 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2082 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2083 goto err_map_xdp; 2084 2085 if (ice_xdp_alloc_setup_rings(vsi)) 2086 goto clear_xdp_rings; 2087 2088 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2089 ice_for_each_q_vector(vsi, v_idx) { 2090 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2091 int xdp_rings_per_v, q_id, q_base; 2092 2093 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2094 vsi->num_q_vectors - v_idx); 2095 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2096 2097 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2098 struct ice_ring *xdp_ring = vsi->xdp_rings[q_id]; 2099 2100 xdp_ring->q_vector = q_vector; 2101 xdp_ring->next = q_vector->tx.ring; 2102 q_vector->tx.ring = xdp_ring; 2103 } 2104 xdp_rings_rem -= xdp_rings_per_v; 2105 } 2106 2107 /* omit the scheduler update if in reset path; XDP queues will be 2108 * taken into account at the end of ice_vsi_rebuild, where 2109 * ice_cfg_vsi_lan is being called 2110 */ 2111 if (ice_is_reset_in_progress(pf->state)) 2112 return 0; 2113 2114 /* tell the Tx scheduler that right now we have 2115 * additional queues 2116 */ 2117 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2118 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2119 2120 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2121 max_txqs); 2122 if (status) { 2123 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %s\n", 2124 ice_stat_str(status)); 2125 goto clear_xdp_rings; 2126 } 2127 ice_vsi_assign_bpf_prog(vsi, prog); 2128 2129 return 0; 2130 clear_xdp_rings: 2131 for (i = 0; i < vsi->num_xdp_txq; i++) 2132 if (vsi->xdp_rings[i]) { 2133 kfree_rcu(vsi->xdp_rings[i], rcu); 2134 vsi->xdp_rings[i] = NULL; 2135 } 2136 2137 err_map_xdp: 2138 mutex_lock(&pf->avail_q_mutex); 2139 for (i = 0; i < vsi->num_xdp_txq; i++) { 2140 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2141 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2142 } 2143 mutex_unlock(&pf->avail_q_mutex); 2144 2145 devm_kfree(dev, vsi->xdp_rings); 2146 return -ENOMEM; 2147 } 2148 2149 /** 2150 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2151 * @vsi: VSI to remove XDP rings 2152 * 2153 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2154 * resources 2155 */ 2156 int ice_destroy_xdp_rings(struct ice_vsi *vsi) 2157 { 2158 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2159 struct ice_pf *pf = vsi->back; 2160 int i, v_idx; 2161 2162 /* q_vectors are freed in reset path so there's no point in detaching 2163 * rings; in case of rebuild being triggered not from reset reset bits 2164 * in pf->state won't be set, so additionally check first q_vector 2165 * against NULL 2166 */ 2167 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2168 goto free_qmap; 2169 2170 ice_for_each_q_vector(vsi, v_idx) { 2171 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2172 struct ice_ring *ring; 2173 2174 ice_for_each_ring(ring, q_vector->tx) 2175 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2176 break; 2177 2178 /* restore the value of last node prior to XDP setup */ 2179 q_vector->tx.ring = ring; 2180 } 2181 2182 free_qmap: 2183 mutex_lock(&pf->avail_q_mutex); 2184 for (i = 0; i < vsi->num_xdp_txq; i++) { 2185 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2186 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2187 } 2188 mutex_unlock(&pf->avail_q_mutex); 2189 2190 for (i = 0; i < vsi->num_xdp_txq; i++) 2191 if (vsi->xdp_rings[i]) { 2192 if (vsi->xdp_rings[i]->desc) 2193 ice_free_tx_ring(vsi->xdp_rings[i]); 2194 kfree_rcu(vsi->xdp_rings[i], rcu); 2195 vsi->xdp_rings[i] = NULL; 2196 } 2197 2198 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2199 vsi->xdp_rings = NULL; 2200 2201 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2202 return 0; 2203 2204 ice_vsi_assign_bpf_prog(vsi, NULL); 2205 2206 /* notify Tx scheduler that we destroyed XDP queues and bring 2207 * back the old number of child nodes 2208 */ 2209 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2210 max_txqs[i] = vsi->num_txq; 2211 2212 /* change number of XDP Tx queues to 0 */ 2213 vsi->num_xdp_txq = 0; 2214 2215 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2216 max_txqs); 2217 } 2218 2219 /** 2220 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2221 * @vsi: VSI to setup XDP for 2222 * @prog: XDP program 2223 * @extack: netlink extended ack 2224 */ 2225 static int 2226 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2227 struct netlink_ext_ack *extack) 2228 { 2229 int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2230 bool if_running = netif_running(vsi->netdev); 2231 int ret = 0, xdp_ring_err = 0; 2232 2233 if (frame_size > vsi->rx_buf_len) { 2234 NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP"); 2235 return -EOPNOTSUPP; 2236 } 2237 2238 /* need to stop netdev while setting up the program for Rx rings */ 2239 if (if_running && !test_and_set_bit(__ICE_DOWN, vsi->state)) { 2240 ret = ice_down(vsi); 2241 if (ret) { 2242 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 2243 return ret; 2244 } 2245 } 2246 2247 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 2248 vsi->num_xdp_txq = vsi->alloc_rxq; 2249 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); 2250 if (xdp_ring_err) 2251 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 2252 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 2253 xdp_ring_err = ice_destroy_xdp_rings(vsi); 2254 if (xdp_ring_err) 2255 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 2256 } else { 2257 ice_vsi_assign_bpf_prog(vsi, prog); 2258 } 2259 2260 if (if_running) 2261 ret = ice_up(vsi); 2262 2263 if (!ret && prog && vsi->xsk_umems) { 2264 int i; 2265 2266 ice_for_each_rxq(vsi, i) { 2267 struct ice_ring *rx_ring = vsi->rx_rings[i]; 2268 2269 if (rx_ring->xsk_umem) 2270 napi_schedule(&rx_ring->q_vector->napi); 2271 } 2272 } 2273 2274 return (ret || xdp_ring_err) ? -ENOMEM : 0; 2275 } 2276 2277 /** 2278 * ice_xdp - implements XDP handler 2279 * @dev: netdevice 2280 * @xdp: XDP command 2281 */ 2282 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2283 { 2284 struct ice_netdev_priv *np = netdev_priv(dev); 2285 struct ice_vsi *vsi = np->vsi; 2286 2287 if (vsi->type != ICE_VSI_PF) { 2288 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); 2289 return -EINVAL; 2290 } 2291 2292 switch (xdp->command) { 2293 case XDP_SETUP_PROG: 2294 return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 2295 case XDP_SETUP_XSK_UMEM: 2296 return ice_xsk_umem_setup(vsi, xdp->xsk.umem, 2297 xdp->xsk.queue_id); 2298 default: 2299 return -EINVAL; 2300 } 2301 } 2302 2303 /** 2304 * ice_ena_misc_vector - enable the non-queue interrupts 2305 * @pf: board private structure 2306 */ 2307 static void ice_ena_misc_vector(struct ice_pf *pf) 2308 { 2309 struct ice_hw *hw = &pf->hw; 2310 u32 val; 2311 2312 /* Disable anti-spoof detection interrupt to prevent spurious event 2313 * interrupts during a function reset. Anti-spoof functionally is 2314 * still supported. 2315 */ 2316 val = rd32(hw, GL_MDCK_TX_TDPU); 2317 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 2318 wr32(hw, GL_MDCK_TX_TDPU, val); 2319 2320 /* clear things first */ 2321 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 2322 rd32(hw, PFINT_OICR); /* read to clear */ 2323 2324 val = (PFINT_OICR_ECC_ERR_M | 2325 PFINT_OICR_MAL_DETECT_M | 2326 PFINT_OICR_GRST_M | 2327 PFINT_OICR_PCI_EXCEPTION_M | 2328 PFINT_OICR_VFLR_M | 2329 PFINT_OICR_HMC_ERR_M | 2330 PFINT_OICR_PE_CRITERR_M); 2331 2332 wr32(hw, PFINT_OICR_ENA, val); 2333 2334 /* SW_ITR_IDX = 0, but don't change INTENA */ 2335 wr32(hw, GLINT_DYN_CTL(pf->oicr_idx), 2336 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 2337 } 2338 2339 /** 2340 * ice_misc_intr - misc interrupt handler 2341 * @irq: interrupt number 2342 * @data: pointer to a q_vector 2343 */ 2344 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 2345 { 2346 struct ice_pf *pf = (struct ice_pf *)data; 2347 struct ice_hw *hw = &pf->hw; 2348 irqreturn_t ret = IRQ_NONE; 2349 struct device *dev; 2350 u32 oicr, ena_mask; 2351 2352 dev = ice_pf_to_dev(pf); 2353 set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 2354 set_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 2355 2356 oicr = rd32(hw, PFINT_OICR); 2357 ena_mask = rd32(hw, PFINT_OICR_ENA); 2358 2359 if (oicr & PFINT_OICR_SWINT_M) { 2360 ena_mask &= ~PFINT_OICR_SWINT_M; 2361 pf->sw_int_count++; 2362 } 2363 2364 if (oicr & PFINT_OICR_MAL_DETECT_M) { 2365 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 2366 set_bit(__ICE_MDD_EVENT_PENDING, pf->state); 2367 } 2368 if (oicr & PFINT_OICR_VFLR_M) { 2369 /* disable any further VFLR event notifications */ 2370 if (test_bit(__ICE_VF_RESETS_DISABLED, pf->state)) { 2371 u32 reg = rd32(hw, PFINT_OICR_ENA); 2372 2373 reg &= ~PFINT_OICR_VFLR_M; 2374 wr32(hw, PFINT_OICR_ENA, reg); 2375 } else { 2376 ena_mask &= ~PFINT_OICR_VFLR_M; 2377 set_bit(__ICE_VFLR_EVENT_PENDING, pf->state); 2378 } 2379 } 2380 2381 if (oicr & PFINT_OICR_GRST_M) { 2382 u32 reset; 2383 2384 /* we have a reset warning */ 2385 ena_mask &= ~PFINT_OICR_GRST_M; 2386 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 2387 GLGEN_RSTAT_RESET_TYPE_S; 2388 2389 if (reset == ICE_RESET_CORER) 2390 pf->corer_count++; 2391 else if (reset == ICE_RESET_GLOBR) 2392 pf->globr_count++; 2393 else if (reset == ICE_RESET_EMPR) 2394 pf->empr_count++; 2395 else 2396 dev_dbg(dev, "Invalid reset type %d\n", reset); 2397 2398 /* If a reset cycle isn't already in progress, we set a bit in 2399 * pf->state so that the service task can start a reset/rebuild. 2400 * We also make note of which reset happened so that peer 2401 * devices/drivers can be informed. 2402 */ 2403 if (!test_and_set_bit(__ICE_RESET_OICR_RECV, pf->state)) { 2404 if (reset == ICE_RESET_CORER) 2405 set_bit(__ICE_CORER_RECV, pf->state); 2406 else if (reset == ICE_RESET_GLOBR) 2407 set_bit(__ICE_GLOBR_RECV, pf->state); 2408 else 2409 set_bit(__ICE_EMPR_RECV, pf->state); 2410 2411 /* There are couple of different bits at play here. 2412 * hw->reset_ongoing indicates whether the hardware is 2413 * in reset. This is set to true when a reset interrupt 2414 * is received and set back to false after the driver 2415 * has determined that the hardware is out of reset. 2416 * 2417 * __ICE_RESET_OICR_RECV in pf->state indicates 2418 * that a post reset rebuild is required before the 2419 * driver is operational again. This is set above. 2420 * 2421 * As this is the start of the reset/rebuild cycle, set 2422 * both to indicate that. 2423 */ 2424 hw->reset_ongoing = true; 2425 } 2426 } 2427 2428 if (oicr & PFINT_OICR_HMC_ERR_M) { 2429 ena_mask &= ~PFINT_OICR_HMC_ERR_M; 2430 dev_dbg(dev, "HMC Error interrupt - info 0x%x, data 0x%x\n", 2431 rd32(hw, PFHMC_ERRORINFO), 2432 rd32(hw, PFHMC_ERRORDATA)); 2433 } 2434 2435 /* Report any remaining unexpected interrupts */ 2436 oicr &= ena_mask; 2437 if (oicr) { 2438 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 2439 /* If a critical error is pending there is no choice but to 2440 * reset the device. 2441 */ 2442 if (oicr & (PFINT_OICR_PE_CRITERR_M | 2443 PFINT_OICR_PCI_EXCEPTION_M | 2444 PFINT_OICR_ECC_ERR_M)) { 2445 set_bit(__ICE_PFR_REQ, pf->state); 2446 ice_service_task_schedule(pf); 2447 } 2448 } 2449 ret = IRQ_HANDLED; 2450 2451 ice_service_task_schedule(pf); 2452 ice_irq_dynamic_ena(hw, NULL, NULL); 2453 2454 return ret; 2455 } 2456 2457 /** 2458 * ice_dis_ctrlq_interrupts - disable control queue interrupts 2459 * @hw: pointer to HW structure 2460 */ 2461 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 2462 { 2463 /* disable Admin queue Interrupt causes */ 2464 wr32(hw, PFINT_FW_CTL, 2465 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 2466 2467 /* disable Mailbox queue Interrupt causes */ 2468 wr32(hw, PFINT_MBX_CTL, 2469 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 2470 2471 /* disable Control queue Interrupt causes */ 2472 wr32(hw, PFINT_OICR_CTL, 2473 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 2474 2475 ice_flush(hw); 2476 } 2477 2478 /** 2479 * ice_free_irq_msix_misc - Unroll misc vector setup 2480 * @pf: board private structure 2481 */ 2482 static void ice_free_irq_msix_misc(struct ice_pf *pf) 2483 { 2484 struct ice_hw *hw = &pf->hw; 2485 2486 ice_dis_ctrlq_interrupts(hw); 2487 2488 /* disable OICR interrupt */ 2489 wr32(hw, PFINT_OICR_ENA, 0); 2490 ice_flush(hw); 2491 2492 if (pf->msix_entries) { 2493 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector); 2494 devm_free_irq(ice_pf_to_dev(pf), 2495 pf->msix_entries[pf->oicr_idx].vector, pf); 2496 } 2497 2498 pf->num_avail_sw_msix += 1; 2499 ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID); 2500 } 2501 2502 /** 2503 * ice_ena_ctrlq_interrupts - enable control queue interrupts 2504 * @hw: pointer to HW structure 2505 * @reg_idx: HW vector index to associate the control queue interrupts with 2506 */ 2507 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 2508 { 2509 u32 val; 2510 2511 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 2512 PFINT_OICR_CTL_CAUSE_ENA_M); 2513 wr32(hw, PFINT_OICR_CTL, val); 2514 2515 /* enable Admin queue Interrupt causes */ 2516 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 2517 PFINT_FW_CTL_CAUSE_ENA_M); 2518 wr32(hw, PFINT_FW_CTL, val); 2519 2520 /* enable Mailbox queue Interrupt causes */ 2521 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 2522 PFINT_MBX_CTL_CAUSE_ENA_M); 2523 wr32(hw, PFINT_MBX_CTL, val); 2524 2525 ice_flush(hw); 2526 } 2527 2528 /** 2529 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 2530 * @pf: board private structure 2531 * 2532 * This sets up the handler for MSIX 0, which is used to manage the 2533 * non-queue interrupts, e.g. AdminQ and errors. This is not used 2534 * when in MSI or Legacy interrupt mode. 2535 */ 2536 static int ice_req_irq_msix_misc(struct ice_pf *pf) 2537 { 2538 struct device *dev = ice_pf_to_dev(pf); 2539 struct ice_hw *hw = &pf->hw; 2540 int oicr_idx, err = 0; 2541 2542 if (!pf->int_name[0]) 2543 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 2544 dev_driver_string(dev), dev_name(dev)); 2545 2546 /* Do not request IRQ but do enable OICR interrupt since settings are 2547 * lost during reset. Note that this function is called only during 2548 * rebuild path and not while reset is in progress. 2549 */ 2550 if (ice_is_reset_in_progress(pf->state)) 2551 goto skip_req_irq; 2552 2553 /* reserve one vector in irq_tracker for misc interrupts */ 2554 oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2555 if (oicr_idx < 0) 2556 return oicr_idx; 2557 2558 pf->num_avail_sw_msix -= 1; 2559 pf->oicr_idx = (u16)oicr_idx; 2560 2561 err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, 2562 ice_misc_intr, 0, pf->int_name, pf); 2563 if (err) { 2564 dev_err(dev, "devm_request_irq for %s failed: %d\n", 2565 pf->int_name, err); 2566 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2567 pf->num_avail_sw_msix += 1; 2568 return err; 2569 } 2570 2571 skip_req_irq: 2572 ice_ena_misc_vector(pf); 2573 2574 ice_ena_ctrlq_interrupts(hw, pf->oicr_idx); 2575 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx), 2576 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 2577 2578 ice_flush(hw); 2579 ice_irq_dynamic_ena(hw, NULL, NULL); 2580 2581 return 0; 2582 } 2583 2584 /** 2585 * ice_napi_add - register NAPI handler for the VSI 2586 * @vsi: VSI for which NAPI handler is to be registered 2587 * 2588 * This function is only called in the driver's load path. Registering the NAPI 2589 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 2590 * reset/rebuild, etc.) 2591 */ 2592 static void ice_napi_add(struct ice_vsi *vsi) 2593 { 2594 int v_idx; 2595 2596 if (!vsi->netdev) 2597 return; 2598 2599 ice_for_each_q_vector(vsi, v_idx) 2600 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 2601 ice_napi_poll, NAPI_POLL_WEIGHT); 2602 } 2603 2604 /** 2605 * ice_set_ops - set netdev and ethtools ops for the given netdev 2606 * @netdev: netdev instance 2607 */ 2608 static void ice_set_ops(struct net_device *netdev) 2609 { 2610 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2611 2612 if (ice_is_safe_mode(pf)) { 2613 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 2614 ice_set_ethtool_safe_mode_ops(netdev); 2615 return; 2616 } 2617 2618 netdev->netdev_ops = &ice_netdev_ops; 2619 ice_set_ethtool_ops(netdev); 2620 } 2621 2622 /** 2623 * ice_set_netdev_features - set features for the given netdev 2624 * @netdev: netdev instance 2625 */ 2626 static void ice_set_netdev_features(struct net_device *netdev) 2627 { 2628 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2629 netdev_features_t csumo_features; 2630 netdev_features_t vlano_features; 2631 netdev_features_t dflt_features; 2632 netdev_features_t tso_features; 2633 2634 if (ice_is_safe_mode(pf)) { 2635 /* safe mode */ 2636 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 2637 netdev->hw_features = netdev->features; 2638 return; 2639 } 2640 2641 dflt_features = NETIF_F_SG | 2642 NETIF_F_HIGHDMA | 2643 NETIF_F_NTUPLE | 2644 NETIF_F_RXHASH; 2645 2646 csumo_features = NETIF_F_RXCSUM | 2647 NETIF_F_IP_CSUM | 2648 NETIF_F_SCTP_CRC | 2649 NETIF_F_IPV6_CSUM; 2650 2651 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 2652 NETIF_F_HW_VLAN_CTAG_TX | 2653 NETIF_F_HW_VLAN_CTAG_RX; 2654 2655 tso_features = NETIF_F_TSO | 2656 NETIF_F_TSO_ECN | 2657 NETIF_F_TSO6 | 2658 NETIF_F_GSO_GRE | 2659 NETIF_F_GSO_UDP_TUNNEL | 2660 NETIF_F_GSO_GRE_CSUM | 2661 NETIF_F_GSO_UDP_TUNNEL_CSUM | 2662 NETIF_F_GSO_PARTIAL | 2663 NETIF_F_GSO_IPXIP4 | 2664 NETIF_F_GSO_IPXIP6 | 2665 NETIF_F_GSO_UDP_L4; 2666 2667 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 2668 NETIF_F_GSO_GRE_CSUM; 2669 /* set features that user can change */ 2670 netdev->hw_features = dflt_features | csumo_features | 2671 vlano_features | tso_features; 2672 2673 /* add support for HW_CSUM on packets with MPLS header */ 2674 netdev->mpls_features = NETIF_F_HW_CSUM; 2675 2676 /* enable features */ 2677 netdev->features |= netdev->hw_features; 2678 /* encap and VLAN devices inherit default, csumo and tso features */ 2679 netdev->hw_enc_features |= dflt_features | csumo_features | 2680 tso_features; 2681 netdev->vlan_features |= dflt_features | csumo_features | 2682 tso_features; 2683 } 2684 2685 /** 2686 * ice_cfg_netdev - Allocate, configure and register a netdev 2687 * @vsi: the VSI associated with the new netdev 2688 * 2689 * Returns 0 on success, negative value on failure 2690 */ 2691 static int ice_cfg_netdev(struct ice_vsi *vsi) 2692 { 2693 struct ice_pf *pf = vsi->back; 2694 struct ice_netdev_priv *np; 2695 struct net_device *netdev; 2696 u8 mac_addr[ETH_ALEN]; 2697 int err; 2698 2699 err = ice_devlink_create_port(pf); 2700 if (err) 2701 return err; 2702 2703 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 2704 vsi->alloc_rxq); 2705 if (!netdev) { 2706 err = -ENOMEM; 2707 goto err_destroy_devlink_port; 2708 } 2709 2710 vsi->netdev = netdev; 2711 np = netdev_priv(netdev); 2712 np->vsi = vsi; 2713 2714 ice_set_netdev_features(netdev); 2715 2716 ice_set_ops(netdev); 2717 2718 if (vsi->type == ICE_VSI_PF) { 2719 SET_NETDEV_DEV(netdev, ice_pf_to_dev(pf)); 2720 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 2721 ether_addr_copy(netdev->dev_addr, mac_addr); 2722 ether_addr_copy(netdev->perm_addr, mac_addr); 2723 } 2724 2725 netdev->priv_flags |= IFF_UNICAST_FLT; 2726 2727 /* Setup netdev TC information */ 2728 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 2729 2730 /* setup watchdog timeout value to be 5 second */ 2731 netdev->watchdog_timeo = 5 * HZ; 2732 2733 netdev->min_mtu = ETH_MIN_MTU; 2734 netdev->max_mtu = ICE_MAX_MTU; 2735 2736 err = register_netdev(vsi->netdev); 2737 if (err) 2738 goto err_free_netdev; 2739 2740 devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev); 2741 2742 netif_carrier_off(vsi->netdev); 2743 2744 /* make sure transmit queues start off as stopped */ 2745 netif_tx_stop_all_queues(vsi->netdev); 2746 2747 return 0; 2748 2749 err_free_netdev: 2750 free_netdev(vsi->netdev); 2751 vsi->netdev = NULL; 2752 err_destroy_devlink_port: 2753 ice_devlink_destroy_port(pf); 2754 return err; 2755 } 2756 2757 /** 2758 * ice_fill_rss_lut - Fill the RSS lookup table with default values 2759 * @lut: Lookup table 2760 * @rss_table_size: Lookup table size 2761 * @rss_size: Range of queue number for hashing 2762 */ 2763 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 2764 { 2765 u16 i; 2766 2767 for (i = 0; i < rss_table_size; i++) 2768 lut[i] = i % rss_size; 2769 } 2770 2771 /** 2772 * ice_pf_vsi_setup - Set up a PF VSI 2773 * @pf: board private structure 2774 * @pi: pointer to the port_info instance 2775 * 2776 * Returns pointer to the successfully allocated VSI software struct 2777 * on success, otherwise returns NULL on failure. 2778 */ 2779 static struct ice_vsi * 2780 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2781 { 2782 return ice_vsi_setup(pf, pi, ICE_VSI_PF, ICE_INVAL_VFID); 2783 } 2784 2785 /** 2786 * ice_ctrl_vsi_setup - Set up a control VSI 2787 * @pf: board private structure 2788 * @pi: pointer to the port_info instance 2789 * 2790 * Returns pointer to the successfully allocated VSI software struct 2791 * on success, otherwise returns NULL on failure. 2792 */ 2793 static struct ice_vsi * 2794 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2795 { 2796 return ice_vsi_setup(pf, pi, ICE_VSI_CTRL, ICE_INVAL_VFID); 2797 } 2798 2799 /** 2800 * ice_lb_vsi_setup - Set up a loopback VSI 2801 * @pf: board private structure 2802 * @pi: pointer to the port_info instance 2803 * 2804 * Returns pointer to the successfully allocated VSI software struct 2805 * on success, otherwise returns NULL on failure. 2806 */ 2807 struct ice_vsi * 2808 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 2809 { 2810 return ice_vsi_setup(pf, pi, ICE_VSI_LB, ICE_INVAL_VFID); 2811 } 2812 2813 /** 2814 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 2815 * @netdev: network interface to be adjusted 2816 * @proto: unused protocol 2817 * @vid: VLAN ID to be added 2818 * 2819 * net_device_ops implementation for adding VLAN IDs 2820 */ 2821 static int 2822 ice_vlan_rx_add_vid(struct net_device *netdev, __always_unused __be16 proto, 2823 u16 vid) 2824 { 2825 struct ice_netdev_priv *np = netdev_priv(netdev); 2826 struct ice_vsi *vsi = np->vsi; 2827 int ret; 2828 2829 if (vid >= VLAN_N_VID) { 2830 netdev_err(netdev, "VLAN id requested %d is out of range %d\n", 2831 vid, VLAN_N_VID); 2832 return -EINVAL; 2833 } 2834 2835 if (vsi->info.pvid) 2836 return -EINVAL; 2837 2838 /* VLAN 0 is added by default during load/reset */ 2839 if (!vid) 2840 return 0; 2841 2842 /* Enable VLAN pruning when a VLAN other than 0 is added */ 2843 if (!ice_vsi_is_vlan_pruning_ena(vsi)) { 2844 ret = ice_cfg_vlan_pruning(vsi, true, false); 2845 if (ret) 2846 return ret; 2847 } 2848 2849 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 2850 * packets aren't pruned by the device's internal switch on Rx 2851 */ 2852 ret = ice_vsi_add_vlan(vsi, vid, ICE_FWD_TO_VSI); 2853 if (!ret) { 2854 vsi->vlan_ena = true; 2855 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 2856 } 2857 2858 return ret; 2859 } 2860 2861 /** 2862 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 2863 * @netdev: network interface to be adjusted 2864 * @proto: unused protocol 2865 * @vid: VLAN ID to be removed 2866 * 2867 * net_device_ops implementation for removing VLAN IDs 2868 */ 2869 static int 2870 ice_vlan_rx_kill_vid(struct net_device *netdev, __always_unused __be16 proto, 2871 u16 vid) 2872 { 2873 struct ice_netdev_priv *np = netdev_priv(netdev); 2874 struct ice_vsi *vsi = np->vsi; 2875 int ret; 2876 2877 if (vsi->info.pvid) 2878 return -EINVAL; 2879 2880 /* don't allow removal of VLAN 0 */ 2881 if (!vid) 2882 return 0; 2883 2884 /* Make sure ice_vsi_kill_vlan is successful before updating VLAN 2885 * information 2886 */ 2887 ret = ice_vsi_kill_vlan(vsi, vid); 2888 if (ret) 2889 return ret; 2890 2891 /* Disable pruning when VLAN 0 is the only VLAN rule */ 2892 if (vsi->num_vlan == 1 && ice_vsi_is_vlan_pruning_ena(vsi)) 2893 ret = ice_cfg_vlan_pruning(vsi, false, false); 2894 2895 vsi->vlan_ena = false; 2896 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 2897 return ret; 2898 } 2899 2900 /** 2901 * ice_setup_pf_sw - Setup the HW switch on startup or after reset 2902 * @pf: board private structure 2903 * 2904 * Returns 0 on success, negative value on failure 2905 */ 2906 static int ice_setup_pf_sw(struct ice_pf *pf) 2907 { 2908 struct ice_vsi *vsi; 2909 int status = 0; 2910 2911 if (ice_is_reset_in_progress(pf->state)) 2912 return -EBUSY; 2913 2914 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 2915 if (!vsi) { 2916 status = -ENOMEM; 2917 goto unroll_vsi_setup; 2918 } 2919 2920 status = ice_cfg_netdev(vsi); 2921 if (status) { 2922 status = -ENODEV; 2923 goto unroll_vsi_setup; 2924 } 2925 /* netdev has to be configured before setting frame size */ 2926 ice_vsi_cfg_frame_size(vsi); 2927 2928 /* Setup DCB netlink interface */ 2929 ice_dcbnl_setup(vsi); 2930 2931 /* registering the NAPI handler requires both the queues and 2932 * netdev to be created, which are done in ice_pf_vsi_setup() 2933 * and ice_cfg_netdev() respectively 2934 */ 2935 ice_napi_add(vsi); 2936 2937 status = ice_set_cpu_rx_rmap(vsi); 2938 if (status) { 2939 dev_err(ice_pf_to_dev(pf), "Failed to set CPU Rx map VSI %d error %d\n", 2940 vsi->vsi_num, status); 2941 status = -EINVAL; 2942 goto unroll_napi_add; 2943 } 2944 status = ice_init_mac_fltr(pf); 2945 if (status) 2946 goto free_cpu_rx_map; 2947 2948 return status; 2949 2950 free_cpu_rx_map: 2951 ice_free_cpu_rx_rmap(vsi); 2952 2953 unroll_napi_add: 2954 if (vsi) { 2955 ice_napi_del(vsi); 2956 if (vsi->netdev) { 2957 if (vsi->netdev->reg_state == NETREG_REGISTERED) 2958 unregister_netdev(vsi->netdev); 2959 free_netdev(vsi->netdev); 2960 vsi->netdev = NULL; 2961 } 2962 } 2963 2964 unroll_vsi_setup: 2965 if (vsi) { 2966 ice_vsi_free_q_vectors(vsi); 2967 ice_vsi_delete(vsi); 2968 ice_vsi_put_qs(vsi); 2969 ice_vsi_clear(vsi); 2970 } 2971 return status; 2972 } 2973 2974 /** 2975 * ice_get_avail_q_count - Get count of queues in use 2976 * @pf_qmap: bitmap to get queue use count from 2977 * @lock: pointer to a mutex that protects access to pf_qmap 2978 * @size: size of the bitmap 2979 */ 2980 static u16 2981 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 2982 { 2983 unsigned long bit; 2984 u16 count = 0; 2985 2986 mutex_lock(lock); 2987 for_each_clear_bit(bit, pf_qmap, size) 2988 count++; 2989 mutex_unlock(lock); 2990 2991 return count; 2992 } 2993 2994 /** 2995 * ice_get_avail_txq_count - Get count of Tx queues in use 2996 * @pf: pointer to an ice_pf instance 2997 */ 2998 u16 ice_get_avail_txq_count(struct ice_pf *pf) 2999 { 3000 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3001 pf->max_pf_txqs); 3002 } 3003 3004 /** 3005 * ice_get_avail_rxq_count - Get count of Rx queues in use 3006 * @pf: pointer to an ice_pf instance 3007 */ 3008 u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3009 { 3010 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3011 pf->max_pf_rxqs); 3012 } 3013 3014 /** 3015 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3016 * @pf: board private structure to initialize 3017 */ 3018 static void ice_deinit_pf(struct ice_pf *pf) 3019 { 3020 ice_service_task_stop(pf); 3021 mutex_destroy(&pf->sw_mutex); 3022 mutex_destroy(&pf->tc_mutex); 3023 mutex_destroy(&pf->avail_q_mutex); 3024 3025 if (pf->avail_txqs) { 3026 bitmap_free(pf->avail_txqs); 3027 pf->avail_txqs = NULL; 3028 } 3029 3030 if (pf->avail_rxqs) { 3031 bitmap_free(pf->avail_rxqs); 3032 pf->avail_rxqs = NULL; 3033 } 3034 } 3035 3036 /** 3037 * ice_set_pf_caps - set PFs capability flags 3038 * @pf: pointer to the PF instance 3039 */ 3040 static void ice_set_pf_caps(struct ice_pf *pf) 3041 { 3042 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3043 3044 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3045 if (func_caps->common_cap.dcb) 3046 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3047 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3048 if (func_caps->common_cap.sr_iov_1_1) { 3049 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3050 pf->num_vfs_supported = min_t(int, func_caps->num_allocd_vfs, 3051 ICE_MAX_VF_COUNT); 3052 } 3053 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 3054 if (func_caps->common_cap.rss_table_size) 3055 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 3056 3057 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 3058 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 3059 u16 unused; 3060 3061 /* ctrl_vsi_idx will be set to a valid value when flow director 3062 * is setup by ice_init_fdir 3063 */ 3064 pf->ctrl_vsi_idx = ICE_NO_VSI; 3065 set_bit(ICE_FLAG_FD_ENA, pf->flags); 3066 /* force guaranteed filter pool for PF */ 3067 ice_alloc_fd_guar_item(&pf->hw, &unused, 3068 func_caps->fd_fltr_guar); 3069 /* force shared filter pool for PF */ 3070 ice_alloc_fd_shrd_item(&pf->hw, &unused, 3071 func_caps->fd_fltr_best_effort); 3072 } 3073 3074 pf->max_pf_txqs = func_caps->common_cap.num_txq; 3075 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 3076 } 3077 3078 /** 3079 * ice_init_pf - Initialize general software structures (struct ice_pf) 3080 * @pf: board private structure to initialize 3081 */ 3082 static int ice_init_pf(struct ice_pf *pf) 3083 { 3084 ice_set_pf_caps(pf); 3085 3086 mutex_init(&pf->sw_mutex); 3087 mutex_init(&pf->tc_mutex); 3088 3089 /* setup service timer and periodic service task */ 3090 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 3091 pf->serv_tmr_period = HZ; 3092 INIT_WORK(&pf->serv_task, ice_service_task); 3093 clear_bit(__ICE_SERVICE_SCHED, pf->state); 3094 3095 mutex_init(&pf->avail_q_mutex); 3096 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 3097 if (!pf->avail_txqs) 3098 return -ENOMEM; 3099 3100 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 3101 if (!pf->avail_rxqs) { 3102 devm_kfree(ice_pf_to_dev(pf), pf->avail_txqs); 3103 pf->avail_txqs = NULL; 3104 return -ENOMEM; 3105 } 3106 3107 return 0; 3108 } 3109 3110 /** 3111 * ice_ena_msix_range - Request a range of MSIX vectors from the OS 3112 * @pf: board private structure 3113 * 3114 * compute the number of MSIX vectors required (v_budget) and request from 3115 * the OS. Return the number of vectors reserved or negative on failure 3116 */ 3117 static int ice_ena_msix_range(struct ice_pf *pf) 3118 { 3119 struct device *dev = ice_pf_to_dev(pf); 3120 int v_left, v_actual, v_budget = 0; 3121 int needed, err, i; 3122 3123 v_left = pf->hw.func_caps.common_cap.num_msix_vectors; 3124 3125 /* reserve one vector for miscellaneous handler */ 3126 needed = 1; 3127 if (v_left < needed) 3128 goto no_hw_vecs_left_err; 3129 v_budget += needed; 3130 v_left -= needed; 3131 3132 /* reserve vectors for LAN traffic */ 3133 needed = min_t(int, num_online_cpus(), v_left); 3134 if (v_left < needed) 3135 goto no_hw_vecs_left_err; 3136 pf->num_lan_msix = needed; 3137 v_budget += needed; 3138 v_left -= needed; 3139 3140 /* reserve one vector for flow director */ 3141 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 3142 needed = ICE_FDIR_MSIX; 3143 if (v_left < needed) 3144 goto no_hw_vecs_left_err; 3145 v_budget += needed; 3146 v_left -= needed; 3147 } 3148 3149 pf->msix_entries = devm_kcalloc(dev, v_budget, 3150 sizeof(*pf->msix_entries), GFP_KERNEL); 3151 3152 if (!pf->msix_entries) { 3153 err = -ENOMEM; 3154 goto exit_err; 3155 } 3156 3157 for (i = 0; i < v_budget; i++) 3158 pf->msix_entries[i].entry = i; 3159 3160 /* actually reserve the vectors */ 3161 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, 3162 ICE_MIN_MSIX, v_budget); 3163 3164 if (v_actual < 0) { 3165 dev_err(dev, "unable to reserve MSI-X vectors\n"); 3166 err = v_actual; 3167 goto msix_err; 3168 } 3169 3170 if (v_actual < v_budget) { 3171 dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", 3172 v_budget, v_actual); 3173 /* 2 vectors each for LAN and RDMA (traffic + OICR), one for flow director */ 3174 #define ICE_MIN_LAN_VECS 2 3175 #define ICE_MIN_RDMA_VECS 2 3176 #define ICE_MIN_VECS (ICE_MIN_LAN_VECS + ICE_MIN_RDMA_VECS + 1) 3177 3178 if (v_actual < ICE_MIN_LAN_VECS) { 3179 /* error if we can't get minimum vectors */ 3180 pci_disable_msix(pf->pdev); 3181 err = -ERANGE; 3182 goto msix_err; 3183 } else { 3184 pf->num_lan_msix = ICE_MIN_LAN_VECS; 3185 } 3186 } 3187 3188 return v_actual; 3189 3190 msix_err: 3191 devm_kfree(dev, pf->msix_entries); 3192 goto exit_err; 3193 3194 no_hw_vecs_left_err: 3195 dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n", 3196 needed, v_left); 3197 err = -ERANGE; 3198 exit_err: 3199 pf->num_lan_msix = 0; 3200 return err; 3201 } 3202 3203 /** 3204 * ice_dis_msix - Disable MSI-X interrupt setup in OS 3205 * @pf: board private structure 3206 */ 3207 static void ice_dis_msix(struct ice_pf *pf) 3208 { 3209 pci_disable_msix(pf->pdev); 3210 devm_kfree(ice_pf_to_dev(pf), pf->msix_entries); 3211 pf->msix_entries = NULL; 3212 } 3213 3214 /** 3215 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme 3216 * @pf: board private structure 3217 */ 3218 static void ice_clear_interrupt_scheme(struct ice_pf *pf) 3219 { 3220 ice_dis_msix(pf); 3221 3222 if (pf->irq_tracker) { 3223 devm_kfree(ice_pf_to_dev(pf), pf->irq_tracker); 3224 pf->irq_tracker = NULL; 3225 } 3226 } 3227 3228 /** 3229 * ice_init_interrupt_scheme - Determine proper interrupt scheme 3230 * @pf: board private structure to initialize 3231 */ 3232 static int ice_init_interrupt_scheme(struct ice_pf *pf) 3233 { 3234 int vectors; 3235 3236 vectors = ice_ena_msix_range(pf); 3237 3238 if (vectors < 0) 3239 return vectors; 3240 3241 /* set up vector assignment tracking */ 3242 pf->irq_tracker = 3243 devm_kzalloc(ice_pf_to_dev(pf), sizeof(*pf->irq_tracker) + 3244 (sizeof(u16) * vectors), GFP_KERNEL); 3245 if (!pf->irq_tracker) { 3246 ice_dis_msix(pf); 3247 return -ENOMEM; 3248 } 3249 3250 /* populate SW interrupts pool with number of OS granted IRQs. */ 3251 pf->num_avail_sw_msix = (u16)vectors; 3252 pf->irq_tracker->num_entries = (u16)vectors; 3253 pf->irq_tracker->end = pf->irq_tracker->num_entries; 3254 3255 return 0; 3256 } 3257 3258 /** 3259 * ice_is_wol_supported - get NVM state of WoL 3260 * @pf: board private structure 3261 * 3262 * Check if WoL is supported based on the HW configuration. 3263 * Returns true if NVM supports and enables WoL for this port, false otherwise 3264 */ 3265 bool ice_is_wol_supported(struct ice_pf *pf) 3266 { 3267 struct ice_hw *hw = &pf->hw; 3268 u16 wol_ctrl; 3269 3270 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 3271 * word) indicates WoL is not supported on the corresponding PF ID. 3272 */ 3273 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 3274 return false; 3275 3276 return !(BIT(hw->pf_id) & wol_ctrl); 3277 } 3278 3279 /** 3280 * ice_vsi_recfg_qs - Change the number of queues on a VSI 3281 * @vsi: VSI being changed 3282 * @new_rx: new number of Rx queues 3283 * @new_tx: new number of Tx queues 3284 * 3285 * Only change the number of queues if new_tx, or new_rx is non-0. 3286 * 3287 * Returns 0 on success. 3288 */ 3289 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx) 3290 { 3291 struct ice_pf *pf = vsi->back; 3292 int err = 0, timeout = 50; 3293 3294 if (!new_rx && !new_tx) 3295 return -EINVAL; 3296 3297 while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) { 3298 timeout--; 3299 if (!timeout) 3300 return -EBUSY; 3301 usleep_range(1000, 2000); 3302 } 3303 3304 if (new_tx) 3305 vsi->req_txq = (u16)new_tx; 3306 if (new_rx) 3307 vsi->req_rxq = (u16)new_rx; 3308 3309 /* set for the next time the netdev is started */ 3310 if (!netif_running(vsi->netdev)) { 3311 ice_vsi_rebuild(vsi, false); 3312 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 3313 goto done; 3314 } 3315 3316 ice_vsi_close(vsi); 3317 ice_vsi_rebuild(vsi, false); 3318 ice_pf_dcb_recfg(pf); 3319 ice_vsi_open(vsi); 3320 done: 3321 clear_bit(__ICE_CFG_BUSY, pf->state); 3322 return err; 3323 } 3324 3325 /** 3326 * ice_log_pkg_init - log result of DDP package load 3327 * @hw: pointer to hardware info 3328 * @status: status of package load 3329 */ 3330 static void 3331 ice_log_pkg_init(struct ice_hw *hw, enum ice_status *status) 3332 { 3333 struct ice_pf *pf = (struct ice_pf *)hw->back; 3334 struct device *dev = ice_pf_to_dev(pf); 3335 3336 switch (*status) { 3337 case ICE_SUCCESS: 3338 /* The package download AdminQ command returned success because 3339 * this download succeeded or ICE_ERR_AQ_NO_WORK since there is 3340 * already a package loaded on the device. 3341 */ 3342 if (hw->pkg_ver.major == hw->active_pkg_ver.major && 3343 hw->pkg_ver.minor == hw->active_pkg_ver.minor && 3344 hw->pkg_ver.update == hw->active_pkg_ver.update && 3345 hw->pkg_ver.draft == hw->active_pkg_ver.draft && 3346 !memcmp(hw->pkg_name, hw->active_pkg_name, 3347 sizeof(hw->pkg_name))) { 3348 if (hw->pkg_dwnld_status == ICE_AQ_RC_EEXIST) 3349 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 3350 hw->active_pkg_name, 3351 hw->active_pkg_ver.major, 3352 hw->active_pkg_ver.minor, 3353 hw->active_pkg_ver.update, 3354 hw->active_pkg_ver.draft); 3355 else 3356 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 3357 hw->active_pkg_name, 3358 hw->active_pkg_ver.major, 3359 hw->active_pkg_ver.minor, 3360 hw->active_pkg_ver.update, 3361 hw->active_pkg_ver.draft); 3362 } else if (hw->active_pkg_ver.major != ICE_PKG_SUPP_VER_MAJ || 3363 hw->active_pkg_ver.minor != ICE_PKG_SUPP_VER_MNR) { 3364 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 3365 hw->active_pkg_name, 3366 hw->active_pkg_ver.major, 3367 hw->active_pkg_ver.minor, 3368 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3369 *status = ICE_ERR_NOT_SUPPORTED; 3370 } else if (hw->active_pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3371 hw->active_pkg_ver.minor == ICE_PKG_SUPP_VER_MNR) { 3372 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 3373 hw->active_pkg_name, 3374 hw->active_pkg_ver.major, 3375 hw->active_pkg_ver.minor, 3376 hw->active_pkg_ver.update, 3377 hw->active_pkg_ver.draft, 3378 hw->pkg_name, 3379 hw->pkg_ver.major, 3380 hw->pkg_ver.minor, 3381 hw->pkg_ver.update, 3382 hw->pkg_ver.draft); 3383 } else { 3384 dev_err(dev, "An unknown error occurred when loading the DDP package, please reboot the system. If the problem persists, update the NVM. Entering Safe Mode.\n"); 3385 *status = ICE_ERR_NOT_SUPPORTED; 3386 } 3387 break; 3388 case ICE_ERR_FW_DDP_MISMATCH: 3389 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 3390 break; 3391 case ICE_ERR_BUF_TOO_SHORT: 3392 case ICE_ERR_CFG: 3393 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 3394 break; 3395 case ICE_ERR_NOT_SUPPORTED: 3396 /* Package File version not supported */ 3397 if (hw->pkg_ver.major > ICE_PKG_SUPP_VER_MAJ || 3398 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3399 hw->pkg_ver.minor > ICE_PKG_SUPP_VER_MNR)) 3400 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 3401 else if (hw->pkg_ver.major < ICE_PKG_SUPP_VER_MAJ || 3402 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3403 hw->pkg_ver.minor < ICE_PKG_SUPP_VER_MNR)) 3404 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 3405 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3406 break; 3407 case ICE_ERR_AQ_ERROR: 3408 switch (hw->pkg_dwnld_status) { 3409 case ICE_AQ_RC_ENOSEC: 3410 case ICE_AQ_RC_EBADSIG: 3411 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 3412 return; 3413 case ICE_AQ_RC_ESVN: 3414 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 3415 return; 3416 case ICE_AQ_RC_EBADMAN: 3417 case ICE_AQ_RC_EBADBUF: 3418 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 3419 /* poll for reset to complete */ 3420 if (ice_check_reset(hw)) 3421 dev_err(dev, "Error resetting device. Please reload the driver\n"); 3422 return; 3423 default: 3424 break; 3425 } 3426 fallthrough; 3427 default: 3428 dev_err(dev, "An unknown error (%d) occurred when loading the DDP package. Entering Safe Mode.\n", 3429 *status); 3430 break; 3431 } 3432 } 3433 3434 /** 3435 * ice_load_pkg - load/reload the DDP Package file 3436 * @firmware: firmware structure when firmware requested or NULL for reload 3437 * @pf: pointer to the PF instance 3438 * 3439 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 3440 * initialize HW tables. 3441 */ 3442 static void 3443 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 3444 { 3445 enum ice_status status = ICE_ERR_PARAM; 3446 struct device *dev = ice_pf_to_dev(pf); 3447 struct ice_hw *hw = &pf->hw; 3448 3449 /* Load DDP Package */ 3450 if (firmware && !hw->pkg_copy) { 3451 status = ice_copy_and_init_pkg(hw, firmware->data, 3452 firmware->size); 3453 ice_log_pkg_init(hw, &status); 3454 } else if (!firmware && hw->pkg_copy) { 3455 /* Reload package during rebuild after CORER/GLOBR reset */ 3456 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 3457 ice_log_pkg_init(hw, &status); 3458 } else { 3459 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 3460 } 3461 3462 if (status) { 3463 /* Safe Mode */ 3464 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3465 return; 3466 } 3467 3468 /* Successful download package is the precondition for advanced 3469 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 3470 */ 3471 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3472 } 3473 3474 /** 3475 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 3476 * @pf: pointer to the PF structure 3477 * 3478 * There is no error returned here because the driver should be able to handle 3479 * 128 Byte cache lines, so we only print a warning in case issues are seen, 3480 * specifically with Tx. 3481 */ 3482 static void ice_verify_cacheline_size(struct ice_pf *pf) 3483 { 3484 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 3485 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 3486 ICE_CACHE_LINE_BYTES); 3487 } 3488 3489 /** 3490 * ice_send_version - update firmware with driver version 3491 * @pf: PF struct 3492 * 3493 * Returns ICE_SUCCESS on success, else error code 3494 */ 3495 static enum ice_status ice_send_version(struct ice_pf *pf) 3496 { 3497 struct ice_driver_ver dv; 3498 3499 dv.major_ver = 0xff; 3500 dv.minor_ver = 0xff; 3501 dv.build_ver = 0xff; 3502 dv.subbuild_ver = 0; 3503 strscpy((char *)dv.driver_string, UTS_RELEASE, 3504 sizeof(dv.driver_string)); 3505 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 3506 } 3507 3508 /** 3509 * ice_init_fdir - Initialize flow director VSI and configuration 3510 * @pf: pointer to the PF instance 3511 * 3512 * returns 0 on success, negative on error 3513 */ 3514 static int ice_init_fdir(struct ice_pf *pf) 3515 { 3516 struct device *dev = ice_pf_to_dev(pf); 3517 struct ice_vsi *ctrl_vsi; 3518 int err; 3519 3520 /* Side Band Flow Director needs to have a control VSI. 3521 * Allocate it and store it in the PF. 3522 */ 3523 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 3524 if (!ctrl_vsi) { 3525 dev_dbg(dev, "could not create control VSI\n"); 3526 return -ENOMEM; 3527 } 3528 3529 err = ice_vsi_open_ctrl(ctrl_vsi); 3530 if (err) { 3531 dev_dbg(dev, "could not open control VSI\n"); 3532 goto err_vsi_open; 3533 } 3534 3535 mutex_init(&pf->hw.fdir_fltr_lock); 3536 3537 err = ice_fdir_create_dflt_rules(pf); 3538 if (err) 3539 goto err_fdir_rule; 3540 3541 return 0; 3542 3543 err_fdir_rule: 3544 ice_fdir_release_flows(&pf->hw); 3545 ice_vsi_close(ctrl_vsi); 3546 err_vsi_open: 3547 ice_vsi_release(ctrl_vsi); 3548 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 3549 pf->vsi[pf->ctrl_vsi_idx] = NULL; 3550 pf->ctrl_vsi_idx = ICE_NO_VSI; 3551 } 3552 return err; 3553 } 3554 3555 /** 3556 * ice_get_opt_fw_name - return optional firmware file name or NULL 3557 * @pf: pointer to the PF instance 3558 */ 3559 static char *ice_get_opt_fw_name(struct ice_pf *pf) 3560 { 3561 /* Optional firmware name same as default with additional dash 3562 * followed by a EUI-64 identifier (PCIe Device Serial Number) 3563 */ 3564 struct pci_dev *pdev = pf->pdev; 3565 char *opt_fw_filename; 3566 u64 dsn; 3567 3568 /* Determine the name of the optional file using the DSN (two 3569 * dwords following the start of the DSN Capability). 3570 */ 3571 dsn = pci_get_dsn(pdev); 3572 if (!dsn) 3573 return NULL; 3574 3575 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 3576 if (!opt_fw_filename) 3577 return NULL; 3578 3579 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 3580 ICE_DDP_PKG_PATH, dsn); 3581 3582 return opt_fw_filename; 3583 } 3584 3585 /** 3586 * ice_request_fw - Device initialization routine 3587 * @pf: pointer to the PF instance 3588 */ 3589 static void ice_request_fw(struct ice_pf *pf) 3590 { 3591 char *opt_fw_filename = ice_get_opt_fw_name(pf); 3592 const struct firmware *firmware = NULL; 3593 struct device *dev = ice_pf_to_dev(pf); 3594 int err = 0; 3595 3596 /* optional device-specific DDP (if present) overrides the default DDP 3597 * package file. kernel logs a debug message if the file doesn't exist, 3598 * and warning messages for other errors. 3599 */ 3600 if (opt_fw_filename) { 3601 err = firmware_request_nowarn(&firmware, opt_fw_filename, dev); 3602 if (err) { 3603 kfree(opt_fw_filename); 3604 goto dflt_pkg_load; 3605 } 3606 3607 /* request for firmware was successful. Download to device */ 3608 ice_load_pkg(firmware, pf); 3609 kfree(opt_fw_filename); 3610 release_firmware(firmware); 3611 return; 3612 } 3613 3614 dflt_pkg_load: 3615 err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev); 3616 if (err) { 3617 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 3618 return; 3619 } 3620 3621 /* request for firmware was successful. Download to device */ 3622 ice_load_pkg(firmware, pf); 3623 release_firmware(firmware); 3624 } 3625 3626 /** 3627 * ice_print_wake_reason - show the wake up cause in the log 3628 * @pf: pointer to the PF struct 3629 */ 3630 static void ice_print_wake_reason(struct ice_pf *pf) 3631 { 3632 u32 wus = pf->wakeup_reason; 3633 const char *wake_str; 3634 3635 /* if no wake event, nothing to print */ 3636 if (!wus) 3637 return; 3638 3639 if (wus & PFPM_WUS_LNKC_M) 3640 wake_str = "Link\n"; 3641 else if (wus & PFPM_WUS_MAG_M) 3642 wake_str = "Magic Packet\n"; 3643 else if (wus & PFPM_WUS_MNG_M) 3644 wake_str = "Management\n"; 3645 else if (wus & PFPM_WUS_FW_RST_WK_M) 3646 wake_str = "Firmware Reset\n"; 3647 else 3648 wake_str = "Unknown\n"; 3649 3650 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 3651 } 3652 3653 /** 3654 * ice_probe - Device initialization routine 3655 * @pdev: PCI device information struct 3656 * @ent: entry in ice_pci_tbl 3657 * 3658 * Returns 0 on success, negative on failure 3659 */ 3660 static int 3661 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 3662 { 3663 struct device *dev = &pdev->dev; 3664 struct ice_pf *pf; 3665 struct ice_hw *hw; 3666 int err; 3667 3668 /* this driver uses devres, see 3669 * Documentation/driver-api/driver-model/devres.rst 3670 */ 3671 err = pcim_enable_device(pdev); 3672 if (err) 3673 return err; 3674 3675 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev)); 3676 if (err) { 3677 dev_err(dev, "BAR0 I/O map error %d\n", err); 3678 return err; 3679 } 3680 3681 pf = ice_allocate_pf(dev); 3682 if (!pf) 3683 return -ENOMEM; 3684 3685 /* set up for high or low DMA */ 3686 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 3687 if (err) 3688 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 3689 if (err) { 3690 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 3691 return err; 3692 } 3693 3694 pci_enable_pcie_error_reporting(pdev); 3695 pci_set_master(pdev); 3696 3697 pf->pdev = pdev; 3698 pci_set_drvdata(pdev, pf); 3699 set_bit(__ICE_DOWN, pf->state); 3700 /* Disable service task until DOWN bit is cleared */ 3701 set_bit(__ICE_SERVICE_DIS, pf->state); 3702 3703 hw = &pf->hw; 3704 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 3705 pci_save_state(pdev); 3706 3707 hw->back = pf; 3708 hw->vendor_id = pdev->vendor; 3709 hw->device_id = pdev->device; 3710 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 3711 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3712 hw->subsystem_device_id = pdev->subsystem_device; 3713 hw->bus.device = PCI_SLOT(pdev->devfn); 3714 hw->bus.func = PCI_FUNC(pdev->devfn); 3715 ice_set_ctrlq_len(hw); 3716 3717 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 3718 3719 err = ice_devlink_register(pf); 3720 if (err) { 3721 dev_err(dev, "ice_devlink_register failed: %d\n", err); 3722 goto err_exit_unroll; 3723 } 3724 3725 #ifndef CONFIG_DYNAMIC_DEBUG 3726 if (debug < -1) 3727 hw->debug_mask = debug; 3728 #endif 3729 3730 err = ice_init_hw(hw); 3731 if (err) { 3732 dev_err(dev, "ice_init_hw failed: %d\n", err); 3733 err = -EIO; 3734 goto err_exit_unroll; 3735 } 3736 3737 ice_request_fw(pf); 3738 3739 /* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be 3740 * set in pf->state, which will cause ice_is_safe_mode to return 3741 * true 3742 */ 3743 if (ice_is_safe_mode(pf)) { 3744 dev_err(dev, "Package download failed. Advanced features disabled - Device now in Safe Mode\n"); 3745 /* we already got function/device capabilities but these don't 3746 * reflect what the driver needs to do in safe mode. Instead of 3747 * adding conditional logic everywhere to ignore these 3748 * device/function capabilities, override them. 3749 */ 3750 ice_set_safe_mode_caps(hw); 3751 } 3752 3753 err = ice_init_pf(pf); 3754 if (err) { 3755 dev_err(dev, "ice_init_pf failed: %d\n", err); 3756 goto err_init_pf_unroll; 3757 } 3758 3759 ice_devlink_init_regions(pf); 3760 3761 pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; 3762 if (!pf->num_alloc_vsi) { 3763 err = -EIO; 3764 goto err_init_pf_unroll; 3765 } 3766 3767 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 3768 GFP_KERNEL); 3769 if (!pf->vsi) { 3770 err = -ENOMEM; 3771 goto err_init_pf_unroll; 3772 } 3773 3774 err = ice_init_interrupt_scheme(pf); 3775 if (err) { 3776 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 3777 err = -EIO; 3778 goto err_init_vsi_unroll; 3779 } 3780 3781 /* In case of MSIX we are going to setup the misc vector right here 3782 * to handle admin queue events etc. In case of legacy and MSI 3783 * the misc functionality and queue processing is combined in 3784 * the same vector and that gets setup at open. 3785 */ 3786 err = ice_req_irq_msix_misc(pf); 3787 if (err) { 3788 dev_err(dev, "setup of misc vector failed: %d\n", err); 3789 goto err_init_interrupt_unroll; 3790 } 3791 3792 /* create switch struct for the switch element created by FW on boot */ 3793 pf->first_sw = devm_kzalloc(dev, sizeof(*pf->first_sw), GFP_KERNEL); 3794 if (!pf->first_sw) { 3795 err = -ENOMEM; 3796 goto err_msix_misc_unroll; 3797 } 3798 3799 if (hw->evb_veb) 3800 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 3801 else 3802 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 3803 3804 pf->first_sw->pf = pf; 3805 3806 /* record the sw_id available for later use */ 3807 pf->first_sw->sw_id = hw->port_info->sw_id; 3808 3809 err = ice_setup_pf_sw(pf); 3810 if (err) { 3811 dev_err(dev, "probe failed due to setup PF switch: %d\n", err); 3812 goto err_alloc_sw_unroll; 3813 } 3814 3815 clear_bit(__ICE_SERVICE_DIS, pf->state); 3816 3817 /* tell the firmware we are up */ 3818 err = ice_send_version(pf); 3819 if (err) { 3820 dev_err(dev, "probe failed sending driver version %s. error: %d\n", 3821 UTS_RELEASE, err); 3822 goto err_alloc_sw_unroll; 3823 } 3824 3825 /* since everything is good, start the service timer */ 3826 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 3827 3828 err = ice_init_link_events(pf->hw.port_info); 3829 if (err) { 3830 dev_err(dev, "ice_init_link_events failed: %d\n", err); 3831 goto err_alloc_sw_unroll; 3832 } 3833 3834 err = ice_init_nvm_phy_type(pf->hw.port_info); 3835 if (err) { 3836 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 3837 goto err_alloc_sw_unroll; 3838 } 3839 3840 err = ice_update_link_info(pf->hw.port_info); 3841 if (err) { 3842 dev_err(dev, "ice_update_link_info failed: %d\n", err); 3843 goto err_alloc_sw_unroll; 3844 } 3845 3846 ice_init_link_dflt_override(pf->hw.port_info); 3847 3848 /* if media available, initialize PHY settings */ 3849 if (pf->hw.port_info->phy.link_info.link_info & 3850 ICE_AQ_MEDIA_AVAILABLE) { 3851 err = ice_init_phy_user_cfg(pf->hw.port_info); 3852 if (err) { 3853 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 3854 goto err_alloc_sw_unroll; 3855 } 3856 3857 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 3858 struct ice_vsi *vsi = ice_get_main_vsi(pf); 3859 3860 if (vsi) 3861 ice_configure_phy(vsi); 3862 } 3863 } else { 3864 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 3865 } 3866 3867 ice_verify_cacheline_size(pf); 3868 3869 /* Save wakeup reason register for later use */ 3870 pf->wakeup_reason = rd32(hw, PFPM_WUS); 3871 3872 /* check for a power management event */ 3873 ice_print_wake_reason(pf); 3874 3875 /* clear wake status, all bits */ 3876 wr32(hw, PFPM_WUS, U32_MAX); 3877 3878 /* Disable WoL at init, wait for user to enable */ 3879 device_set_wakeup_enable(dev, false); 3880 3881 /* If no DDP driven features have to be setup, we are done with probe */ 3882 if (ice_is_safe_mode(pf)) 3883 goto probe_done; 3884 3885 /* initialize DDP driven features */ 3886 3887 /* Note: Flow director init failure is non-fatal to load */ 3888 if (ice_init_fdir(pf)) 3889 dev_err(dev, "could not initialize flow director\n"); 3890 3891 /* Note: DCB init failure is non-fatal to load */ 3892 if (ice_init_pf_dcb(pf, false)) { 3893 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3894 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 3895 } else { 3896 ice_cfg_lldp_mib_change(&pf->hw, true); 3897 } 3898 3899 /* print PCI link speed and width */ 3900 pcie_print_link_status(pf->pdev); 3901 3902 probe_done: 3903 /* ready to go, so clear down state bit */ 3904 clear_bit(__ICE_DOWN, pf->state); 3905 return 0; 3906 3907 err_alloc_sw_unroll: 3908 ice_devlink_destroy_port(pf); 3909 set_bit(__ICE_SERVICE_DIS, pf->state); 3910 set_bit(__ICE_DOWN, pf->state); 3911 devm_kfree(dev, pf->first_sw); 3912 err_msix_misc_unroll: 3913 ice_free_irq_msix_misc(pf); 3914 err_init_interrupt_unroll: 3915 ice_clear_interrupt_scheme(pf); 3916 err_init_vsi_unroll: 3917 devm_kfree(dev, pf->vsi); 3918 err_init_pf_unroll: 3919 ice_deinit_pf(pf); 3920 ice_devlink_destroy_regions(pf); 3921 ice_deinit_hw(hw); 3922 err_exit_unroll: 3923 ice_devlink_unregister(pf); 3924 pci_disable_pcie_error_reporting(pdev); 3925 pci_disable_device(pdev); 3926 return err; 3927 } 3928 3929 /** 3930 * ice_set_wake - enable or disable Wake on LAN 3931 * @pf: pointer to the PF struct 3932 * 3933 * Simple helper for WoL control 3934 */ 3935 static void ice_set_wake(struct ice_pf *pf) 3936 { 3937 struct ice_hw *hw = &pf->hw; 3938 bool wol = pf->wol_ena; 3939 3940 /* clear wake state, otherwise new wake events won't fire */ 3941 wr32(hw, PFPM_WUS, U32_MAX); 3942 3943 /* enable / disable APM wake up, no RMW needed */ 3944 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 3945 3946 /* set magic packet filter enabled */ 3947 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 3948 } 3949 3950 /** 3951 * ice_setup_magic_mc_wake - setup device to wake on multicast magic packet 3952 * @pf: pointer to the PF struct 3953 * 3954 * Issue firmware command to enable multicast magic wake, making 3955 * sure that any locally administered address (LAA) is used for 3956 * wake, and that PF reset doesn't undo the LAA. 3957 */ 3958 static void ice_setup_mc_magic_wake(struct ice_pf *pf) 3959 { 3960 struct device *dev = ice_pf_to_dev(pf); 3961 struct ice_hw *hw = &pf->hw; 3962 enum ice_status status; 3963 u8 mac_addr[ETH_ALEN]; 3964 struct ice_vsi *vsi; 3965 u8 flags; 3966 3967 if (!pf->wol_ena) 3968 return; 3969 3970 vsi = ice_get_main_vsi(pf); 3971 if (!vsi) 3972 return; 3973 3974 /* Get current MAC address in case it's an LAA */ 3975 if (vsi->netdev) 3976 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 3977 else 3978 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 3979 3980 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 3981 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 3982 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 3983 3984 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 3985 if (status) 3986 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %s aq_err %s\n", 3987 ice_stat_str(status), 3988 ice_aq_str(hw->adminq.sq_last_status)); 3989 } 3990 3991 /** 3992 * ice_remove - Device removal routine 3993 * @pdev: PCI device information struct 3994 */ 3995 static void ice_remove(struct pci_dev *pdev) 3996 { 3997 struct ice_pf *pf = pci_get_drvdata(pdev); 3998 int i; 3999 4000 if (!pf) 4001 return; 4002 4003 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 4004 if (!ice_is_reset_in_progress(pf->state)) 4005 break; 4006 msleep(100); 4007 } 4008 4009 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 4010 set_bit(__ICE_VF_RESETS_DISABLED, pf->state); 4011 ice_free_vfs(pf); 4012 } 4013 4014 set_bit(__ICE_DOWN, pf->state); 4015 ice_service_task_stop(pf); 4016 4017 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 4018 if (!ice_is_safe_mode(pf)) 4019 ice_remove_arfs(pf); 4020 ice_setup_mc_magic_wake(pf); 4021 ice_devlink_destroy_port(pf); 4022 ice_vsi_release_all(pf); 4023 ice_set_wake(pf); 4024 ice_free_irq_msix_misc(pf); 4025 ice_for_each_vsi(pf, i) { 4026 if (!pf->vsi[i]) 4027 continue; 4028 ice_vsi_free_q_vectors(pf->vsi[i]); 4029 } 4030 ice_deinit_pf(pf); 4031 ice_devlink_destroy_regions(pf); 4032 ice_deinit_hw(&pf->hw); 4033 ice_devlink_unregister(pf); 4034 4035 /* Issue a PFR as part of the prescribed driver unload flow. Do not 4036 * do it via ice_schedule_reset() since there is no need to rebuild 4037 * and the service task is already stopped. 4038 */ 4039 ice_reset(&pf->hw, ICE_RESET_PFR); 4040 pci_wait_for_pending_transaction(pdev); 4041 ice_clear_interrupt_scheme(pf); 4042 pci_disable_pcie_error_reporting(pdev); 4043 pci_disable_device(pdev); 4044 } 4045 4046 /** 4047 * ice_shutdown - PCI callback for shutting down device 4048 * @pdev: PCI device information struct 4049 */ 4050 static void ice_shutdown(struct pci_dev *pdev) 4051 { 4052 struct ice_pf *pf = pci_get_drvdata(pdev); 4053 4054 ice_remove(pdev); 4055 4056 if (system_state == SYSTEM_POWER_OFF) { 4057 pci_wake_from_d3(pdev, pf->wol_ena); 4058 pci_set_power_state(pdev, PCI_D3hot); 4059 } 4060 } 4061 4062 #ifdef CONFIG_PM 4063 /** 4064 * ice_prepare_for_shutdown - prep for PCI shutdown 4065 * @pf: board private structure 4066 * 4067 * Inform or close all dependent features in prep for PCI device shutdown 4068 */ 4069 static void ice_prepare_for_shutdown(struct ice_pf *pf) 4070 { 4071 struct ice_hw *hw = &pf->hw; 4072 u32 v; 4073 4074 /* Notify VFs of impending reset */ 4075 if (ice_check_sq_alive(hw, &hw->mailboxq)) 4076 ice_vc_notify_reset(pf); 4077 4078 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 4079 4080 /* disable the VSIs and their queues that are not already DOWN */ 4081 ice_pf_dis_all_vsi(pf, false); 4082 4083 ice_for_each_vsi(pf, v) 4084 if (pf->vsi[v]) 4085 pf->vsi[v]->vsi_num = 0; 4086 4087 ice_shutdown_all_ctrlq(hw); 4088 } 4089 4090 /** 4091 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 4092 * @pf: board private structure to reinitialize 4093 * 4094 * This routine reinitialize interrupt scheme that was cleared during 4095 * power management suspend callback. 4096 * 4097 * This should be called during resume routine to re-allocate the q_vectors 4098 * and reacquire interrupts. 4099 */ 4100 static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 4101 { 4102 struct device *dev = ice_pf_to_dev(pf); 4103 int ret, v; 4104 4105 /* Since we clear MSIX flag during suspend, we need to 4106 * set it back during resume... 4107 */ 4108 4109 ret = ice_init_interrupt_scheme(pf); 4110 if (ret) { 4111 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 4112 return ret; 4113 } 4114 4115 /* Remap vectors and rings, after successful re-init interrupts */ 4116 ice_for_each_vsi(pf, v) { 4117 if (!pf->vsi[v]) 4118 continue; 4119 4120 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 4121 if (ret) 4122 goto err_reinit; 4123 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 4124 } 4125 4126 ret = ice_req_irq_msix_misc(pf); 4127 if (ret) { 4128 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 4129 ret); 4130 goto err_reinit; 4131 } 4132 4133 return 0; 4134 4135 err_reinit: 4136 while (v--) 4137 if (pf->vsi[v]) 4138 ice_vsi_free_q_vectors(pf->vsi[v]); 4139 4140 return ret; 4141 } 4142 4143 /** 4144 * ice_suspend 4145 * @dev: generic device information structure 4146 * 4147 * Power Management callback to quiesce the device and prepare 4148 * for D3 transition. 4149 */ 4150 static int ice_suspend(struct device *dev) 4151 { 4152 struct pci_dev *pdev = to_pci_dev(dev); 4153 struct ice_pf *pf; 4154 int disabled, v; 4155 4156 pf = pci_get_drvdata(pdev); 4157 4158 if (!ice_pf_state_is_nominal(pf)) { 4159 dev_err(dev, "Device is not ready, no need to suspend it\n"); 4160 return -EBUSY; 4161 } 4162 4163 /* Stop watchdog tasks until resume completion. 4164 * Even though it is most likely that the service task is 4165 * disabled if the device is suspended or down, the service task's 4166 * state is controlled by a different state bit, and we should 4167 * store and honor whatever state that bit is in at this point. 4168 */ 4169 disabled = ice_service_task_stop(pf); 4170 4171 /* Already suspended?, then there is nothing to do */ 4172 if (test_and_set_bit(__ICE_SUSPENDED, pf->state)) { 4173 if (!disabled) 4174 ice_service_task_restart(pf); 4175 return 0; 4176 } 4177 4178 if (test_bit(__ICE_DOWN, pf->state) || 4179 ice_is_reset_in_progress(pf->state)) { 4180 dev_err(dev, "can't suspend device in reset or already down\n"); 4181 if (!disabled) 4182 ice_service_task_restart(pf); 4183 return 0; 4184 } 4185 4186 ice_setup_mc_magic_wake(pf); 4187 4188 ice_prepare_for_shutdown(pf); 4189 4190 ice_set_wake(pf); 4191 4192 /* Free vectors, clear the interrupt scheme and release IRQs 4193 * for proper hibernation, especially with large number of CPUs. 4194 * Otherwise hibernation might fail when mapping all the vectors back 4195 * to CPU0. 4196 */ 4197 ice_free_irq_msix_misc(pf); 4198 ice_for_each_vsi(pf, v) { 4199 if (!pf->vsi[v]) 4200 continue; 4201 ice_vsi_free_q_vectors(pf->vsi[v]); 4202 } 4203 ice_clear_interrupt_scheme(pf); 4204 4205 pci_wake_from_d3(pdev, pf->wol_ena); 4206 pci_set_power_state(pdev, PCI_D3hot); 4207 return 0; 4208 } 4209 4210 /** 4211 * ice_resume - PM callback for waking up from D3 4212 * @dev: generic device information structure 4213 */ 4214 static int ice_resume(struct device *dev) 4215 { 4216 struct pci_dev *pdev = to_pci_dev(dev); 4217 enum ice_reset_req reset_type; 4218 struct ice_pf *pf; 4219 struct ice_hw *hw; 4220 int ret; 4221 4222 pci_set_power_state(pdev, PCI_D0); 4223 pci_restore_state(pdev); 4224 pci_save_state(pdev); 4225 4226 if (!pci_device_is_present(pdev)) 4227 return -ENODEV; 4228 4229 ret = pci_enable_device_mem(pdev); 4230 if (ret) { 4231 dev_err(dev, "Cannot enable device after suspend\n"); 4232 return ret; 4233 } 4234 4235 pf = pci_get_drvdata(pdev); 4236 hw = &pf->hw; 4237 4238 pf->wakeup_reason = rd32(hw, PFPM_WUS); 4239 ice_print_wake_reason(pf); 4240 4241 /* We cleared the interrupt scheme when we suspended, so we need to 4242 * restore it now to resume device functionality. 4243 */ 4244 ret = ice_reinit_interrupt_scheme(pf); 4245 if (ret) 4246 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 4247 4248 clear_bit(__ICE_DOWN, pf->state); 4249 /* Now perform PF reset and rebuild */ 4250 reset_type = ICE_RESET_PFR; 4251 /* re-enable service task for reset, but allow reset to schedule it */ 4252 clear_bit(__ICE_SERVICE_DIS, pf->state); 4253 4254 if (ice_schedule_reset(pf, reset_type)) 4255 dev_err(dev, "Reset during resume failed.\n"); 4256 4257 clear_bit(__ICE_SUSPENDED, pf->state); 4258 ice_service_task_restart(pf); 4259 4260 /* Restart the service task */ 4261 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4262 4263 return 0; 4264 } 4265 #endif /* CONFIG_PM */ 4266 4267 /** 4268 * ice_pci_err_detected - warning that PCI error has been detected 4269 * @pdev: PCI device information struct 4270 * @err: the type of PCI error 4271 * 4272 * Called to warn that something happened on the PCI bus and the error handling 4273 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 4274 */ 4275 static pci_ers_result_t 4276 ice_pci_err_detected(struct pci_dev *pdev, enum pci_channel_state err) 4277 { 4278 struct ice_pf *pf = pci_get_drvdata(pdev); 4279 4280 if (!pf) { 4281 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 4282 __func__, err); 4283 return PCI_ERS_RESULT_DISCONNECT; 4284 } 4285 4286 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4287 ice_service_task_stop(pf); 4288 4289 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4290 set_bit(__ICE_PFR_REQ, pf->state); 4291 ice_prepare_for_reset(pf); 4292 } 4293 } 4294 4295 return PCI_ERS_RESULT_NEED_RESET; 4296 } 4297 4298 /** 4299 * ice_pci_err_slot_reset - a PCI slot reset has just happened 4300 * @pdev: PCI device information struct 4301 * 4302 * Called to determine if the driver can recover from the PCI slot reset by 4303 * using a register read to determine if the device is recoverable. 4304 */ 4305 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 4306 { 4307 struct ice_pf *pf = pci_get_drvdata(pdev); 4308 pci_ers_result_t result; 4309 int err; 4310 u32 reg; 4311 4312 err = pci_enable_device_mem(pdev); 4313 if (err) { 4314 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 4315 err); 4316 result = PCI_ERS_RESULT_DISCONNECT; 4317 } else { 4318 pci_set_master(pdev); 4319 pci_restore_state(pdev); 4320 pci_save_state(pdev); 4321 pci_wake_from_d3(pdev, false); 4322 4323 /* Check for life */ 4324 reg = rd32(&pf->hw, GLGEN_RTRIG); 4325 if (!reg) 4326 result = PCI_ERS_RESULT_RECOVERED; 4327 else 4328 result = PCI_ERS_RESULT_DISCONNECT; 4329 } 4330 4331 err = pci_aer_clear_nonfatal_status(pdev); 4332 if (err) 4333 dev_dbg(&pdev->dev, "pci_aer_clear_nonfatal_status() failed, error %d\n", 4334 err); 4335 /* non-fatal, continue */ 4336 4337 return result; 4338 } 4339 4340 /** 4341 * ice_pci_err_resume - restart operations after PCI error recovery 4342 * @pdev: PCI device information struct 4343 * 4344 * Called to allow the driver to bring things back up after PCI error and/or 4345 * reset recovery have finished 4346 */ 4347 static void ice_pci_err_resume(struct pci_dev *pdev) 4348 { 4349 struct ice_pf *pf = pci_get_drvdata(pdev); 4350 4351 if (!pf) { 4352 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 4353 __func__); 4354 return; 4355 } 4356 4357 if (test_bit(__ICE_SUSPENDED, pf->state)) { 4358 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 4359 __func__); 4360 return; 4361 } 4362 4363 ice_do_reset(pf, ICE_RESET_PFR); 4364 ice_service_task_restart(pf); 4365 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4366 } 4367 4368 /** 4369 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 4370 * @pdev: PCI device information struct 4371 */ 4372 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 4373 { 4374 struct ice_pf *pf = pci_get_drvdata(pdev); 4375 4376 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4377 ice_service_task_stop(pf); 4378 4379 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4380 set_bit(__ICE_PFR_REQ, pf->state); 4381 ice_prepare_for_reset(pf); 4382 } 4383 } 4384 } 4385 4386 /** 4387 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 4388 * @pdev: PCI device information struct 4389 */ 4390 static void ice_pci_err_reset_done(struct pci_dev *pdev) 4391 { 4392 ice_pci_err_resume(pdev); 4393 } 4394 4395 /* ice_pci_tbl - PCI Device ID Table 4396 * 4397 * Wildcard entries (PCI_ANY_ID) should come last 4398 * Last entry must be all 0s 4399 * 4400 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 4401 * Class, Class Mask, private data (not used) } 4402 */ 4403 static const struct pci_device_id ice_pci_tbl[] = { 4404 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 4405 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 4406 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 4407 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 }, 4408 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 }, 4409 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 }, 4410 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 }, 4411 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 }, 4412 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 }, 4413 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 }, 4414 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 }, 4415 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 }, 4416 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 }, 4417 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 }, 4418 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 }, 4419 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 }, 4420 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 }, 4421 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 }, 4422 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 }, 4423 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 }, 4424 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 }, 4425 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 }, 4426 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 }, 4427 /* required last entry */ 4428 { 0, } 4429 }; 4430 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 4431 4432 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 4433 4434 static const struct pci_error_handlers ice_pci_err_handler = { 4435 .error_detected = ice_pci_err_detected, 4436 .slot_reset = ice_pci_err_slot_reset, 4437 .reset_prepare = ice_pci_err_reset_prepare, 4438 .reset_done = ice_pci_err_reset_done, 4439 .resume = ice_pci_err_resume 4440 }; 4441 4442 static struct pci_driver ice_driver = { 4443 .name = KBUILD_MODNAME, 4444 .id_table = ice_pci_tbl, 4445 .probe = ice_probe, 4446 .remove = ice_remove, 4447 #ifdef CONFIG_PM 4448 .driver.pm = &ice_pm_ops, 4449 #endif /* CONFIG_PM */ 4450 .shutdown = ice_shutdown, 4451 .sriov_configure = ice_sriov_configure, 4452 .err_handler = &ice_pci_err_handler 4453 }; 4454 4455 /** 4456 * ice_module_init - Driver registration routine 4457 * 4458 * ice_module_init is the first routine called when the driver is 4459 * loaded. All it does is register with the PCI subsystem. 4460 */ 4461 static int __init ice_module_init(void) 4462 { 4463 int status; 4464 4465 pr_info("%s\n", ice_driver_string); 4466 pr_info("%s\n", ice_copyright); 4467 4468 ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME); 4469 if (!ice_wq) { 4470 pr_err("Failed to create workqueue\n"); 4471 return -ENOMEM; 4472 } 4473 4474 status = pci_register_driver(&ice_driver); 4475 if (status) { 4476 pr_err("failed to register PCI driver, err %d\n", status); 4477 destroy_workqueue(ice_wq); 4478 } 4479 4480 return status; 4481 } 4482 module_init(ice_module_init); 4483 4484 /** 4485 * ice_module_exit - Driver exit cleanup routine 4486 * 4487 * ice_module_exit is called just before the driver is removed 4488 * from memory. 4489 */ 4490 static void __exit ice_module_exit(void) 4491 { 4492 pci_unregister_driver(&ice_driver); 4493 destroy_workqueue(ice_wq); 4494 pr_info("module unloaded\n"); 4495 } 4496 module_exit(ice_module_exit); 4497 4498 /** 4499 * ice_set_mac_address - NDO callback to set MAC address 4500 * @netdev: network interface device structure 4501 * @pi: pointer to an address structure 4502 * 4503 * Returns 0 on success, negative on failure 4504 */ 4505 static int ice_set_mac_address(struct net_device *netdev, void *pi) 4506 { 4507 struct ice_netdev_priv *np = netdev_priv(netdev); 4508 struct ice_vsi *vsi = np->vsi; 4509 struct ice_pf *pf = vsi->back; 4510 struct ice_hw *hw = &pf->hw; 4511 struct sockaddr *addr = pi; 4512 enum ice_status status; 4513 u8 flags = 0; 4514 int err = 0; 4515 u8 *mac; 4516 4517 mac = (u8 *)addr->sa_data; 4518 4519 if (!is_valid_ether_addr(mac)) 4520 return -EADDRNOTAVAIL; 4521 4522 if (ether_addr_equal(netdev->dev_addr, mac)) { 4523 netdev_warn(netdev, "already using mac %pM\n", mac); 4524 return 0; 4525 } 4526 4527 if (test_bit(__ICE_DOWN, pf->state) || 4528 ice_is_reset_in_progress(pf->state)) { 4529 netdev_err(netdev, "can't set mac %pM. device not ready\n", 4530 mac); 4531 return -EBUSY; 4532 } 4533 4534 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 4535 status = ice_fltr_remove_mac(vsi, netdev->dev_addr, ICE_FWD_TO_VSI); 4536 if (status && status != ICE_ERR_DOES_NOT_EXIST) { 4537 err = -EADDRNOTAVAIL; 4538 goto err_update_filters; 4539 } 4540 4541 /* Add filter for new MAC. If filter exists, just return success */ 4542 status = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 4543 if (status == ICE_ERR_ALREADY_EXISTS) { 4544 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 4545 return 0; 4546 } 4547 4548 /* error if the new filter addition failed */ 4549 if (status) 4550 err = -EADDRNOTAVAIL; 4551 4552 err_update_filters: 4553 if (err) { 4554 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 4555 mac); 4556 return err; 4557 } 4558 4559 /* change the netdev's MAC address */ 4560 memcpy(netdev->dev_addr, mac, netdev->addr_len); 4561 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 4562 netdev->dev_addr); 4563 4564 /* write new MAC address to the firmware */ 4565 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 4566 status = ice_aq_manage_mac_write(hw, mac, flags, NULL); 4567 if (status) { 4568 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %s\n", 4569 mac, ice_stat_str(status)); 4570 } 4571 return 0; 4572 } 4573 4574 /** 4575 * ice_set_rx_mode - NDO callback to set the netdev filters 4576 * @netdev: network interface device structure 4577 */ 4578 static void ice_set_rx_mode(struct net_device *netdev) 4579 { 4580 struct ice_netdev_priv *np = netdev_priv(netdev); 4581 struct ice_vsi *vsi = np->vsi; 4582 4583 if (!vsi) 4584 return; 4585 4586 /* Set the flags to synchronize filters 4587 * ndo_set_rx_mode may be triggered even without a change in netdev 4588 * flags 4589 */ 4590 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 4591 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 4592 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 4593 4594 /* schedule our worker thread which will take care of 4595 * applying the new filter changes 4596 */ 4597 ice_service_task_schedule(vsi->back); 4598 } 4599 4600 /** 4601 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 4602 * @netdev: network interface device structure 4603 * @queue_index: Queue ID 4604 * @maxrate: maximum bandwidth in Mbps 4605 */ 4606 static int 4607 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 4608 { 4609 struct ice_netdev_priv *np = netdev_priv(netdev); 4610 struct ice_vsi *vsi = np->vsi; 4611 enum ice_status status; 4612 u16 q_handle; 4613 u8 tc; 4614 4615 /* Validate maxrate requested is within permitted range */ 4616 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 4617 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 4618 maxrate, queue_index); 4619 return -EINVAL; 4620 } 4621 4622 q_handle = vsi->tx_rings[queue_index]->q_handle; 4623 tc = ice_dcb_get_tc(vsi, queue_index); 4624 4625 /* Set BW back to default, when user set maxrate to 0 */ 4626 if (!maxrate) 4627 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 4628 q_handle, ICE_MAX_BW); 4629 else 4630 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 4631 q_handle, ICE_MAX_BW, maxrate * 1000); 4632 if (status) { 4633 netdev_err(netdev, "Unable to set Tx max rate, error %s\n", 4634 ice_stat_str(status)); 4635 return -EIO; 4636 } 4637 4638 return 0; 4639 } 4640 4641 /** 4642 * ice_fdb_add - add an entry to the hardware database 4643 * @ndm: the input from the stack 4644 * @tb: pointer to array of nladdr (unused) 4645 * @dev: the net device pointer 4646 * @addr: the MAC address entry being added 4647 * @vid: VLAN ID 4648 * @flags: instructions from stack about fdb operation 4649 * @extack: netlink extended ack 4650 */ 4651 static int 4652 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 4653 struct net_device *dev, const unsigned char *addr, u16 vid, 4654 u16 flags, struct netlink_ext_ack __always_unused *extack) 4655 { 4656 int err; 4657 4658 if (vid) { 4659 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 4660 return -EINVAL; 4661 } 4662 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 4663 netdev_err(dev, "FDB only supports static addresses\n"); 4664 return -EINVAL; 4665 } 4666 4667 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 4668 err = dev_uc_add_excl(dev, addr); 4669 else if (is_multicast_ether_addr(addr)) 4670 err = dev_mc_add_excl(dev, addr); 4671 else 4672 err = -EINVAL; 4673 4674 /* Only return duplicate errors if NLM_F_EXCL is set */ 4675 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 4676 err = 0; 4677 4678 return err; 4679 } 4680 4681 /** 4682 * ice_fdb_del - delete an entry from the hardware database 4683 * @ndm: the input from the stack 4684 * @tb: pointer to array of nladdr (unused) 4685 * @dev: the net device pointer 4686 * @addr: the MAC address entry being added 4687 * @vid: VLAN ID 4688 */ 4689 static int 4690 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 4691 struct net_device *dev, const unsigned char *addr, 4692 __always_unused u16 vid) 4693 { 4694 int err; 4695 4696 if (ndm->ndm_state & NUD_PERMANENT) { 4697 netdev_err(dev, "FDB only supports static addresses\n"); 4698 return -EINVAL; 4699 } 4700 4701 if (is_unicast_ether_addr(addr)) 4702 err = dev_uc_del(dev, addr); 4703 else if (is_multicast_ether_addr(addr)) 4704 err = dev_mc_del(dev, addr); 4705 else 4706 err = -EINVAL; 4707 4708 return err; 4709 } 4710 4711 /** 4712 * ice_set_features - set the netdev feature flags 4713 * @netdev: ptr to the netdev being adjusted 4714 * @features: the feature set that the stack is suggesting 4715 */ 4716 static int 4717 ice_set_features(struct net_device *netdev, netdev_features_t features) 4718 { 4719 struct ice_netdev_priv *np = netdev_priv(netdev); 4720 struct ice_vsi *vsi = np->vsi; 4721 struct ice_pf *pf = vsi->back; 4722 int ret = 0; 4723 4724 /* Don't set any netdev advanced features with device in Safe Mode */ 4725 if (ice_is_safe_mode(vsi->back)) { 4726 dev_err(ice_pf_to_dev(vsi->back), "Device is in Safe Mode - not enabling advanced netdev features\n"); 4727 return ret; 4728 } 4729 4730 /* Do not change setting during reset */ 4731 if (ice_is_reset_in_progress(pf->state)) { 4732 dev_err(ice_pf_to_dev(vsi->back), "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 4733 return -EBUSY; 4734 } 4735 4736 /* Multiple features can be changed in one call so keep features in 4737 * separate if/else statements to guarantee each feature is checked 4738 */ 4739 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 4740 ret = ice_vsi_manage_rss_lut(vsi, true); 4741 else if (!(features & NETIF_F_RXHASH) && 4742 netdev->features & NETIF_F_RXHASH) 4743 ret = ice_vsi_manage_rss_lut(vsi, false); 4744 4745 if ((features & NETIF_F_HW_VLAN_CTAG_RX) && 4746 !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 4747 ret = ice_vsi_manage_vlan_stripping(vsi, true); 4748 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && 4749 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 4750 ret = ice_vsi_manage_vlan_stripping(vsi, false); 4751 4752 if ((features & NETIF_F_HW_VLAN_CTAG_TX) && 4753 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 4754 ret = ice_vsi_manage_vlan_insertion(vsi); 4755 else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) && 4756 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 4757 ret = ice_vsi_manage_vlan_insertion(vsi); 4758 4759 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && 4760 !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 4761 ret = ice_cfg_vlan_pruning(vsi, true, false); 4762 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && 4763 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 4764 ret = ice_cfg_vlan_pruning(vsi, false, false); 4765 4766 if ((features & NETIF_F_NTUPLE) && 4767 !(netdev->features & NETIF_F_NTUPLE)) { 4768 ice_vsi_manage_fdir(vsi, true); 4769 ice_init_arfs(vsi); 4770 } else if (!(features & NETIF_F_NTUPLE) && 4771 (netdev->features & NETIF_F_NTUPLE)) { 4772 ice_vsi_manage_fdir(vsi, false); 4773 ice_clear_arfs(vsi); 4774 } 4775 4776 return ret; 4777 } 4778 4779 /** 4780 * ice_vsi_vlan_setup - Setup VLAN offload properties on a VSI 4781 * @vsi: VSI to setup VLAN properties for 4782 */ 4783 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 4784 { 4785 int ret = 0; 4786 4787 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 4788 ret = ice_vsi_manage_vlan_stripping(vsi, true); 4789 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX) 4790 ret = ice_vsi_manage_vlan_insertion(vsi); 4791 4792 return ret; 4793 } 4794 4795 /** 4796 * ice_vsi_cfg - Setup the VSI 4797 * @vsi: the VSI being configured 4798 * 4799 * Return 0 on success and negative value on error 4800 */ 4801 int ice_vsi_cfg(struct ice_vsi *vsi) 4802 { 4803 int err; 4804 4805 if (vsi->netdev) { 4806 ice_set_rx_mode(vsi->netdev); 4807 4808 err = ice_vsi_vlan_setup(vsi); 4809 4810 if (err) 4811 return err; 4812 } 4813 ice_vsi_cfg_dcb_rings(vsi); 4814 4815 err = ice_vsi_cfg_lan_txqs(vsi); 4816 if (!err && ice_is_xdp_ena_vsi(vsi)) 4817 err = ice_vsi_cfg_xdp_txqs(vsi); 4818 if (!err) 4819 err = ice_vsi_cfg_rxqs(vsi); 4820 4821 return err; 4822 } 4823 4824 /** 4825 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4826 * @vsi: the VSI being configured 4827 */ 4828 static void ice_napi_enable_all(struct ice_vsi *vsi) 4829 { 4830 int q_idx; 4831 4832 if (!vsi->netdev) 4833 return; 4834 4835 ice_for_each_q_vector(vsi, q_idx) { 4836 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 4837 4838 if (q_vector->rx.ring || q_vector->tx.ring) 4839 napi_enable(&q_vector->napi); 4840 } 4841 } 4842 4843 /** 4844 * ice_up_complete - Finish the last steps of bringing up a connection 4845 * @vsi: The VSI being configured 4846 * 4847 * Return 0 on success and negative value on error 4848 */ 4849 static int ice_up_complete(struct ice_vsi *vsi) 4850 { 4851 struct ice_pf *pf = vsi->back; 4852 int err; 4853 4854 ice_vsi_cfg_msix(vsi); 4855 4856 /* Enable only Rx rings, Tx rings were enabled by the FW when the 4857 * Tx queue group list was configured and the context bits were 4858 * programmed using ice_vsi_cfg_txqs 4859 */ 4860 err = ice_vsi_start_all_rx_rings(vsi); 4861 if (err) 4862 return err; 4863 4864 clear_bit(__ICE_DOWN, vsi->state); 4865 ice_napi_enable_all(vsi); 4866 ice_vsi_ena_irq(vsi); 4867 4868 if (vsi->port_info && 4869 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 4870 vsi->netdev) { 4871 ice_print_link_msg(vsi, true); 4872 netif_tx_start_all_queues(vsi->netdev); 4873 netif_carrier_on(vsi->netdev); 4874 } 4875 4876 ice_service_task_schedule(pf); 4877 4878 return 0; 4879 } 4880 4881 /** 4882 * ice_up - Bring the connection back up after being down 4883 * @vsi: VSI being configured 4884 */ 4885 int ice_up(struct ice_vsi *vsi) 4886 { 4887 int err; 4888 4889 err = ice_vsi_cfg(vsi); 4890 if (!err) 4891 err = ice_up_complete(vsi); 4892 4893 return err; 4894 } 4895 4896 /** 4897 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 4898 * @ring: Tx or Rx ring to read stats from 4899 * @pkts: packets stats counter 4900 * @bytes: bytes stats counter 4901 * 4902 * This function fetches stats from the ring considering the atomic operations 4903 * that needs to be performed to read u64 values in 32 bit machine. 4904 */ 4905 static void 4906 ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts, u64 *bytes) 4907 { 4908 unsigned int start; 4909 *pkts = 0; 4910 *bytes = 0; 4911 4912 if (!ring) 4913 return; 4914 do { 4915 start = u64_stats_fetch_begin_irq(&ring->syncp); 4916 *pkts = ring->stats.pkts; 4917 *bytes = ring->stats.bytes; 4918 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 4919 } 4920 4921 /** 4922 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 4923 * @vsi: the VSI to be updated 4924 * @rings: rings to work on 4925 * @count: number of rings 4926 */ 4927 static void 4928 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, struct ice_ring **rings, 4929 u16 count) 4930 { 4931 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 4932 u16 i; 4933 4934 for (i = 0; i < count; i++) { 4935 struct ice_ring *ring; 4936 u64 pkts, bytes; 4937 4938 ring = READ_ONCE(rings[i]); 4939 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 4940 vsi_stats->tx_packets += pkts; 4941 vsi_stats->tx_bytes += bytes; 4942 vsi->tx_restart += ring->tx_stats.restart_q; 4943 vsi->tx_busy += ring->tx_stats.tx_busy; 4944 vsi->tx_linearize += ring->tx_stats.tx_linearize; 4945 } 4946 } 4947 4948 /** 4949 * ice_update_vsi_ring_stats - Update VSI stats counters 4950 * @vsi: the VSI to be updated 4951 */ 4952 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 4953 { 4954 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 4955 struct ice_ring *ring; 4956 u64 pkts, bytes; 4957 int i; 4958 4959 /* reset netdev stats */ 4960 vsi_stats->tx_packets = 0; 4961 vsi_stats->tx_bytes = 0; 4962 vsi_stats->rx_packets = 0; 4963 vsi_stats->rx_bytes = 0; 4964 4965 /* reset non-netdev (extended) stats */ 4966 vsi->tx_restart = 0; 4967 vsi->tx_busy = 0; 4968 vsi->tx_linearize = 0; 4969 vsi->rx_buf_failed = 0; 4970 vsi->rx_page_failed = 0; 4971 4972 rcu_read_lock(); 4973 4974 /* update Tx rings counters */ 4975 ice_update_vsi_tx_ring_stats(vsi, vsi->tx_rings, vsi->num_txq); 4976 4977 /* update Rx rings counters */ 4978 ice_for_each_rxq(vsi, i) { 4979 ring = READ_ONCE(vsi->rx_rings[i]); 4980 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 4981 vsi_stats->rx_packets += pkts; 4982 vsi_stats->rx_bytes += bytes; 4983 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed; 4984 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; 4985 } 4986 4987 /* update XDP Tx rings counters */ 4988 if (ice_is_xdp_ena_vsi(vsi)) 4989 ice_update_vsi_tx_ring_stats(vsi, vsi->xdp_rings, 4990 vsi->num_xdp_txq); 4991 4992 rcu_read_unlock(); 4993 } 4994 4995 /** 4996 * ice_update_vsi_stats - Update VSI stats counters 4997 * @vsi: the VSI to be updated 4998 */ 4999 void ice_update_vsi_stats(struct ice_vsi *vsi) 5000 { 5001 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 5002 struct ice_eth_stats *cur_es = &vsi->eth_stats; 5003 struct ice_pf *pf = vsi->back; 5004 5005 if (test_bit(__ICE_DOWN, vsi->state) || 5006 test_bit(__ICE_CFG_BUSY, pf->state)) 5007 return; 5008 5009 /* get stats as recorded by Tx/Rx rings */ 5010 ice_update_vsi_ring_stats(vsi); 5011 5012 /* get VSI stats as recorded by the hardware */ 5013 ice_update_eth_stats(vsi); 5014 5015 cur_ns->tx_errors = cur_es->tx_errors; 5016 cur_ns->rx_dropped = cur_es->rx_discards; 5017 cur_ns->tx_dropped = cur_es->tx_discards; 5018 cur_ns->multicast = cur_es->rx_multicast; 5019 5020 /* update some more netdev stats if this is main VSI */ 5021 if (vsi->type == ICE_VSI_PF) { 5022 cur_ns->rx_crc_errors = pf->stats.crc_errors; 5023 cur_ns->rx_errors = pf->stats.crc_errors + 5024 pf->stats.illegal_bytes + 5025 pf->stats.rx_len_errors + 5026 pf->stats.rx_undersize + 5027 pf->hw_csum_rx_error + 5028 pf->stats.rx_jabber + 5029 pf->stats.rx_fragments + 5030 pf->stats.rx_oversize; 5031 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 5032 /* record drops from the port level */ 5033 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 5034 } 5035 } 5036 5037 /** 5038 * ice_update_pf_stats - Update PF port stats counters 5039 * @pf: PF whose stats needs to be updated 5040 */ 5041 void ice_update_pf_stats(struct ice_pf *pf) 5042 { 5043 struct ice_hw_port_stats *prev_ps, *cur_ps; 5044 struct ice_hw *hw = &pf->hw; 5045 u16 fd_ctr_base; 5046 u8 port; 5047 5048 port = hw->port_info->lport; 5049 prev_ps = &pf->stats_prev; 5050 cur_ps = &pf->stats; 5051 5052 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 5053 &prev_ps->eth.rx_bytes, 5054 &cur_ps->eth.rx_bytes); 5055 5056 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 5057 &prev_ps->eth.rx_unicast, 5058 &cur_ps->eth.rx_unicast); 5059 5060 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 5061 &prev_ps->eth.rx_multicast, 5062 &cur_ps->eth.rx_multicast); 5063 5064 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 5065 &prev_ps->eth.rx_broadcast, 5066 &cur_ps->eth.rx_broadcast); 5067 5068 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 5069 &prev_ps->eth.rx_discards, 5070 &cur_ps->eth.rx_discards); 5071 5072 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 5073 &prev_ps->eth.tx_bytes, 5074 &cur_ps->eth.tx_bytes); 5075 5076 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 5077 &prev_ps->eth.tx_unicast, 5078 &cur_ps->eth.tx_unicast); 5079 5080 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 5081 &prev_ps->eth.tx_multicast, 5082 &cur_ps->eth.tx_multicast); 5083 5084 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 5085 &prev_ps->eth.tx_broadcast, 5086 &cur_ps->eth.tx_broadcast); 5087 5088 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 5089 &prev_ps->tx_dropped_link_down, 5090 &cur_ps->tx_dropped_link_down); 5091 5092 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 5093 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 5094 5095 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 5096 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 5097 5098 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 5099 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 5100 5101 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 5102 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 5103 5104 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 5105 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 5106 5107 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 5108 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 5109 5110 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 5111 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 5112 5113 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 5114 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 5115 5116 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 5117 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 5118 5119 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 5120 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 5121 5122 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 5123 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 5124 5125 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 5126 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 5127 5128 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 5129 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 5130 5131 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 5132 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 5133 5134 fd_ctr_base = hw->fd_ctr_base; 5135 5136 ice_stat_update40(hw, 5137 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 5138 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 5139 &cur_ps->fd_sb_match); 5140 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 5141 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 5142 5143 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 5144 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 5145 5146 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 5147 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 5148 5149 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 5150 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 5151 5152 ice_update_dcb_stats(pf); 5153 5154 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 5155 &prev_ps->crc_errors, &cur_ps->crc_errors); 5156 5157 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 5158 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 5159 5160 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 5161 &prev_ps->mac_local_faults, 5162 &cur_ps->mac_local_faults); 5163 5164 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 5165 &prev_ps->mac_remote_faults, 5166 &cur_ps->mac_remote_faults); 5167 5168 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 5169 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 5170 5171 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 5172 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 5173 5174 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 5175 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 5176 5177 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 5178 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 5179 5180 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 5181 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 5182 5183 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 5184 5185 pf->stat_prev_loaded = true; 5186 } 5187 5188 /** 5189 * ice_get_stats64 - get statistics for network device structure 5190 * @netdev: network interface device structure 5191 * @stats: main device statistics structure 5192 */ 5193 static 5194 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 5195 { 5196 struct ice_netdev_priv *np = netdev_priv(netdev); 5197 struct rtnl_link_stats64 *vsi_stats; 5198 struct ice_vsi *vsi = np->vsi; 5199 5200 vsi_stats = &vsi->net_stats; 5201 5202 if (!vsi->num_txq || !vsi->num_rxq) 5203 return; 5204 5205 /* netdev packet/byte stats come from ring counter. These are obtained 5206 * by summing up ring counters (done by ice_update_vsi_ring_stats). 5207 * But, only call the update routine and read the registers if VSI is 5208 * not down. 5209 */ 5210 if (!test_bit(__ICE_DOWN, vsi->state)) 5211 ice_update_vsi_ring_stats(vsi); 5212 stats->tx_packets = vsi_stats->tx_packets; 5213 stats->tx_bytes = vsi_stats->tx_bytes; 5214 stats->rx_packets = vsi_stats->rx_packets; 5215 stats->rx_bytes = vsi_stats->rx_bytes; 5216 5217 /* The rest of the stats can be read from the hardware but instead we 5218 * just return values that the watchdog task has already obtained from 5219 * the hardware. 5220 */ 5221 stats->multicast = vsi_stats->multicast; 5222 stats->tx_errors = vsi_stats->tx_errors; 5223 stats->tx_dropped = vsi_stats->tx_dropped; 5224 stats->rx_errors = vsi_stats->rx_errors; 5225 stats->rx_dropped = vsi_stats->rx_dropped; 5226 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 5227 stats->rx_length_errors = vsi_stats->rx_length_errors; 5228 } 5229 5230 /** 5231 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5232 * @vsi: VSI having NAPI disabled 5233 */ 5234 static void ice_napi_disable_all(struct ice_vsi *vsi) 5235 { 5236 int q_idx; 5237 5238 if (!vsi->netdev) 5239 return; 5240 5241 ice_for_each_q_vector(vsi, q_idx) { 5242 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 5243 5244 if (q_vector->rx.ring || q_vector->tx.ring) 5245 napi_disable(&q_vector->napi); 5246 } 5247 } 5248 5249 /** 5250 * ice_down - Shutdown the connection 5251 * @vsi: The VSI being stopped 5252 */ 5253 int ice_down(struct ice_vsi *vsi) 5254 { 5255 int i, tx_err, rx_err, link_err = 0; 5256 5257 /* Caller of this function is expected to set the 5258 * vsi->state __ICE_DOWN bit 5259 */ 5260 if (vsi->netdev) { 5261 netif_carrier_off(vsi->netdev); 5262 netif_tx_disable(vsi->netdev); 5263 } 5264 5265 ice_vsi_dis_irq(vsi); 5266 5267 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 5268 if (tx_err) 5269 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 5270 vsi->vsi_num, tx_err); 5271 if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { 5272 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 5273 if (tx_err) 5274 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 5275 vsi->vsi_num, tx_err); 5276 } 5277 5278 rx_err = ice_vsi_stop_all_rx_rings(vsi); 5279 if (rx_err) 5280 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 5281 vsi->vsi_num, rx_err); 5282 5283 ice_napi_disable_all(vsi); 5284 5285 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 5286 link_err = ice_force_phys_link_state(vsi, false); 5287 if (link_err) 5288 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 5289 vsi->vsi_num, link_err); 5290 } 5291 5292 ice_for_each_txq(vsi, i) 5293 ice_clean_tx_ring(vsi->tx_rings[i]); 5294 5295 ice_for_each_rxq(vsi, i) 5296 ice_clean_rx_ring(vsi->rx_rings[i]); 5297 5298 if (tx_err || rx_err || link_err) { 5299 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 5300 vsi->vsi_num, vsi->vsw->sw_id); 5301 return -EIO; 5302 } 5303 5304 return 0; 5305 } 5306 5307 /** 5308 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 5309 * @vsi: VSI having resources allocated 5310 * 5311 * Return 0 on success, negative on failure 5312 */ 5313 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 5314 { 5315 int i, err = 0; 5316 5317 if (!vsi->num_txq) { 5318 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 5319 vsi->vsi_num); 5320 return -EINVAL; 5321 } 5322 5323 ice_for_each_txq(vsi, i) { 5324 struct ice_ring *ring = vsi->tx_rings[i]; 5325 5326 if (!ring) 5327 return -EINVAL; 5328 5329 ring->netdev = vsi->netdev; 5330 err = ice_setup_tx_ring(ring); 5331 if (err) 5332 break; 5333 } 5334 5335 return err; 5336 } 5337 5338 /** 5339 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 5340 * @vsi: VSI having resources allocated 5341 * 5342 * Return 0 on success, negative on failure 5343 */ 5344 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 5345 { 5346 int i, err = 0; 5347 5348 if (!vsi->num_rxq) { 5349 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 5350 vsi->vsi_num); 5351 return -EINVAL; 5352 } 5353 5354 ice_for_each_rxq(vsi, i) { 5355 struct ice_ring *ring = vsi->rx_rings[i]; 5356 5357 if (!ring) 5358 return -EINVAL; 5359 5360 ring->netdev = vsi->netdev; 5361 err = ice_setup_rx_ring(ring); 5362 if (err) 5363 break; 5364 } 5365 5366 return err; 5367 } 5368 5369 /** 5370 * ice_vsi_open_ctrl - open control VSI for use 5371 * @vsi: the VSI to open 5372 * 5373 * Initialization of the Control VSI 5374 * 5375 * Returns 0 on success, negative value on error 5376 */ 5377 int ice_vsi_open_ctrl(struct ice_vsi *vsi) 5378 { 5379 char int_name[ICE_INT_NAME_STR_LEN]; 5380 struct ice_pf *pf = vsi->back; 5381 struct device *dev; 5382 int err; 5383 5384 dev = ice_pf_to_dev(pf); 5385 /* allocate descriptors */ 5386 err = ice_vsi_setup_tx_rings(vsi); 5387 if (err) 5388 goto err_setup_tx; 5389 5390 err = ice_vsi_setup_rx_rings(vsi); 5391 if (err) 5392 goto err_setup_rx; 5393 5394 err = ice_vsi_cfg(vsi); 5395 if (err) 5396 goto err_setup_rx; 5397 5398 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 5399 dev_driver_string(dev), dev_name(dev)); 5400 err = ice_vsi_req_irq_msix(vsi, int_name); 5401 if (err) 5402 goto err_setup_rx; 5403 5404 ice_vsi_cfg_msix(vsi); 5405 5406 err = ice_vsi_start_all_rx_rings(vsi); 5407 if (err) 5408 goto err_up_complete; 5409 5410 clear_bit(__ICE_DOWN, vsi->state); 5411 ice_vsi_ena_irq(vsi); 5412 5413 return 0; 5414 5415 err_up_complete: 5416 ice_down(vsi); 5417 err_setup_rx: 5418 ice_vsi_free_rx_rings(vsi); 5419 err_setup_tx: 5420 ice_vsi_free_tx_rings(vsi); 5421 5422 return err; 5423 } 5424 5425 /** 5426 * ice_vsi_open - Called when a network interface is made active 5427 * @vsi: the VSI to open 5428 * 5429 * Initialization of the VSI 5430 * 5431 * Returns 0 on success, negative value on error 5432 */ 5433 static int ice_vsi_open(struct ice_vsi *vsi) 5434 { 5435 char int_name[ICE_INT_NAME_STR_LEN]; 5436 struct ice_pf *pf = vsi->back; 5437 int err; 5438 5439 /* allocate descriptors */ 5440 err = ice_vsi_setup_tx_rings(vsi); 5441 if (err) 5442 goto err_setup_tx; 5443 5444 err = ice_vsi_setup_rx_rings(vsi); 5445 if (err) 5446 goto err_setup_rx; 5447 5448 err = ice_vsi_cfg(vsi); 5449 if (err) 5450 goto err_setup_rx; 5451 5452 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5453 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 5454 err = ice_vsi_req_irq_msix(vsi, int_name); 5455 if (err) 5456 goto err_setup_rx; 5457 5458 /* Notify the stack of the actual queue counts. */ 5459 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 5460 if (err) 5461 goto err_set_qs; 5462 5463 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 5464 if (err) 5465 goto err_set_qs; 5466 5467 err = ice_up_complete(vsi); 5468 if (err) 5469 goto err_up_complete; 5470 5471 return 0; 5472 5473 err_up_complete: 5474 ice_down(vsi); 5475 err_set_qs: 5476 ice_vsi_free_irq(vsi); 5477 err_setup_rx: 5478 ice_vsi_free_rx_rings(vsi); 5479 err_setup_tx: 5480 ice_vsi_free_tx_rings(vsi); 5481 5482 return err; 5483 } 5484 5485 /** 5486 * ice_vsi_release_all - Delete all VSIs 5487 * @pf: PF from which all VSIs are being removed 5488 */ 5489 static void ice_vsi_release_all(struct ice_pf *pf) 5490 { 5491 int err, i; 5492 5493 if (!pf->vsi) 5494 return; 5495 5496 ice_for_each_vsi(pf, i) { 5497 if (!pf->vsi[i]) 5498 continue; 5499 5500 err = ice_vsi_release(pf->vsi[i]); 5501 if (err) 5502 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 5503 i, err, pf->vsi[i]->vsi_num); 5504 } 5505 } 5506 5507 /** 5508 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 5509 * @pf: pointer to the PF instance 5510 * @type: VSI type to rebuild 5511 * 5512 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 5513 */ 5514 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 5515 { 5516 struct device *dev = ice_pf_to_dev(pf); 5517 enum ice_status status; 5518 int i, err; 5519 5520 ice_for_each_vsi(pf, i) { 5521 struct ice_vsi *vsi = pf->vsi[i]; 5522 5523 if (!vsi || vsi->type != type) 5524 continue; 5525 5526 /* rebuild the VSI */ 5527 err = ice_vsi_rebuild(vsi, true); 5528 if (err) { 5529 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 5530 err, vsi->idx, ice_vsi_type_str(type)); 5531 return err; 5532 } 5533 5534 /* replay filters for the VSI */ 5535 status = ice_replay_vsi(&pf->hw, vsi->idx); 5536 if (status) { 5537 dev_err(dev, "replay VSI failed, status %s, VSI index %d, type %s\n", 5538 ice_stat_str(status), vsi->idx, 5539 ice_vsi_type_str(type)); 5540 return -EIO; 5541 } 5542 5543 /* Re-map HW VSI number, using VSI handle that has been 5544 * previously validated in ice_replay_vsi() call above 5545 */ 5546 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 5547 5548 /* enable the VSI */ 5549 err = ice_ena_vsi(vsi, false); 5550 if (err) { 5551 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 5552 err, vsi->idx, ice_vsi_type_str(type)); 5553 return err; 5554 } 5555 5556 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 5557 ice_vsi_type_str(type)); 5558 } 5559 5560 return 0; 5561 } 5562 5563 /** 5564 * ice_update_pf_netdev_link - Update PF netdev link status 5565 * @pf: pointer to the PF instance 5566 */ 5567 static void ice_update_pf_netdev_link(struct ice_pf *pf) 5568 { 5569 bool link_up; 5570 int i; 5571 5572 ice_for_each_vsi(pf, i) { 5573 struct ice_vsi *vsi = pf->vsi[i]; 5574 5575 if (!vsi || vsi->type != ICE_VSI_PF) 5576 return; 5577 5578 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 5579 if (link_up) { 5580 netif_carrier_on(pf->vsi[i]->netdev); 5581 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 5582 } else { 5583 netif_carrier_off(pf->vsi[i]->netdev); 5584 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 5585 } 5586 } 5587 } 5588 5589 /** 5590 * ice_rebuild - rebuild after reset 5591 * @pf: PF to rebuild 5592 * @reset_type: type of reset 5593 * 5594 * Do not rebuild VF VSI in this flow because that is already handled via 5595 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 5596 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 5597 * to reset/rebuild all the VF VSI twice. 5598 */ 5599 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 5600 { 5601 struct device *dev = ice_pf_to_dev(pf); 5602 struct ice_hw *hw = &pf->hw; 5603 enum ice_status ret; 5604 int err; 5605 5606 if (test_bit(__ICE_DOWN, pf->state)) 5607 goto clear_recovery; 5608 5609 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 5610 5611 ret = ice_init_all_ctrlq(hw); 5612 if (ret) { 5613 dev_err(dev, "control queues init failed %s\n", 5614 ice_stat_str(ret)); 5615 goto err_init_ctrlq; 5616 } 5617 5618 /* if DDP was previously loaded successfully */ 5619 if (!ice_is_safe_mode(pf)) { 5620 /* reload the SW DB of filter tables */ 5621 if (reset_type == ICE_RESET_PFR) 5622 ice_fill_blk_tbls(hw); 5623 else 5624 /* Reload DDP Package after CORER/GLOBR reset */ 5625 ice_load_pkg(NULL, pf); 5626 } 5627 5628 ret = ice_clear_pf_cfg(hw); 5629 if (ret) { 5630 dev_err(dev, "clear PF configuration failed %s\n", 5631 ice_stat_str(ret)); 5632 goto err_init_ctrlq; 5633 } 5634 5635 if (pf->first_sw->dflt_vsi_ena) 5636 dev_info(dev, "Clearing default VSI, re-enable after reset completes\n"); 5637 /* clear the default VSI configuration if it exists */ 5638 pf->first_sw->dflt_vsi = NULL; 5639 pf->first_sw->dflt_vsi_ena = false; 5640 5641 ice_clear_pxe_mode(hw); 5642 5643 ret = ice_get_caps(hw); 5644 if (ret) { 5645 dev_err(dev, "ice_get_caps failed %s\n", ice_stat_str(ret)); 5646 goto err_init_ctrlq; 5647 } 5648 5649 ret = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 5650 if (ret) { 5651 dev_err(dev, "set_mac_cfg failed %s\n", ice_stat_str(ret)); 5652 goto err_init_ctrlq; 5653 } 5654 5655 err = ice_sched_init_port(hw->port_info); 5656 if (err) 5657 goto err_sched_init_port; 5658 5659 err = ice_update_link_info(hw->port_info); 5660 if (err) 5661 dev_err(dev, "Get link status error %d\n", err); 5662 5663 /* start misc vector */ 5664 err = ice_req_irq_msix_misc(pf); 5665 if (err) { 5666 dev_err(dev, "misc vector setup failed: %d\n", err); 5667 goto err_sched_init_port; 5668 } 5669 5670 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 5671 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 5672 if (!rd32(hw, PFQF_FD_SIZE)) { 5673 u16 unused, guar, b_effort; 5674 5675 guar = hw->func_caps.fd_fltr_guar; 5676 b_effort = hw->func_caps.fd_fltr_best_effort; 5677 5678 /* force guaranteed filter pool for PF */ 5679 ice_alloc_fd_guar_item(hw, &unused, guar); 5680 /* force shared filter pool for PF */ 5681 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 5682 } 5683 } 5684 5685 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 5686 ice_dcb_rebuild(pf); 5687 5688 /* rebuild PF VSI */ 5689 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 5690 if (err) { 5691 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 5692 goto err_vsi_rebuild; 5693 } 5694 5695 /* If Flow Director is active */ 5696 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 5697 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 5698 if (err) { 5699 dev_err(dev, "control VSI rebuild failed: %d\n", err); 5700 goto err_vsi_rebuild; 5701 } 5702 5703 /* replay HW Flow Director recipes */ 5704 if (hw->fdir_prof) 5705 ice_fdir_replay_flows(hw); 5706 5707 /* replay Flow Director filters */ 5708 ice_fdir_replay_fltrs(pf); 5709 5710 ice_rebuild_arfs(pf); 5711 } 5712 5713 ice_update_pf_netdev_link(pf); 5714 5715 /* tell the firmware we are up */ 5716 ret = ice_send_version(pf); 5717 if (ret) { 5718 dev_err(dev, "Rebuild failed due to error sending driver version: %s\n", 5719 ice_stat_str(ret)); 5720 goto err_vsi_rebuild; 5721 } 5722 5723 ice_replay_post(hw); 5724 5725 /* if we get here, reset flow is successful */ 5726 clear_bit(__ICE_RESET_FAILED, pf->state); 5727 return; 5728 5729 err_vsi_rebuild: 5730 err_sched_init_port: 5731 ice_sched_cleanup_all(hw); 5732 err_init_ctrlq: 5733 ice_shutdown_all_ctrlq(hw); 5734 set_bit(__ICE_RESET_FAILED, pf->state); 5735 clear_recovery: 5736 /* set this bit in PF state to control service task scheduling */ 5737 set_bit(__ICE_NEEDS_RESTART, pf->state); 5738 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 5739 } 5740 5741 /** 5742 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 5743 * @vsi: Pointer to VSI structure 5744 */ 5745 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 5746 { 5747 if (PAGE_SIZE >= 8192 || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 5748 return ICE_RXBUF_2048 - XDP_PACKET_HEADROOM; 5749 else 5750 return ICE_RXBUF_3072; 5751 } 5752 5753 /** 5754 * ice_change_mtu - NDO callback to change the MTU 5755 * @netdev: network interface device structure 5756 * @new_mtu: new value for maximum frame size 5757 * 5758 * Returns 0 on success, negative on failure 5759 */ 5760 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 5761 { 5762 struct ice_netdev_priv *np = netdev_priv(netdev); 5763 struct ice_vsi *vsi = np->vsi; 5764 struct ice_pf *pf = vsi->back; 5765 u8 count = 0; 5766 5767 if (new_mtu == (int)netdev->mtu) { 5768 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 5769 return 0; 5770 } 5771 5772 if (ice_is_xdp_ena_vsi(vsi)) { 5773 int frame_size = ice_max_xdp_frame_size(vsi); 5774 5775 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 5776 netdev_err(netdev, "max MTU for XDP usage is %d\n", 5777 frame_size - ICE_ETH_PKT_HDR_PAD); 5778 return -EINVAL; 5779 } 5780 } 5781 5782 if (new_mtu < (int)netdev->min_mtu) { 5783 netdev_err(netdev, "new MTU invalid. min_mtu is %d\n", 5784 netdev->min_mtu); 5785 return -EINVAL; 5786 } else if (new_mtu > (int)netdev->max_mtu) { 5787 netdev_err(netdev, "new MTU invalid. max_mtu is %d\n", 5788 netdev->min_mtu); 5789 return -EINVAL; 5790 } 5791 /* if a reset is in progress, wait for some time for it to complete */ 5792 do { 5793 if (ice_is_reset_in_progress(pf->state)) { 5794 count++; 5795 usleep_range(1000, 2000); 5796 } else { 5797 break; 5798 } 5799 5800 } while (count < 100); 5801 5802 if (count == 100) { 5803 netdev_err(netdev, "can't change MTU. Device is busy\n"); 5804 return -EBUSY; 5805 } 5806 5807 netdev->mtu = (unsigned int)new_mtu; 5808 5809 /* if VSI is up, bring it down and then back up */ 5810 if (!test_and_set_bit(__ICE_DOWN, vsi->state)) { 5811 int err; 5812 5813 err = ice_down(vsi); 5814 if (err) { 5815 netdev_err(netdev, "change MTU if_up err %d\n", err); 5816 return err; 5817 } 5818 5819 err = ice_up(vsi); 5820 if (err) { 5821 netdev_err(netdev, "change MTU if_up err %d\n", err); 5822 return err; 5823 } 5824 } 5825 5826 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 5827 return 0; 5828 } 5829 5830 /** 5831 * ice_aq_str - convert AQ err code to a string 5832 * @aq_err: the AQ error code to convert 5833 */ 5834 const char *ice_aq_str(enum ice_aq_err aq_err) 5835 { 5836 switch (aq_err) { 5837 case ICE_AQ_RC_OK: 5838 return "OK"; 5839 case ICE_AQ_RC_EPERM: 5840 return "ICE_AQ_RC_EPERM"; 5841 case ICE_AQ_RC_ENOENT: 5842 return "ICE_AQ_RC_ENOENT"; 5843 case ICE_AQ_RC_ENOMEM: 5844 return "ICE_AQ_RC_ENOMEM"; 5845 case ICE_AQ_RC_EBUSY: 5846 return "ICE_AQ_RC_EBUSY"; 5847 case ICE_AQ_RC_EEXIST: 5848 return "ICE_AQ_RC_EEXIST"; 5849 case ICE_AQ_RC_EINVAL: 5850 return "ICE_AQ_RC_EINVAL"; 5851 case ICE_AQ_RC_ENOSPC: 5852 return "ICE_AQ_RC_ENOSPC"; 5853 case ICE_AQ_RC_ENOSYS: 5854 return "ICE_AQ_RC_ENOSYS"; 5855 case ICE_AQ_RC_EMODE: 5856 return "ICE_AQ_RC_EMODE"; 5857 case ICE_AQ_RC_ENOSEC: 5858 return "ICE_AQ_RC_ENOSEC"; 5859 case ICE_AQ_RC_EBADSIG: 5860 return "ICE_AQ_RC_EBADSIG"; 5861 case ICE_AQ_RC_ESVN: 5862 return "ICE_AQ_RC_ESVN"; 5863 case ICE_AQ_RC_EBADMAN: 5864 return "ICE_AQ_RC_EBADMAN"; 5865 case ICE_AQ_RC_EBADBUF: 5866 return "ICE_AQ_RC_EBADBUF"; 5867 } 5868 5869 return "ICE_AQ_RC_UNKNOWN"; 5870 } 5871 5872 /** 5873 * ice_stat_str - convert status err code to a string 5874 * @stat_err: the status error code to convert 5875 */ 5876 const char *ice_stat_str(enum ice_status stat_err) 5877 { 5878 switch (stat_err) { 5879 case ICE_SUCCESS: 5880 return "OK"; 5881 case ICE_ERR_PARAM: 5882 return "ICE_ERR_PARAM"; 5883 case ICE_ERR_NOT_IMPL: 5884 return "ICE_ERR_NOT_IMPL"; 5885 case ICE_ERR_NOT_READY: 5886 return "ICE_ERR_NOT_READY"; 5887 case ICE_ERR_NOT_SUPPORTED: 5888 return "ICE_ERR_NOT_SUPPORTED"; 5889 case ICE_ERR_BAD_PTR: 5890 return "ICE_ERR_BAD_PTR"; 5891 case ICE_ERR_INVAL_SIZE: 5892 return "ICE_ERR_INVAL_SIZE"; 5893 case ICE_ERR_DEVICE_NOT_SUPPORTED: 5894 return "ICE_ERR_DEVICE_NOT_SUPPORTED"; 5895 case ICE_ERR_RESET_FAILED: 5896 return "ICE_ERR_RESET_FAILED"; 5897 case ICE_ERR_FW_API_VER: 5898 return "ICE_ERR_FW_API_VER"; 5899 case ICE_ERR_NO_MEMORY: 5900 return "ICE_ERR_NO_MEMORY"; 5901 case ICE_ERR_CFG: 5902 return "ICE_ERR_CFG"; 5903 case ICE_ERR_OUT_OF_RANGE: 5904 return "ICE_ERR_OUT_OF_RANGE"; 5905 case ICE_ERR_ALREADY_EXISTS: 5906 return "ICE_ERR_ALREADY_EXISTS"; 5907 case ICE_ERR_NVM_CHECKSUM: 5908 return "ICE_ERR_NVM_CHECKSUM"; 5909 case ICE_ERR_BUF_TOO_SHORT: 5910 return "ICE_ERR_BUF_TOO_SHORT"; 5911 case ICE_ERR_NVM_BLANK_MODE: 5912 return "ICE_ERR_NVM_BLANK_MODE"; 5913 case ICE_ERR_IN_USE: 5914 return "ICE_ERR_IN_USE"; 5915 case ICE_ERR_MAX_LIMIT: 5916 return "ICE_ERR_MAX_LIMIT"; 5917 case ICE_ERR_RESET_ONGOING: 5918 return "ICE_ERR_RESET_ONGOING"; 5919 case ICE_ERR_HW_TABLE: 5920 return "ICE_ERR_HW_TABLE"; 5921 case ICE_ERR_DOES_NOT_EXIST: 5922 return "ICE_ERR_DOES_NOT_EXIST"; 5923 case ICE_ERR_FW_DDP_MISMATCH: 5924 return "ICE_ERR_FW_DDP_MISMATCH"; 5925 case ICE_ERR_AQ_ERROR: 5926 return "ICE_ERR_AQ_ERROR"; 5927 case ICE_ERR_AQ_TIMEOUT: 5928 return "ICE_ERR_AQ_TIMEOUT"; 5929 case ICE_ERR_AQ_FULL: 5930 return "ICE_ERR_AQ_FULL"; 5931 case ICE_ERR_AQ_NO_WORK: 5932 return "ICE_ERR_AQ_NO_WORK"; 5933 case ICE_ERR_AQ_EMPTY: 5934 return "ICE_ERR_AQ_EMPTY"; 5935 case ICE_ERR_AQ_FW_CRITICAL: 5936 return "ICE_ERR_AQ_FW_CRITICAL"; 5937 } 5938 5939 return "ICE_ERR_UNKNOWN"; 5940 } 5941 5942 /** 5943 * ice_set_rss - Set RSS keys and lut 5944 * @vsi: Pointer to VSI structure 5945 * @seed: RSS hash seed 5946 * @lut: Lookup table 5947 * @lut_size: Lookup table size 5948 * 5949 * Returns 0 on success, negative on failure 5950 */ 5951 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 5952 { 5953 struct ice_pf *pf = vsi->back; 5954 struct ice_hw *hw = &pf->hw; 5955 enum ice_status status; 5956 struct device *dev; 5957 5958 dev = ice_pf_to_dev(pf); 5959 if (seed) { 5960 struct ice_aqc_get_set_rss_keys *buf = 5961 (struct ice_aqc_get_set_rss_keys *)seed; 5962 5963 status = ice_aq_set_rss_key(hw, vsi->idx, buf); 5964 5965 if (status) { 5966 dev_err(dev, "Cannot set RSS key, err %s aq_err %s\n", 5967 ice_stat_str(status), 5968 ice_aq_str(hw->adminq.sq_last_status)); 5969 return -EIO; 5970 } 5971 } 5972 5973 if (lut) { 5974 status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 5975 lut, lut_size); 5976 if (status) { 5977 dev_err(dev, "Cannot set RSS lut, err %s aq_err %s\n", 5978 ice_stat_str(status), 5979 ice_aq_str(hw->adminq.sq_last_status)); 5980 return -EIO; 5981 } 5982 } 5983 5984 return 0; 5985 } 5986 5987 /** 5988 * ice_get_rss - Get RSS keys and lut 5989 * @vsi: Pointer to VSI structure 5990 * @seed: Buffer to store the keys 5991 * @lut: Buffer to store the lookup table entries 5992 * @lut_size: Size of buffer to store the lookup table entries 5993 * 5994 * Returns 0 on success, negative on failure 5995 */ 5996 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 5997 { 5998 struct ice_pf *pf = vsi->back; 5999 struct ice_hw *hw = &pf->hw; 6000 enum ice_status status; 6001 struct device *dev; 6002 6003 dev = ice_pf_to_dev(pf); 6004 if (seed) { 6005 struct ice_aqc_get_set_rss_keys *buf = 6006 (struct ice_aqc_get_set_rss_keys *)seed; 6007 6008 status = ice_aq_get_rss_key(hw, vsi->idx, buf); 6009 if (status) { 6010 dev_err(dev, "Cannot get RSS key, err %s aq_err %s\n", 6011 ice_stat_str(status), 6012 ice_aq_str(hw->adminq.sq_last_status)); 6013 return -EIO; 6014 } 6015 } 6016 6017 if (lut) { 6018 status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 6019 lut, lut_size); 6020 if (status) { 6021 dev_err(dev, "Cannot get RSS lut, err %s aq_err %s\n", 6022 ice_stat_str(status), 6023 ice_aq_str(hw->adminq.sq_last_status)); 6024 return -EIO; 6025 } 6026 } 6027 6028 return 0; 6029 } 6030 6031 /** 6032 * ice_bridge_getlink - Get the hardware bridge mode 6033 * @skb: skb buff 6034 * @pid: process ID 6035 * @seq: RTNL message seq 6036 * @dev: the netdev being configured 6037 * @filter_mask: filter mask passed in 6038 * @nlflags: netlink flags passed in 6039 * 6040 * Return the bridge mode (VEB/VEPA) 6041 */ 6042 static int 6043 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 6044 struct net_device *dev, u32 filter_mask, int nlflags) 6045 { 6046 struct ice_netdev_priv *np = netdev_priv(dev); 6047 struct ice_vsi *vsi = np->vsi; 6048 struct ice_pf *pf = vsi->back; 6049 u16 bmode; 6050 6051 bmode = pf->first_sw->bridge_mode; 6052 6053 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 6054 filter_mask, NULL); 6055 } 6056 6057 /** 6058 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 6059 * @vsi: Pointer to VSI structure 6060 * @bmode: Hardware bridge mode (VEB/VEPA) 6061 * 6062 * Returns 0 on success, negative on failure 6063 */ 6064 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 6065 { 6066 struct ice_aqc_vsi_props *vsi_props; 6067 struct ice_hw *hw = &vsi->back->hw; 6068 struct ice_vsi_ctx *ctxt; 6069 enum ice_status status; 6070 int ret = 0; 6071 6072 vsi_props = &vsi->info; 6073 6074 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 6075 if (!ctxt) 6076 return -ENOMEM; 6077 6078 ctxt->info = vsi->info; 6079 6080 if (bmode == BRIDGE_MODE_VEB) 6081 /* change from VEPA to VEB mode */ 6082 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6083 else 6084 /* change from VEB to VEPA mode */ 6085 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6086 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 6087 6088 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 6089 if (status) { 6090 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %s aq_err %s\n", 6091 bmode, ice_stat_str(status), 6092 ice_aq_str(hw->adminq.sq_last_status)); 6093 ret = -EIO; 6094 goto out; 6095 } 6096 /* Update sw flags for book keeping */ 6097 vsi_props->sw_flags = ctxt->info.sw_flags; 6098 6099 out: 6100 kfree(ctxt); 6101 return ret; 6102 } 6103 6104 /** 6105 * ice_bridge_setlink - Set the hardware bridge mode 6106 * @dev: the netdev being configured 6107 * @nlh: RTNL message 6108 * @flags: bridge setlink flags 6109 * @extack: netlink extended ack 6110 * 6111 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 6112 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 6113 * not already set for all VSIs connected to this switch. And also update the 6114 * unicast switch filter rules for the corresponding switch of the netdev. 6115 */ 6116 static int 6117 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 6118 u16 __always_unused flags, 6119 struct netlink_ext_ack __always_unused *extack) 6120 { 6121 struct ice_netdev_priv *np = netdev_priv(dev); 6122 struct ice_pf *pf = np->vsi->back; 6123 struct nlattr *attr, *br_spec; 6124 struct ice_hw *hw = &pf->hw; 6125 enum ice_status status; 6126 struct ice_sw *pf_sw; 6127 int rem, v, err = 0; 6128 6129 pf_sw = pf->first_sw; 6130 /* find the attribute in the netlink message */ 6131 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 6132 6133 nla_for_each_nested(attr, br_spec, rem) { 6134 __u16 mode; 6135 6136 if (nla_type(attr) != IFLA_BRIDGE_MODE) 6137 continue; 6138 mode = nla_get_u16(attr); 6139 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 6140 return -EINVAL; 6141 /* Continue if bridge mode is not being flipped */ 6142 if (mode == pf_sw->bridge_mode) 6143 continue; 6144 /* Iterates through the PF VSI list and update the loopback 6145 * mode of the VSI 6146 */ 6147 ice_for_each_vsi(pf, v) { 6148 if (!pf->vsi[v]) 6149 continue; 6150 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 6151 if (err) 6152 return err; 6153 } 6154 6155 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 6156 /* Update the unicast switch filter rules for the corresponding 6157 * switch of the netdev 6158 */ 6159 status = ice_update_sw_rule_bridge_mode(hw); 6160 if (status) { 6161 netdev_err(dev, "switch rule update failed, mode = %d err %s aq_err %s\n", 6162 mode, ice_stat_str(status), 6163 ice_aq_str(hw->adminq.sq_last_status)); 6164 /* revert hw->evb_veb */ 6165 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 6166 return -EIO; 6167 } 6168 6169 pf_sw->bridge_mode = mode; 6170 } 6171 6172 return 0; 6173 } 6174 6175 /** 6176 * ice_tx_timeout - Respond to a Tx Hang 6177 * @netdev: network interface device structure 6178 * @txqueue: Tx queue 6179 */ 6180 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 6181 { 6182 struct ice_netdev_priv *np = netdev_priv(netdev); 6183 struct ice_ring *tx_ring = NULL; 6184 struct ice_vsi *vsi = np->vsi; 6185 struct ice_pf *pf = vsi->back; 6186 u32 i; 6187 6188 pf->tx_timeout_count++; 6189 6190 /* Check if PFC is enabled for the TC to which the queue belongs 6191 * to. If yes then Tx timeout is not caused by a hung queue, no 6192 * need to reset and rebuild 6193 */ 6194 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 6195 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 6196 txqueue); 6197 return; 6198 } 6199 6200 /* now that we have an index, find the tx_ring struct */ 6201 for (i = 0; i < vsi->num_txq; i++) 6202 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 6203 if (txqueue == vsi->tx_rings[i]->q_index) { 6204 tx_ring = vsi->tx_rings[i]; 6205 break; 6206 } 6207 6208 /* Reset recovery level if enough time has elapsed after last timeout. 6209 * Also ensure no new reset action happens before next timeout period. 6210 */ 6211 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 6212 pf->tx_timeout_recovery_level = 1; 6213 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 6214 netdev->watchdog_timeo))) 6215 return; 6216 6217 if (tx_ring) { 6218 struct ice_hw *hw = &pf->hw; 6219 u32 head, val = 0; 6220 6221 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) & 6222 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 6223 /* Read interrupt register */ 6224 val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 6225 6226 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 6227 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 6228 head, tx_ring->next_to_use, val); 6229 } 6230 6231 pf->tx_timeout_last_recovery = jiffies; 6232 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 6233 pf->tx_timeout_recovery_level, txqueue); 6234 6235 switch (pf->tx_timeout_recovery_level) { 6236 case 1: 6237 set_bit(__ICE_PFR_REQ, pf->state); 6238 break; 6239 case 2: 6240 set_bit(__ICE_CORER_REQ, pf->state); 6241 break; 6242 case 3: 6243 set_bit(__ICE_GLOBR_REQ, pf->state); 6244 break; 6245 default: 6246 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 6247 set_bit(__ICE_DOWN, pf->state); 6248 set_bit(__ICE_NEEDS_RESTART, vsi->state); 6249 set_bit(__ICE_SERVICE_DIS, pf->state); 6250 break; 6251 } 6252 6253 ice_service_task_schedule(pf); 6254 pf->tx_timeout_recovery_level++; 6255 } 6256 6257 /** 6258 * ice_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 6259 * @netdev: This physical port's netdev 6260 * @ti: Tunnel endpoint information 6261 */ 6262 static void 6263 ice_udp_tunnel_add(struct net_device *netdev, struct udp_tunnel_info *ti) 6264 { 6265 struct ice_netdev_priv *np = netdev_priv(netdev); 6266 struct ice_vsi *vsi = np->vsi; 6267 struct ice_pf *pf = vsi->back; 6268 enum ice_tunnel_type tnl_type; 6269 u16 port = ntohs(ti->port); 6270 enum ice_status status; 6271 6272 switch (ti->type) { 6273 case UDP_TUNNEL_TYPE_VXLAN: 6274 tnl_type = TNL_VXLAN; 6275 break; 6276 case UDP_TUNNEL_TYPE_GENEVE: 6277 tnl_type = TNL_GENEVE; 6278 break; 6279 default: 6280 netdev_err(netdev, "Unknown tunnel type\n"); 6281 return; 6282 } 6283 6284 status = ice_create_tunnel(&pf->hw, tnl_type, port); 6285 if (status == ICE_ERR_OUT_OF_RANGE) 6286 netdev_info(netdev, "Max tunneled UDP ports reached, port %d not added\n", 6287 port); 6288 else if (status) 6289 netdev_err(netdev, "Error adding UDP tunnel - %s\n", 6290 ice_stat_str(status)); 6291 } 6292 6293 /** 6294 * ice_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 6295 * @netdev: This physical port's netdev 6296 * @ti: Tunnel endpoint information 6297 */ 6298 static void 6299 ice_udp_tunnel_del(struct net_device *netdev, struct udp_tunnel_info *ti) 6300 { 6301 struct ice_netdev_priv *np = netdev_priv(netdev); 6302 struct ice_vsi *vsi = np->vsi; 6303 struct ice_pf *pf = vsi->back; 6304 u16 port = ntohs(ti->port); 6305 enum ice_status status; 6306 bool retval; 6307 6308 retval = ice_tunnel_port_in_use(&pf->hw, port, NULL); 6309 if (!retval) { 6310 netdev_info(netdev, "port %d not found in UDP tunnels list\n", 6311 port); 6312 return; 6313 } 6314 6315 status = ice_destroy_tunnel(&pf->hw, port, false); 6316 if (status) 6317 netdev_err(netdev, "error deleting port %d from UDP tunnels list\n", 6318 port); 6319 } 6320 6321 /** 6322 * ice_open - Called when a network interface becomes active 6323 * @netdev: network interface device structure 6324 * 6325 * The open entry point is called when a network interface is made 6326 * active by the system (IFF_UP). At this point all resources needed 6327 * for transmit and receive operations are allocated, the interrupt 6328 * handler is registered with the OS, the netdev watchdog is enabled, 6329 * and the stack is notified that the interface is ready. 6330 * 6331 * Returns 0 on success, negative value on failure 6332 */ 6333 int ice_open(struct net_device *netdev) 6334 { 6335 struct ice_netdev_priv *np = netdev_priv(netdev); 6336 struct ice_vsi *vsi = np->vsi; 6337 struct ice_pf *pf = vsi->back; 6338 struct ice_port_info *pi; 6339 int err; 6340 6341 if (test_bit(__ICE_NEEDS_RESTART, pf->state)) { 6342 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 6343 return -EIO; 6344 } 6345 6346 if (test_bit(__ICE_DOWN, pf->state)) { 6347 netdev_err(netdev, "device is not ready yet\n"); 6348 return -EBUSY; 6349 } 6350 6351 netif_carrier_off(netdev); 6352 6353 pi = vsi->port_info; 6354 err = ice_update_link_info(pi); 6355 if (err) { 6356 netdev_err(netdev, "Failed to get link info, error %d\n", 6357 err); 6358 return err; 6359 } 6360 6361 /* Set PHY if there is media, otherwise, turn off PHY */ 6362 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 6363 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6364 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) { 6365 err = ice_init_phy_user_cfg(pi); 6366 if (err) { 6367 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 6368 err); 6369 return err; 6370 } 6371 } 6372 6373 err = ice_configure_phy(vsi); 6374 if (err) { 6375 netdev_err(netdev, "Failed to set physical link up, error %d\n", 6376 err); 6377 return err; 6378 } 6379 } else { 6380 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6381 err = ice_aq_set_link_restart_an(pi, false, NULL); 6382 if (err) { 6383 netdev_err(netdev, "Failed to set PHY state, VSI %d error %d\n", 6384 vsi->vsi_num, err); 6385 return err; 6386 } 6387 } 6388 6389 err = ice_vsi_open(vsi); 6390 if (err) 6391 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 6392 vsi->vsi_num, vsi->vsw->sw_id); 6393 6394 /* Update existing tunnels information */ 6395 udp_tunnel_get_rx_info(netdev); 6396 6397 return err; 6398 } 6399 6400 /** 6401 * ice_stop - Disables a network interface 6402 * @netdev: network interface device structure 6403 * 6404 * The stop entry point is called when an interface is de-activated by the OS, 6405 * and the netdevice enters the DOWN state. The hardware is still under the 6406 * driver's control, but the netdev interface is disabled. 6407 * 6408 * Returns success only - not allowed to fail 6409 */ 6410 int ice_stop(struct net_device *netdev) 6411 { 6412 struct ice_netdev_priv *np = netdev_priv(netdev); 6413 struct ice_vsi *vsi = np->vsi; 6414 6415 ice_vsi_close(vsi); 6416 6417 return 0; 6418 } 6419 6420 /** 6421 * ice_features_check - Validate encapsulated packet conforms to limits 6422 * @skb: skb buffer 6423 * @netdev: This port's netdev 6424 * @features: Offload features that the stack believes apply 6425 */ 6426 static netdev_features_t 6427 ice_features_check(struct sk_buff *skb, 6428 struct net_device __always_unused *netdev, 6429 netdev_features_t features) 6430 { 6431 size_t len; 6432 6433 /* No point in doing any of this if neither checksum nor GSO are 6434 * being requested for this frame. We can rule out both by just 6435 * checking for CHECKSUM_PARTIAL 6436 */ 6437 if (skb->ip_summed != CHECKSUM_PARTIAL) 6438 return features; 6439 6440 /* We cannot support GSO if the MSS is going to be less than 6441 * 64 bytes. If it is then we need to drop support for GSO. 6442 */ 6443 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 6444 features &= ~NETIF_F_GSO_MASK; 6445 6446 len = skb_network_header(skb) - skb->data; 6447 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 6448 goto out_rm_features; 6449 6450 len = skb_transport_header(skb) - skb_network_header(skb); 6451 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6452 goto out_rm_features; 6453 6454 if (skb->encapsulation) { 6455 len = skb_inner_network_header(skb) - skb_transport_header(skb); 6456 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 6457 goto out_rm_features; 6458 6459 len = skb_inner_transport_header(skb) - 6460 skb_inner_network_header(skb); 6461 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6462 goto out_rm_features; 6463 } 6464 6465 return features; 6466 out_rm_features: 6467 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 6468 } 6469 6470 static const struct net_device_ops ice_netdev_safe_mode_ops = { 6471 .ndo_open = ice_open, 6472 .ndo_stop = ice_stop, 6473 .ndo_start_xmit = ice_start_xmit, 6474 .ndo_set_mac_address = ice_set_mac_address, 6475 .ndo_validate_addr = eth_validate_addr, 6476 .ndo_change_mtu = ice_change_mtu, 6477 .ndo_get_stats64 = ice_get_stats64, 6478 .ndo_tx_timeout = ice_tx_timeout, 6479 }; 6480 6481 static const struct net_device_ops ice_netdev_ops = { 6482 .ndo_open = ice_open, 6483 .ndo_stop = ice_stop, 6484 .ndo_start_xmit = ice_start_xmit, 6485 .ndo_features_check = ice_features_check, 6486 .ndo_set_rx_mode = ice_set_rx_mode, 6487 .ndo_set_mac_address = ice_set_mac_address, 6488 .ndo_validate_addr = eth_validate_addr, 6489 .ndo_change_mtu = ice_change_mtu, 6490 .ndo_get_stats64 = ice_get_stats64, 6491 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 6492 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 6493 .ndo_set_vf_mac = ice_set_vf_mac, 6494 .ndo_get_vf_config = ice_get_vf_cfg, 6495 .ndo_set_vf_trust = ice_set_vf_trust, 6496 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 6497 .ndo_set_vf_link_state = ice_set_vf_link_state, 6498 .ndo_get_vf_stats = ice_get_vf_stats, 6499 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 6500 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 6501 .ndo_set_features = ice_set_features, 6502 .ndo_bridge_getlink = ice_bridge_getlink, 6503 .ndo_bridge_setlink = ice_bridge_setlink, 6504 .ndo_fdb_add = ice_fdb_add, 6505 .ndo_fdb_del = ice_fdb_del, 6506 #ifdef CONFIG_RFS_ACCEL 6507 .ndo_rx_flow_steer = ice_rx_flow_steer, 6508 #endif 6509 .ndo_tx_timeout = ice_tx_timeout, 6510 .ndo_bpf = ice_xdp, 6511 .ndo_xdp_xmit = ice_xdp_xmit, 6512 .ndo_xsk_wakeup = ice_xsk_wakeup, 6513 .ndo_udp_tunnel_add = ice_udp_tunnel_add, 6514 .ndo_udp_tunnel_del = ice_udp_tunnel_del, 6515 }; 6516