1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <generated/utsrelease.h> 9 #include "ice.h" 10 #include "ice_base.h" 11 #include "ice_lib.h" 12 #include "ice_fltr.h" 13 #include "ice_dcb_lib.h" 14 #include "ice_dcb_nl.h" 15 #include "ice_devlink.h" 16 /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the 17 * ice tracepoint functions. This must be done exactly once across the 18 * ice driver. 19 */ 20 #define CREATE_TRACE_POINTS 21 #include "ice_trace.h" 22 #include "ice_eswitch.h" 23 #include "ice_tc_lib.h" 24 #include "ice_vsi_vlan_ops.h" 25 #include <net/xdp_sock_drv.h> 26 27 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 28 static const char ice_driver_string[] = DRV_SUMMARY; 29 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 30 31 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 32 #define ICE_DDP_PKG_PATH "intel/ice/ddp/" 33 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 34 35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 36 MODULE_DESCRIPTION(DRV_SUMMARY); 37 MODULE_LICENSE("GPL v2"); 38 MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 39 40 static int debug = -1; 41 module_param(debug, int, 0644); 42 #ifndef CONFIG_DYNAMIC_DEBUG 43 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 44 #else 45 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 46 #endif /* !CONFIG_DYNAMIC_DEBUG */ 47 48 DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key); 49 EXPORT_SYMBOL(ice_xdp_locking_key); 50 51 /** 52 * ice_hw_to_dev - Get device pointer from the hardware structure 53 * @hw: pointer to the device HW structure 54 * 55 * Used to access the device pointer from compilation units which can't easily 56 * include the definition of struct ice_pf without leading to circular header 57 * dependencies. 58 */ 59 struct device *ice_hw_to_dev(struct ice_hw *hw) 60 { 61 struct ice_pf *pf = container_of(hw, struct ice_pf, hw); 62 63 return &pf->pdev->dev; 64 } 65 66 static struct workqueue_struct *ice_wq; 67 static const struct net_device_ops ice_netdev_safe_mode_ops; 68 static const struct net_device_ops ice_netdev_ops; 69 70 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 71 72 static void ice_vsi_release_all(struct ice_pf *pf); 73 74 static int ice_rebuild_channels(struct ice_pf *pf); 75 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr); 76 77 static int 78 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 79 void *cb_priv, enum tc_setup_type type, void *type_data, 80 void *data, 81 void (*cleanup)(struct flow_block_cb *block_cb)); 82 83 bool netif_is_ice(const struct net_device *dev) 84 { 85 return dev && (dev->netdev_ops == &ice_netdev_ops); 86 } 87 88 /** 89 * ice_get_tx_pending - returns number of Tx descriptors not processed 90 * @ring: the ring of descriptors 91 */ 92 static u16 ice_get_tx_pending(struct ice_tx_ring *ring) 93 { 94 u16 head, tail; 95 96 head = ring->next_to_clean; 97 tail = ring->next_to_use; 98 99 if (head != tail) 100 return (head < tail) ? 101 tail - head : (tail + ring->count - head); 102 return 0; 103 } 104 105 /** 106 * ice_check_for_hang_subtask - check for and recover hung queues 107 * @pf: pointer to PF struct 108 */ 109 static void ice_check_for_hang_subtask(struct ice_pf *pf) 110 { 111 struct ice_vsi *vsi = NULL; 112 struct ice_hw *hw; 113 unsigned int i; 114 int packets; 115 u32 v; 116 117 ice_for_each_vsi(pf, v) 118 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 119 vsi = pf->vsi[v]; 120 break; 121 } 122 123 if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state)) 124 return; 125 126 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 127 return; 128 129 hw = &vsi->back->hw; 130 131 ice_for_each_txq(vsi, i) { 132 struct ice_tx_ring *tx_ring = vsi->tx_rings[i]; 133 struct ice_ring_stats *ring_stats; 134 135 if (!tx_ring) 136 continue; 137 if (ice_ring_ch_enabled(tx_ring)) 138 continue; 139 140 ring_stats = tx_ring->ring_stats; 141 if (!ring_stats) 142 continue; 143 144 if (tx_ring->desc) { 145 /* If packet counter has not changed the queue is 146 * likely stalled, so force an interrupt for this 147 * queue. 148 * 149 * prev_pkt would be negative if there was no 150 * pending work. 151 */ 152 packets = ring_stats->stats.pkts & INT_MAX; 153 if (ring_stats->tx_stats.prev_pkt == packets) { 154 /* Trigger sw interrupt to revive the queue */ 155 ice_trigger_sw_intr(hw, tx_ring->q_vector); 156 continue; 157 } 158 159 /* Memory barrier between read of packet count and call 160 * to ice_get_tx_pending() 161 */ 162 smp_rmb(); 163 ring_stats->tx_stats.prev_pkt = 164 ice_get_tx_pending(tx_ring) ? packets : -1; 165 } 166 } 167 } 168 169 /** 170 * ice_init_mac_fltr - Set initial MAC filters 171 * @pf: board private structure 172 * 173 * Set initial set of MAC filters for PF VSI; configure filters for permanent 174 * address and broadcast address. If an error is encountered, netdevice will be 175 * unregistered. 176 */ 177 static int ice_init_mac_fltr(struct ice_pf *pf) 178 { 179 struct ice_vsi *vsi; 180 u8 *perm_addr; 181 182 vsi = ice_get_main_vsi(pf); 183 if (!vsi) 184 return -EINVAL; 185 186 perm_addr = vsi->port_info->mac.perm_addr; 187 return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 188 } 189 190 /** 191 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 192 * @netdev: the net device on which the sync is happening 193 * @addr: MAC address to sync 194 * 195 * This is a callback function which is called by the in kernel device sync 196 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 197 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 198 * MAC filters from the hardware. 199 */ 200 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 201 { 202 struct ice_netdev_priv *np = netdev_priv(netdev); 203 struct ice_vsi *vsi = np->vsi; 204 205 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 206 ICE_FWD_TO_VSI)) 207 return -EINVAL; 208 209 return 0; 210 } 211 212 /** 213 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 214 * @netdev: the net device on which the unsync is happening 215 * @addr: MAC address to unsync 216 * 217 * This is a callback function which is called by the in kernel device unsync 218 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 219 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 220 * delete the MAC filters from the hardware. 221 */ 222 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 223 { 224 struct ice_netdev_priv *np = netdev_priv(netdev); 225 struct ice_vsi *vsi = np->vsi; 226 227 /* Under some circumstances, we might receive a request to delete our 228 * own device address from our uc list. Because we store the device 229 * address in the VSI's MAC filter list, we need to ignore such 230 * requests and not delete our device address from this list. 231 */ 232 if (ether_addr_equal(addr, netdev->dev_addr)) 233 return 0; 234 235 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 236 ICE_FWD_TO_VSI)) 237 return -EINVAL; 238 239 return 0; 240 } 241 242 /** 243 * ice_vsi_fltr_changed - check if filter state changed 244 * @vsi: VSI to be checked 245 * 246 * returns true if filter state has changed, false otherwise. 247 */ 248 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 249 { 250 return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) || 251 test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 252 } 253 254 /** 255 * ice_set_promisc - Enable promiscuous mode for a given PF 256 * @vsi: the VSI being configured 257 * @promisc_m: mask of promiscuous config bits 258 * 259 */ 260 static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m) 261 { 262 int status; 263 264 if (vsi->type != ICE_VSI_PF) 265 return 0; 266 267 if (ice_vsi_has_non_zero_vlans(vsi)) { 268 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 269 status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi, 270 promisc_m); 271 } else { 272 status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 273 promisc_m, 0); 274 } 275 if (status && status != -EEXIST) 276 return status; 277 278 netdev_dbg(vsi->netdev, "set promisc filter bits for VSI %i: 0x%x\n", 279 vsi->vsi_num, promisc_m); 280 return 0; 281 } 282 283 /** 284 * ice_clear_promisc - Disable promiscuous mode for a given PF 285 * @vsi: the VSI being configured 286 * @promisc_m: mask of promiscuous config bits 287 * 288 */ 289 static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m) 290 { 291 int status; 292 293 if (vsi->type != ICE_VSI_PF) 294 return 0; 295 296 if (ice_vsi_has_non_zero_vlans(vsi)) { 297 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 298 status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi, 299 promisc_m); 300 } else { 301 status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 302 promisc_m, 0); 303 } 304 305 netdev_dbg(vsi->netdev, "clear promisc filter bits for VSI %i: 0x%x\n", 306 vsi->vsi_num, promisc_m); 307 return status; 308 } 309 310 /** 311 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 312 * @vsi: ptr to the VSI 313 * 314 * Push any outstanding VSI filter changes through the AdminQ. 315 */ 316 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 317 { 318 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 319 struct device *dev = ice_pf_to_dev(vsi->back); 320 struct net_device *netdev = vsi->netdev; 321 bool promisc_forced_on = false; 322 struct ice_pf *pf = vsi->back; 323 struct ice_hw *hw = &pf->hw; 324 u32 changed_flags = 0; 325 int err; 326 327 if (!vsi->netdev) 328 return -EINVAL; 329 330 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 331 usleep_range(1000, 2000); 332 333 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 334 vsi->current_netdev_flags = vsi->netdev->flags; 335 336 INIT_LIST_HEAD(&vsi->tmp_sync_list); 337 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 338 339 if (ice_vsi_fltr_changed(vsi)) { 340 clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 341 clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 342 343 /* grab the netdev's addr_list_lock */ 344 netif_addr_lock_bh(netdev); 345 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 346 ice_add_mac_to_unsync_list); 347 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 348 ice_add_mac_to_unsync_list); 349 /* our temp lists are populated. release lock */ 350 netif_addr_unlock_bh(netdev); 351 } 352 353 /* Remove MAC addresses in the unsync list */ 354 err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 355 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 356 if (err) { 357 netdev_err(netdev, "Failed to delete MAC filters\n"); 358 /* if we failed because of alloc failures, just bail */ 359 if (err == -ENOMEM) 360 goto out; 361 } 362 363 /* Add MAC addresses in the sync list */ 364 err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 365 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 366 /* If filter is added successfully or already exists, do not go into 367 * 'if' condition and report it as error. Instead continue processing 368 * rest of the function. 369 */ 370 if (err && err != -EEXIST) { 371 netdev_err(netdev, "Failed to add MAC filters\n"); 372 /* If there is no more space for new umac filters, VSI 373 * should go into promiscuous mode. There should be some 374 * space reserved for promiscuous filters. 375 */ 376 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 377 !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC, 378 vsi->state)) { 379 promisc_forced_on = true; 380 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 381 vsi->vsi_num); 382 } else { 383 goto out; 384 } 385 } 386 err = 0; 387 /* check for changes in promiscuous modes */ 388 if (changed_flags & IFF_ALLMULTI) { 389 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 390 err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS); 391 if (err) { 392 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 393 goto out_promisc; 394 } 395 } else { 396 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 397 err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS); 398 if (err) { 399 vsi->current_netdev_flags |= IFF_ALLMULTI; 400 goto out_promisc; 401 } 402 } 403 } 404 405 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 406 test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) { 407 clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 408 if (vsi->current_netdev_flags & IFF_PROMISC) { 409 /* Apply Rx filter rule to get traffic from wire */ 410 if (!ice_is_dflt_vsi_in_use(vsi->port_info)) { 411 err = ice_set_dflt_vsi(vsi); 412 if (err && err != -EEXIST) { 413 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 414 err, vsi->vsi_num); 415 vsi->current_netdev_flags &= 416 ~IFF_PROMISC; 417 goto out_promisc; 418 } 419 err = 0; 420 vlan_ops->dis_rx_filtering(vsi); 421 422 /* promiscuous mode implies allmulticast so 423 * that VSIs that are in promiscuous mode are 424 * subscribed to multicast packets coming to 425 * the port 426 */ 427 err = ice_set_promisc(vsi, 428 ICE_MCAST_PROMISC_BITS); 429 if (err) 430 goto out_promisc; 431 } 432 } else { 433 /* Clear Rx filter to remove traffic from wire */ 434 if (ice_is_vsi_dflt_vsi(vsi)) { 435 err = ice_clear_dflt_vsi(vsi); 436 if (err) { 437 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 438 err, vsi->vsi_num); 439 vsi->current_netdev_flags |= 440 IFF_PROMISC; 441 goto out_promisc; 442 } 443 if (vsi->netdev->features & 444 NETIF_F_HW_VLAN_CTAG_FILTER) 445 vlan_ops->ena_rx_filtering(vsi); 446 } 447 448 /* disable allmulti here, but only if allmulti is not 449 * still enabled for the netdev 450 */ 451 if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) { 452 err = ice_clear_promisc(vsi, 453 ICE_MCAST_PROMISC_BITS); 454 if (err) { 455 netdev_err(netdev, "Error %d clearing multicast promiscuous on VSI %i\n", 456 err, vsi->vsi_num); 457 } 458 } 459 } 460 } 461 goto exit; 462 463 out_promisc: 464 set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 465 goto exit; 466 out: 467 /* if something went wrong then set the changed flag so we try again */ 468 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 469 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 470 exit: 471 clear_bit(ICE_CFG_BUSY, vsi->state); 472 return err; 473 } 474 475 /** 476 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 477 * @pf: board private structure 478 */ 479 static void ice_sync_fltr_subtask(struct ice_pf *pf) 480 { 481 int v; 482 483 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 484 return; 485 486 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 487 488 ice_for_each_vsi(pf, v) 489 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 490 ice_vsi_sync_fltr(pf->vsi[v])) { 491 /* come back and try again later */ 492 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 493 break; 494 } 495 } 496 497 /** 498 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 499 * @pf: the PF 500 * @locked: is the rtnl_lock already held 501 */ 502 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 503 { 504 int node; 505 int v; 506 507 ice_for_each_vsi(pf, v) 508 if (pf->vsi[v]) 509 ice_dis_vsi(pf->vsi[v], locked); 510 511 for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++) 512 pf->pf_agg_node[node].num_vsis = 0; 513 514 for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++) 515 pf->vf_agg_node[node].num_vsis = 0; 516 } 517 518 /** 519 * ice_clear_sw_switch_recipes - clear switch recipes 520 * @pf: board private structure 521 * 522 * Mark switch recipes as not created in sw structures. There are cases where 523 * rules (especially advanced rules) need to be restored, either re-read from 524 * hardware or added again. For example after the reset. 'recp_created' flag 525 * prevents from doing that and need to be cleared upfront. 526 */ 527 static void ice_clear_sw_switch_recipes(struct ice_pf *pf) 528 { 529 struct ice_sw_recipe *recp; 530 u8 i; 531 532 recp = pf->hw.switch_info->recp_list; 533 for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) 534 recp[i].recp_created = false; 535 } 536 537 /** 538 * ice_prepare_for_reset - prep for reset 539 * @pf: board private structure 540 * @reset_type: reset type requested 541 * 542 * Inform or close all dependent features in prep for reset. 543 */ 544 static void 545 ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 546 { 547 struct ice_hw *hw = &pf->hw; 548 struct ice_vsi *vsi; 549 struct ice_vf *vf; 550 unsigned int bkt; 551 552 dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type); 553 554 /* already prepared for reset */ 555 if (test_bit(ICE_PREPARED_FOR_RESET, pf->state)) 556 return; 557 558 ice_unplug_aux_dev(pf); 559 560 /* Notify VFs of impending reset */ 561 if (ice_check_sq_alive(hw, &hw->mailboxq)) 562 ice_vc_notify_reset(pf); 563 564 /* Disable VFs until reset is completed */ 565 mutex_lock(&pf->vfs.table_lock); 566 ice_for_each_vf(pf, bkt, vf) 567 ice_set_vf_state_dis(vf); 568 mutex_unlock(&pf->vfs.table_lock); 569 570 if (ice_is_eswitch_mode_switchdev(pf)) { 571 if (reset_type != ICE_RESET_PFR) 572 ice_clear_sw_switch_recipes(pf); 573 } 574 575 /* release ADQ specific HW and SW resources */ 576 vsi = ice_get_main_vsi(pf); 577 if (!vsi) 578 goto skip; 579 580 /* to be on safe side, reset orig_rss_size so that normal flow 581 * of deciding rss_size can take precedence 582 */ 583 vsi->orig_rss_size = 0; 584 585 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 586 if (reset_type == ICE_RESET_PFR) { 587 vsi->old_ena_tc = vsi->all_enatc; 588 vsi->old_numtc = vsi->all_numtc; 589 } else { 590 ice_remove_q_channels(vsi, true); 591 592 /* for other reset type, do not support channel rebuild 593 * hence reset needed info 594 */ 595 vsi->old_ena_tc = 0; 596 vsi->all_enatc = 0; 597 vsi->old_numtc = 0; 598 vsi->all_numtc = 0; 599 vsi->req_txq = 0; 600 vsi->req_rxq = 0; 601 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 602 memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt)); 603 } 604 } 605 skip: 606 607 /* clear SW filtering DB */ 608 ice_clear_hw_tbls(hw); 609 /* disable the VSIs and their queues that are not already DOWN */ 610 ice_pf_dis_all_vsi(pf, false); 611 612 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 613 ice_ptp_prepare_for_reset(pf); 614 615 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 616 ice_gnss_exit(pf); 617 618 if (hw->port_info) 619 ice_sched_clear_port(hw->port_info); 620 621 ice_shutdown_all_ctrlq(hw); 622 623 set_bit(ICE_PREPARED_FOR_RESET, pf->state); 624 } 625 626 /** 627 * ice_do_reset - Initiate one of many types of resets 628 * @pf: board private structure 629 * @reset_type: reset type requested before this function was called. 630 */ 631 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 632 { 633 struct device *dev = ice_pf_to_dev(pf); 634 struct ice_hw *hw = &pf->hw; 635 636 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 637 638 ice_prepare_for_reset(pf, reset_type); 639 640 /* trigger the reset */ 641 if (ice_reset(hw, reset_type)) { 642 dev_err(dev, "reset %d failed\n", reset_type); 643 set_bit(ICE_RESET_FAILED, pf->state); 644 clear_bit(ICE_RESET_OICR_RECV, pf->state); 645 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 646 clear_bit(ICE_PFR_REQ, pf->state); 647 clear_bit(ICE_CORER_REQ, pf->state); 648 clear_bit(ICE_GLOBR_REQ, pf->state); 649 wake_up(&pf->reset_wait_queue); 650 return; 651 } 652 653 /* PFR is a bit of a special case because it doesn't result in an OICR 654 * interrupt. So for PFR, rebuild after the reset and clear the reset- 655 * associated state bits. 656 */ 657 if (reset_type == ICE_RESET_PFR) { 658 pf->pfr_count++; 659 ice_rebuild(pf, reset_type); 660 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 661 clear_bit(ICE_PFR_REQ, pf->state); 662 wake_up(&pf->reset_wait_queue); 663 ice_reset_all_vfs(pf); 664 } 665 } 666 667 /** 668 * ice_reset_subtask - Set up for resetting the device and driver 669 * @pf: board private structure 670 */ 671 static void ice_reset_subtask(struct ice_pf *pf) 672 { 673 enum ice_reset_req reset_type = ICE_RESET_INVAL; 674 675 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 676 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 677 * of reset is pending and sets bits in pf->state indicating the reset 678 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set 679 * prepare for pending reset if not already (for PF software-initiated 680 * global resets the software should already be prepared for it as 681 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated 682 * by firmware or software on other PFs, that bit is not set so prepare 683 * for the reset now), poll for reset done, rebuild and return. 684 */ 685 if (test_bit(ICE_RESET_OICR_RECV, pf->state)) { 686 /* Perform the largest reset requested */ 687 if (test_and_clear_bit(ICE_CORER_RECV, pf->state)) 688 reset_type = ICE_RESET_CORER; 689 if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state)) 690 reset_type = ICE_RESET_GLOBR; 691 if (test_and_clear_bit(ICE_EMPR_RECV, pf->state)) 692 reset_type = ICE_RESET_EMPR; 693 /* return if no valid reset type requested */ 694 if (reset_type == ICE_RESET_INVAL) 695 return; 696 ice_prepare_for_reset(pf, reset_type); 697 698 /* make sure we are ready to rebuild */ 699 if (ice_check_reset(&pf->hw)) { 700 set_bit(ICE_RESET_FAILED, pf->state); 701 } else { 702 /* done with reset. start rebuild */ 703 pf->hw.reset_ongoing = false; 704 ice_rebuild(pf, reset_type); 705 /* clear bit to resume normal operations, but 706 * ICE_NEEDS_RESTART bit is set in case rebuild failed 707 */ 708 clear_bit(ICE_RESET_OICR_RECV, pf->state); 709 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 710 clear_bit(ICE_PFR_REQ, pf->state); 711 clear_bit(ICE_CORER_REQ, pf->state); 712 clear_bit(ICE_GLOBR_REQ, pf->state); 713 wake_up(&pf->reset_wait_queue); 714 ice_reset_all_vfs(pf); 715 } 716 717 return; 718 } 719 720 /* No pending resets to finish processing. Check for new resets */ 721 if (test_bit(ICE_PFR_REQ, pf->state)) 722 reset_type = ICE_RESET_PFR; 723 if (test_bit(ICE_CORER_REQ, pf->state)) 724 reset_type = ICE_RESET_CORER; 725 if (test_bit(ICE_GLOBR_REQ, pf->state)) 726 reset_type = ICE_RESET_GLOBR; 727 /* If no valid reset type requested just return */ 728 if (reset_type == ICE_RESET_INVAL) 729 return; 730 731 /* reset if not already down or busy */ 732 if (!test_bit(ICE_DOWN, pf->state) && 733 !test_bit(ICE_CFG_BUSY, pf->state)) { 734 ice_do_reset(pf, reset_type); 735 } 736 } 737 738 /** 739 * ice_print_topo_conflict - print topology conflict message 740 * @vsi: the VSI whose topology status is being checked 741 */ 742 static void ice_print_topo_conflict(struct ice_vsi *vsi) 743 { 744 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 745 case ICE_AQ_LINK_TOPO_CONFLICT: 746 case ICE_AQ_LINK_MEDIA_CONFLICT: 747 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 748 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 749 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 750 netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n"); 751 break; 752 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 753 if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags)) 754 netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n"); 755 else 756 netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 757 break; 758 default: 759 break; 760 } 761 } 762 763 /** 764 * ice_print_link_msg - print link up or down message 765 * @vsi: the VSI whose link status is being queried 766 * @isup: boolean for if the link is now up or down 767 */ 768 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 769 { 770 struct ice_aqc_get_phy_caps_data *caps; 771 const char *an_advertised; 772 const char *fec_req; 773 const char *speed; 774 const char *fec; 775 const char *fc; 776 const char *an; 777 int status; 778 779 if (!vsi) 780 return; 781 782 if (vsi->current_isup == isup) 783 return; 784 785 vsi->current_isup = isup; 786 787 if (!isup) { 788 netdev_info(vsi->netdev, "NIC Link is Down\n"); 789 return; 790 } 791 792 switch (vsi->port_info->phy.link_info.link_speed) { 793 case ICE_AQ_LINK_SPEED_100GB: 794 speed = "100 G"; 795 break; 796 case ICE_AQ_LINK_SPEED_50GB: 797 speed = "50 G"; 798 break; 799 case ICE_AQ_LINK_SPEED_40GB: 800 speed = "40 G"; 801 break; 802 case ICE_AQ_LINK_SPEED_25GB: 803 speed = "25 G"; 804 break; 805 case ICE_AQ_LINK_SPEED_20GB: 806 speed = "20 G"; 807 break; 808 case ICE_AQ_LINK_SPEED_10GB: 809 speed = "10 G"; 810 break; 811 case ICE_AQ_LINK_SPEED_5GB: 812 speed = "5 G"; 813 break; 814 case ICE_AQ_LINK_SPEED_2500MB: 815 speed = "2.5 G"; 816 break; 817 case ICE_AQ_LINK_SPEED_1000MB: 818 speed = "1 G"; 819 break; 820 case ICE_AQ_LINK_SPEED_100MB: 821 speed = "100 M"; 822 break; 823 default: 824 speed = "Unknown "; 825 break; 826 } 827 828 switch (vsi->port_info->fc.current_mode) { 829 case ICE_FC_FULL: 830 fc = "Rx/Tx"; 831 break; 832 case ICE_FC_TX_PAUSE: 833 fc = "Tx"; 834 break; 835 case ICE_FC_RX_PAUSE: 836 fc = "Rx"; 837 break; 838 case ICE_FC_NONE: 839 fc = "None"; 840 break; 841 default: 842 fc = "Unknown"; 843 break; 844 } 845 846 /* Get FEC mode based on negotiated link info */ 847 switch (vsi->port_info->phy.link_info.fec_info) { 848 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 849 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 850 fec = "RS-FEC"; 851 break; 852 case ICE_AQ_LINK_25G_KR_FEC_EN: 853 fec = "FC-FEC/BASE-R"; 854 break; 855 default: 856 fec = "NONE"; 857 break; 858 } 859 860 /* check if autoneg completed, might be false due to not supported */ 861 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 862 an = "True"; 863 else 864 an = "False"; 865 866 /* Get FEC mode requested based on PHY caps last SW configuration */ 867 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 868 if (!caps) { 869 fec_req = "Unknown"; 870 an_advertised = "Unknown"; 871 goto done; 872 } 873 874 status = ice_aq_get_phy_caps(vsi->port_info, false, 875 ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL); 876 if (status) 877 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 878 879 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 880 881 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 882 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 883 fec_req = "RS-FEC"; 884 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 885 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 886 fec_req = "FC-FEC/BASE-R"; 887 else 888 fec_req = "NONE"; 889 890 kfree(caps); 891 892 done: 893 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 894 speed, fec_req, fec, an_advertised, an, fc); 895 ice_print_topo_conflict(vsi); 896 } 897 898 /** 899 * ice_vsi_link_event - update the VSI's netdev 900 * @vsi: the VSI on which the link event occurred 901 * @link_up: whether or not the VSI needs to be set up or down 902 */ 903 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 904 { 905 if (!vsi) 906 return; 907 908 if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev) 909 return; 910 911 if (vsi->type == ICE_VSI_PF) { 912 if (link_up == netif_carrier_ok(vsi->netdev)) 913 return; 914 915 if (link_up) { 916 netif_carrier_on(vsi->netdev); 917 netif_tx_wake_all_queues(vsi->netdev); 918 } else { 919 netif_carrier_off(vsi->netdev); 920 netif_tx_stop_all_queues(vsi->netdev); 921 } 922 } 923 } 924 925 /** 926 * ice_set_dflt_mib - send a default config MIB to the FW 927 * @pf: private PF struct 928 * 929 * This function sends a default configuration MIB to the FW. 930 * 931 * If this function errors out at any point, the driver is still able to 932 * function. The main impact is that LFC may not operate as expected. 933 * Therefore an error state in this function should be treated with a DBG 934 * message and continue on with driver rebuild/reenable. 935 */ 936 static void ice_set_dflt_mib(struct ice_pf *pf) 937 { 938 struct device *dev = ice_pf_to_dev(pf); 939 u8 mib_type, *buf, *lldpmib = NULL; 940 u16 len, typelen, offset = 0; 941 struct ice_lldp_org_tlv *tlv; 942 struct ice_hw *hw = &pf->hw; 943 u32 ouisubtype; 944 945 mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB; 946 lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL); 947 if (!lldpmib) { 948 dev_dbg(dev, "%s Failed to allocate MIB memory\n", 949 __func__); 950 return; 951 } 952 953 /* Add ETS CFG TLV */ 954 tlv = (struct ice_lldp_org_tlv *)lldpmib; 955 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 956 ICE_IEEE_ETS_TLV_LEN); 957 tlv->typelen = htons(typelen); 958 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 959 ICE_IEEE_SUBTYPE_ETS_CFG); 960 tlv->ouisubtype = htonl(ouisubtype); 961 962 buf = tlv->tlvinfo; 963 buf[0] = 0; 964 965 /* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0. 966 * Octets 5 - 12 are BW values, set octet 5 to 100% BW. 967 * Octets 13 - 20 are TSA values - leave as zeros 968 */ 969 buf[5] = 0x64; 970 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 971 offset += len + 2; 972 tlv = (struct ice_lldp_org_tlv *) 973 ((char *)tlv + sizeof(tlv->typelen) + len); 974 975 /* Add ETS REC TLV */ 976 buf = tlv->tlvinfo; 977 tlv->typelen = htons(typelen); 978 979 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 980 ICE_IEEE_SUBTYPE_ETS_REC); 981 tlv->ouisubtype = htonl(ouisubtype); 982 983 /* First octet of buf is reserved 984 * Octets 1 - 4 map UP to TC - all UPs map to zero 985 * Octets 5 - 12 are BW values - set TC 0 to 100%. 986 * Octets 13 - 20 are TSA value - leave as zeros 987 */ 988 buf[5] = 0x64; 989 offset += len + 2; 990 tlv = (struct ice_lldp_org_tlv *) 991 ((char *)tlv + sizeof(tlv->typelen) + len); 992 993 /* Add PFC CFG TLV */ 994 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 995 ICE_IEEE_PFC_TLV_LEN); 996 tlv->typelen = htons(typelen); 997 998 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 999 ICE_IEEE_SUBTYPE_PFC_CFG); 1000 tlv->ouisubtype = htonl(ouisubtype); 1001 1002 /* Octet 1 left as all zeros - PFC disabled */ 1003 buf[0] = 0x08; 1004 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 1005 offset += len + 2; 1006 1007 if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL)) 1008 dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__); 1009 1010 kfree(lldpmib); 1011 } 1012 1013 /** 1014 * ice_check_phy_fw_load - check if PHY FW load failed 1015 * @pf: pointer to PF struct 1016 * @link_cfg_err: bitmap from the link info structure 1017 * 1018 * check if external PHY FW load failed and print an error message if it did 1019 */ 1020 static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err) 1021 { 1022 if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { 1023 clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1024 return; 1025 } 1026 1027 if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags)) 1028 return; 1029 1030 if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) { 1031 dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n"); 1032 set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1033 } 1034 } 1035 1036 /** 1037 * ice_check_module_power 1038 * @pf: pointer to PF struct 1039 * @link_cfg_err: bitmap from the link info structure 1040 * 1041 * check module power level returned by a previous call to aq_get_link_info 1042 * and print error messages if module power level is not supported 1043 */ 1044 static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err) 1045 { 1046 /* if module power level is supported, clear the flag */ 1047 if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT | 1048 ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) { 1049 clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1050 return; 1051 } 1052 1053 /* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the 1054 * above block didn't clear this bit, there's nothing to do 1055 */ 1056 if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags)) 1057 return; 1058 1059 if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) { 1060 dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n"); 1061 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1062 } else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) { 1063 dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n"); 1064 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1065 } 1066 } 1067 1068 /** 1069 * ice_check_link_cfg_err - check if link configuration failed 1070 * @pf: pointer to the PF struct 1071 * @link_cfg_err: bitmap from the link info structure 1072 * 1073 * print if any link configuration failure happens due to the value in the 1074 * link_cfg_err parameter in the link info structure 1075 */ 1076 static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err) 1077 { 1078 ice_check_module_power(pf, link_cfg_err); 1079 ice_check_phy_fw_load(pf, link_cfg_err); 1080 } 1081 1082 /** 1083 * ice_link_event - process the link event 1084 * @pf: PF that the link event is associated with 1085 * @pi: port_info for the port that the link event is associated with 1086 * @link_up: true if the physical link is up and false if it is down 1087 * @link_speed: current link speed received from the link event 1088 * 1089 * Returns 0 on success and negative on failure 1090 */ 1091 static int 1092 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 1093 u16 link_speed) 1094 { 1095 struct device *dev = ice_pf_to_dev(pf); 1096 struct ice_phy_info *phy_info; 1097 struct ice_vsi *vsi; 1098 u16 old_link_speed; 1099 bool old_link; 1100 int status; 1101 1102 phy_info = &pi->phy; 1103 phy_info->link_info_old = phy_info->link_info; 1104 1105 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 1106 old_link_speed = phy_info->link_info_old.link_speed; 1107 1108 /* update the link info structures and re-enable link events, 1109 * don't bail on failure due to other book keeping needed 1110 */ 1111 status = ice_update_link_info(pi); 1112 if (status) 1113 dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n", 1114 pi->lport, status, 1115 ice_aq_str(pi->hw->adminq.sq_last_status)); 1116 1117 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 1118 1119 /* Check if the link state is up after updating link info, and treat 1120 * this event as an UP event since the link is actually UP now. 1121 */ 1122 if (phy_info->link_info.link_info & ICE_AQ_LINK_UP) 1123 link_up = true; 1124 1125 vsi = ice_get_main_vsi(pf); 1126 if (!vsi || !vsi->port_info) 1127 return -EINVAL; 1128 1129 /* turn off PHY if media was removed */ 1130 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 1131 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 1132 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 1133 ice_set_link(vsi, false); 1134 } 1135 1136 /* if the old link up/down and speed is the same as the new */ 1137 if (link_up == old_link && link_speed == old_link_speed) 1138 return 0; 1139 1140 ice_ptp_link_change(pf, pf->hw.pf_id, link_up); 1141 1142 if (ice_is_dcb_active(pf)) { 1143 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 1144 ice_dcb_rebuild(pf); 1145 } else { 1146 if (link_up) 1147 ice_set_dflt_mib(pf); 1148 } 1149 ice_vsi_link_event(vsi, link_up); 1150 ice_print_link_msg(vsi, link_up); 1151 1152 ice_vc_notify_link_state(pf); 1153 1154 return 0; 1155 } 1156 1157 /** 1158 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 1159 * @pf: board private structure 1160 */ 1161 static void ice_watchdog_subtask(struct ice_pf *pf) 1162 { 1163 int i; 1164 1165 /* if interface is down do nothing */ 1166 if (test_bit(ICE_DOWN, pf->state) || 1167 test_bit(ICE_CFG_BUSY, pf->state)) 1168 return; 1169 1170 /* make sure we don't do these things too often */ 1171 if (time_before(jiffies, 1172 pf->serv_tmr_prev + pf->serv_tmr_period)) 1173 return; 1174 1175 pf->serv_tmr_prev = jiffies; 1176 1177 /* Update the stats for active netdevs so the network stack 1178 * can look at updated numbers whenever it cares to 1179 */ 1180 ice_update_pf_stats(pf); 1181 ice_for_each_vsi(pf, i) 1182 if (pf->vsi[i] && pf->vsi[i]->netdev) 1183 ice_update_vsi_stats(pf->vsi[i]); 1184 } 1185 1186 /** 1187 * ice_init_link_events - enable/initialize link events 1188 * @pi: pointer to the port_info instance 1189 * 1190 * Returns -EIO on failure, 0 on success 1191 */ 1192 static int ice_init_link_events(struct ice_port_info *pi) 1193 { 1194 u16 mask; 1195 1196 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 1197 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL | 1198 ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL)); 1199 1200 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 1201 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 1202 pi->lport); 1203 return -EIO; 1204 } 1205 1206 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 1207 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 1208 pi->lport); 1209 return -EIO; 1210 } 1211 1212 return 0; 1213 } 1214 1215 /** 1216 * ice_handle_link_event - handle link event via ARQ 1217 * @pf: PF that the link event is associated with 1218 * @event: event structure containing link status info 1219 */ 1220 static int 1221 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 1222 { 1223 struct ice_aqc_get_link_status_data *link_data; 1224 struct ice_port_info *port_info; 1225 int status; 1226 1227 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 1228 port_info = pf->hw.port_info; 1229 if (!port_info) 1230 return -EINVAL; 1231 1232 status = ice_link_event(pf, port_info, 1233 !!(link_data->link_info & ICE_AQ_LINK_UP), 1234 le16_to_cpu(link_data->link_speed)); 1235 if (status) 1236 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 1237 status); 1238 1239 return status; 1240 } 1241 1242 enum ice_aq_task_state { 1243 ICE_AQ_TASK_WAITING = 0, 1244 ICE_AQ_TASK_COMPLETE, 1245 ICE_AQ_TASK_CANCELED, 1246 }; 1247 1248 struct ice_aq_task { 1249 struct hlist_node entry; 1250 1251 u16 opcode; 1252 struct ice_rq_event_info *event; 1253 enum ice_aq_task_state state; 1254 }; 1255 1256 /** 1257 * ice_aq_wait_for_event - Wait for an AdminQ event from firmware 1258 * @pf: pointer to the PF private structure 1259 * @opcode: the opcode to wait for 1260 * @timeout: how long to wait, in jiffies 1261 * @event: storage for the event info 1262 * 1263 * Waits for a specific AdminQ completion event on the ARQ for a given PF. The 1264 * current thread will be put to sleep until the specified event occurs or 1265 * until the given timeout is reached. 1266 * 1267 * To obtain only the descriptor contents, pass an event without an allocated 1268 * msg_buf. If the complete data buffer is desired, allocate the 1269 * event->msg_buf with enough space ahead of time. 1270 * 1271 * Returns: zero on success, or a negative error code on failure. 1272 */ 1273 int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout, 1274 struct ice_rq_event_info *event) 1275 { 1276 struct device *dev = ice_pf_to_dev(pf); 1277 struct ice_aq_task *task; 1278 unsigned long start; 1279 long ret; 1280 int err; 1281 1282 task = kzalloc(sizeof(*task), GFP_KERNEL); 1283 if (!task) 1284 return -ENOMEM; 1285 1286 INIT_HLIST_NODE(&task->entry); 1287 task->opcode = opcode; 1288 task->event = event; 1289 task->state = ICE_AQ_TASK_WAITING; 1290 1291 spin_lock_bh(&pf->aq_wait_lock); 1292 hlist_add_head(&task->entry, &pf->aq_wait_list); 1293 spin_unlock_bh(&pf->aq_wait_lock); 1294 1295 start = jiffies; 1296 1297 ret = wait_event_interruptible_timeout(pf->aq_wait_queue, task->state, 1298 timeout); 1299 switch (task->state) { 1300 case ICE_AQ_TASK_WAITING: 1301 err = ret < 0 ? ret : -ETIMEDOUT; 1302 break; 1303 case ICE_AQ_TASK_CANCELED: 1304 err = ret < 0 ? ret : -ECANCELED; 1305 break; 1306 case ICE_AQ_TASK_COMPLETE: 1307 err = ret < 0 ? ret : 0; 1308 break; 1309 default: 1310 WARN(1, "Unexpected AdminQ wait task state %u", task->state); 1311 err = -EINVAL; 1312 break; 1313 } 1314 1315 dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n", 1316 jiffies_to_msecs(jiffies - start), 1317 jiffies_to_msecs(timeout), 1318 opcode); 1319 1320 spin_lock_bh(&pf->aq_wait_lock); 1321 hlist_del(&task->entry); 1322 spin_unlock_bh(&pf->aq_wait_lock); 1323 kfree(task); 1324 1325 return err; 1326 } 1327 1328 /** 1329 * ice_aq_check_events - Check if any thread is waiting for an AdminQ event 1330 * @pf: pointer to the PF private structure 1331 * @opcode: the opcode of the event 1332 * @event: the event to check 1333 * 1334 * Loops over the current list of pending threads waiting for an AdminQ event. 1335 * For each matching task, copy the contents of the event into the task 1336 * structure and wake up the thread. 1337 * 1338 * If multiple threads wait for the same opcode, they will all be woken up. 1339 * 1340 * Note that event->msg_buf will only be duplicated if the event has a buffer 1341 * with enough space already allocated. Otherwise, only the descriptor and 1342 * message length will be copied. 1343 * 1344 * Returns: true if an event was found, false otherwise 1345 */ 1346 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode, 1347 struct ice_rq_event_info *event) 1348 { 1349 struct ice_aq_task *task; 1350 bool found = false; 1351 1352 spin_lock_bh(&pf->aq_wait_lock); 1353 hlist_for_each_entry(task, &pf->aq_wait_list, entry) { 1354 if (task->state || task->opcode != opcode) 1355 continue; 1356 1357 memcpy(&task->event->desc, &event->desc, sizeof(event->desc)); 1358 task->event->msg_len = event->msg_len; 1359 1360 /* Only copy the data buffer if a destination was set */ 1361 if (task->event->msg_buf && 1362 task->event->buf_len > event->buf_len) { 1363 memcpy(task->event->msg_buf, event->msg_buf, 1364 event->buf_len); 1365 task->event->buf_len = event->buf_len; 1366 } 1367 1368 task->state = ICE_AQ_TASK_COMPLETE; 1369 found = true; 1370 } 1371 spin_unlock_bh(&pf->aq_wait_lock); 1372 1373 if (found) 1374 wake_up(&pf->aq_wait_queue); 1375 } 1376 1377 /** 1378 * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks 1379 * @pf: the PF private structure 1380 * 1381 * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads. 1382 * This will then cause ice_aq_wait_for_event to exit with -ECANCELED. 1383 */ 1384 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf) 1385 { 1386 struct ice_aq_task *task; 1387 1388 spin_lock_bh(&pf->aq_wait_lock); 1389 hlist_for_each_entry(task, &pf->aq_wait_list, entry) 1390 task->state = ICE_AQ_TASK_CANCELED; 1391 spin_unlock_bh(&pf->aq_wait_lock); 1392 1393 wake_up(&pf->aq_wait_queue); 1394 } 1395 1396 #define ICE_MBX_OVERFLOW_WATERMARK 64 1397 1398 /** 1399 * __ice_clean_ctrlq - helper function to clean controlq rings 1400 * @pf: ptr to struct ice_pf 1401 * @q_type: specific Control queue type 1402 */ 1403 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 1404 { 1405 struct device *dev = ice_pf_to_dev(pf); 1406 struct ice_rq_event_info event; 1407 struct ice_hw *hw = &pf->hw; 1408 struct ice_ctl_q_info *cq; 1409 u16 pending, i = 0; 1410 const char *qtype; 1411 u32 oldval, val; 1412 1413 /* Do not clean control queue if/when PF reset fails */ 1414 if (test_bit(ICE_RESET_FAILED, pf->state)) 1415 return 0; 1416 1417 switch (q_type) { 1418 case ICE_CTL_Q_ADMIN: 1419 cq = &hw->adminq; 1420 qtype = "Admin"; 1421 break; 1422 case ICE_CTL_Q_SB: 1423 cq = &hw->sbq; 1424 qtype = "Sideband"; 1425 break; 1426 case ICE_CTL_Q_MAILBOX: 1427 cq = &hw->mailboxq; 1428 qtype = "Mailbox"; 1429 /* we are going to try to detect a malicious VF, so set the 1430 * state to begin detection 1431 */ 1432 hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT; 1433 break; 1434 default: 1435 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 1436 return 0; 1437 } 1438 1439 /* check for error indications - PF_xx_AxQLEN register layout for 1440 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 1441 */ 1442 val = rd32(hw, cq->rq.len); 1443 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1444 PF_FW_ARQLEN_ARQCRIT_M)) { 1445 oldval = val; 1446 if (val & PF_FW_ARQLEN_ARQVFE_M) 1447 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 1448 qtype); 1449 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 1450 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 1451 qtype); 1452 } 1453 if (val & PF_FW_ARQLEN_ARQCRIT_M) 1454 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 1455 qtype); 1456 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1457 PF_FW_ARQLEN_ARQCRIT_M); 1458 if (oldval != val) 1459 wr32(hw, cq->rq.len, val); 1460 } 1461 1462 val = rd32(hw, cq->sq.len); 1463 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1464 PF_FW_ATQLEN_ATQCRIT_M)) { 1465 oldval = val; 1466 if (val & PF_FW_ATQLEN_ATQVFE_M) 1467 dev_dbg(dev, "%s Send Queue VF Error detected\n", 1468 qtype); 1469 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 1470 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 1471 qtype); 1472 } 1473 if (val & PF_FW_ATQLEN_ATQCRIT_M) 1474 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 1475 qtype); 1476 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1477 PF_FW_ATQLEN_ATQCRIT_M); 1478 if (oldval != val) 1479 wr32(hw, cq->sq.len, val); 1480 } 1481 1482 event.buf_len = cq->rq_buf_size; 1483 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 1484 if (!event.msg_buf) 1485 return 0; 1486 1487 do { 1488 struct ice_mbx_data data = {}; 1489 u16 opcode; 1490 int ret; 1491 1492 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1493 if (ret == -EALREADY) 1494 break; 1495 if (ret) { 1496 dev_err(dev, "%s Receive Queue event error %d\n", qtype, 1497 ret); 1498 break; 1499 } 1500 1501 opcode = le16_to_cpu(event.desc.opcode); 1502 1503 /* Notify any thread that might be waiting for this event */ 1504 ice_aq_check_events(pf, opcode, &event); 1505 1506 switch (opcode) { 1507 case ice_aqc_opc_get_link_status: 1508 if (ice_handle_link_event(pf, &event)) 1509 dev_err(dev, "Could not handle link event\n"); 1510 break; 1511 case ice_aqc_opc_event_lan_overflow: 1512 ice_vf_lan_overflow_event(pf, &event); 1513 break; 1514 case ice_mbx_opc_send_msg_to_pf: 1515 data.num_msg_proc = i; 1516 data.num_pending_arq = pending; 1517 data.max_num_msgs_mbx = hw->mailboxq.num_rq_entries; 1518 data.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK; 1519 1520 ice_vc_process_vf_msg(pf, &event, &data); 1521 break; 1522 case ice_aqc_opc_fw_logging: 1523 ice_output_fw_log(hw, &event.desc, event.msg_buf); 1524 break; 1525 case ice_aqc_opc_lldp_set_mib_change: 1526 ice_dcb_process_lldp_set_mib_change(pf, &event); 1527 break; 1528 default: 1529 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1530 qtype, opcode); 1531 break; 1532 } 1533 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1534 1535 kfree(event.msg_buf); 1536 1537 return pending && (i == ICE_DFLT_IRQ_WORK); 1538 } 1539 1540 /** 1541 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1542 * @hw: pointer to hardware info 1543 * @cq: control queue information 1544 * 1545 * returns true if there are pending messages in a queue, false if there aren't 1546 */ 1547 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1548 { 1549 u16 ntu; 1550 1551 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1552 return cq->rq.next_to_clean != ntu; 1553 } 1554 1555 /** 1556 * ice_clean_adminq_subtask - clean the AdminQ rings 1557 * @pf: board private structure 1558 */ 1559 static void ice_clean_adminq_subtask(struct ice_pf *pf) 1560 { 1561 struct ice_hw *hw = &pf->hw; 1562 1563 if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 1564 return; 1565 1566 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1567 return; 1568 1569 clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 1570 1571 /* There might be a situation where new messages arrive to a control 1572 * queue between processing the last message and clearing the 1573 * EVENT_PENDING bit. So before exiting, check queue head again (using 1574 * ice_ctrlq_pending) and process new messages if any. 1575 */ 1576 if (ice_ctrlq_pending(hw, &hw->adminq)) 1577 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1578 1579 ice_flush(hw); 1580 } 1581 1582 /** 1583 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1584 * @pf: board private structure 1585 */ 1586 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1587 { 1588 struct ice_hw *hw = &pf->hw; 1589 1590 if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1591 return; 1592 1593 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1594 return; 1595 1596 clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1597 1598 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1599 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1600 1601 ice_flush(hw); 1602 } 1603 1604 /** 1605 * ice_clean_sbq_subtask - clean the Sideband Queue rings 1606 * @pf: board private structure 1607 */ 1608 static void ice_clean_sbq_subtask(struct ice_pf *pf) 1609 { 1610 struct ice_hw *hw = &pf->hw; 1611 1612 /* Nothing to do here if sideband queue is not supported */ 1613 if (!ice_is_sbq_supported(hw)) { 1614 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1615 return; 1616 } 1617 1618 if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state)) 1619 return; 1620 1621 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB)) 1622 return; 1623 1624 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1625 1626 if (ice_ctrlq_pending(hw, &hw->sbq)) 1627 __ice_clean_ctrlq(pf, ICE_CTL_Q_SB); 1628 1629 ice_flush(hw); 1630 } 1631 1632 /** 1633 * ice_service_task_schedule - schedule the service task to wake up 1634 * @pf: board private structure 1635 * 1636 * If not already scheduled, this puts the task into the work queue. 1637 */ 1638 void ice_service_task_schedule(struct ice_pf *pf) 1639 { 1640 if (!test_bit(ICE_SERVICE_DIS, pf->state) && 1641 !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) && 1642 !test_bit(ICE_NEEDS_RESTART, pf->state)) 1643 queue_work(ice_wq, &pf->serv_task); 1644 } 1645 1646 /** 1647 * ice_service_task_complete - finish up the service task 1648 * @pf: board private structure 1649 */ 1650 static void ice_service_task_complete(struct ice_pf *pf) 1651 { 1652 WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state)); 1653 1654 /* force memory (pf->state) to sync before next service task */ 1655 smp_mb__before_atomic(); 1656 clear_bit(ICE_SERVICE_SCHED, pf->state); 1657 } 1658 1659 /** 1660 * ice_service_task_stop - stop service task and cancel works 1661 * @pf: board private structure 1662 * 1663 * Return 0 if the ICE_SERVICE_DIS bit was not already set, 1664 * 1 otherwise. 1665 */ 1666 static int ice_service_task_stop(struct ice_pf *pf) 1667 { 1668 int ret; 1669 1670 ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state); 1671 1672 if (pf->serv_tmr.function) 1673 del_timer_sync(&pf->serv_tmr); 1674 if (pf->serv_task.func) 1675 cancel_work_sync(&pf->serv_task); 1676 1677 clear_bit(ICE_SERVICE_SCHED, pf->state); 1678 return ret; 1679 } 1680 1681 /** 1682 * ice_service_task_restart - restart service task and schedule works 1683 * @pf: board private structure 1684 * 1685 * This function is needed for suspend and resume works (e.g WoL scenario) 1686 */ 1687 static void ice_service_task_restart(struct ice_pf *pf) 1688 { 1689 clear_bit(ICE_SERVICE_DIS, pf->state); 1690 ice_service_task_schedule(pf); 1691 } 1692 1693 /** 1694 * ice_service_timer - timer callback to schedule service task 1695 * @t: pointer to timer_list 1696 */ 1697 static void ice_service_timer(struct timer_list *t) 1698 { 1699 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1700 1701 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1702 ice_service_task_schedule(pf); 1703 } 1704 1705 /** 1706 * ice_handle_mdd_event - handle malicious driver detect event 1707 * @pf: pointer to the PF structure 1708 * 1709 * Called from service task. OICR interrupt handler indicates MDD event. 1710 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1711 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1712 * disable the queue, the PF can be configured to reset the VF using ethtool 1713 * private flag mdd-auto-reset-vf. 1714 */ 1715 static void ice_handle_mdd_event(struct ice_pf *pf) 1716 { 1717 struct device *dev = ice_pf_to_dev(pf); 1718 struct ice_hw *hw = &pf->hw; 1719 struct ice_vf *vf; 1720 unsigned int bkt; 1721 u32 reg; 1722 1723 if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) { 1724 /* Since the VF MDD event logging is rate limited, check if 1725 * there are pending MDD events. 1726 */ 1727 ice_print_vfs_mdd_events(pf); 1728 return; 1729 } 1730 1731 /* find what triggered an MDD event */ 1732 reg = rd32(hw, GL_MDET_TX_PQM); 1733 if (reg & GL_MDET_TX_PQM_VALID_M) { 1734 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1735 GL_MDET_TX_PQM_PF_NUM_S; 1736 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1737 GL_MDET_TX_PQM_VF_NUM_S; 1738 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1739 GL_MDET_TX_PQM_MAL_TYPE_S; 1740 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1741 GL_MDET_TX_PQM_QNUM_S); 1742 1743 if (netif_msg_tx_err(pf)) 1744 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1745 event, queue, pf_num, vf_num); 1746 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1747 } 1748 1749 reg = rd32(hw, GL_MDET_TX_TCLAN); 1750 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1751 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1752 GL_MDET_TX_TCLAN_PF_NUM_S; 1753 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1754 GL_MDET_TX_TCLAN_VF_NUM_S; 1755 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1756 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1757 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1758 GL_MDET_TX_TCLAN_QNUM_S); 1759 1760 if (netif_msg_tx_err(pf)) 1761 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1762 event, queue, pf_num, vf_num); 1763 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1764 } 1765 1766 reg = rd32(hw, GL_MDET_RX); 1767 if (reg & GL_MDET_RX_VALID_M) { 1768 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1769 GL_MDET_RX_PF_NUM_S; 1770 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1771 GL_MDET_RX_VF_NUM_S; 1772 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1773 GL_MDET_RX_MAL_TYPE_S; 1774 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1775 GL_MDET_RX_QNUM_S); 1776 1777 if (netif_msg_rx_err(pf)) 1778 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1779 event, queue, pf_num, vf_num); 1780 wr32(hw, GL_MDET_RX, 0xffffffff); 1781 } 1782 1783 /* check to see if this PF caused an MDD event */ 1784 reg = rd32(hw, PF_MDET_TX_PQM); 1785 if (reg & PF_MDET_TX_PQM_VALID_M) { 1786 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1787 if (netif_msg_tx_err(pf)) 1788 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1789 } 1790 1791 reg = rd32(hw, PF_MDET_TX_TCLAN); 1792 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1793 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1794 if (netif_msg_tx_err(pf)) 1795 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1796 } 1797 1798 reg = rd32(hw, PF_MDET_RX); 1799 if (reg & PF_MDET_RX_VALID_M) { 1800 wr32(hw, PF_MDET_RX, 0xFFFF); 1801 if (netif_msg_rx_err(pf)) 1802 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1803 } 1804 1805 /* Check to see if one of the VFs caused an MDD event, and then 1806 * increment counters and set print pending 1807 */ 1808 mutex_lock(&pf->vfs.table_lock); 1809 ice_for_each_vf(pf, bkt, vf) { 1810 reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id)); 1811 if (reg & VP_MDET_TX_PQM_VALID_M) { 1812 wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF); 1813 vf->mdd_tx_events.count++; 1814 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1815 if (netif_msg_tx_err(pf)) 1816 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1817 vf->vf_id); 1818 } 1819 1820 reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id)); 1821 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1822 wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF); 1823 vf->mdd_tx_events.count++; 1824 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1825 if (netif_msg_tx_err(pf)) 1826 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1827 vf->vf_id); 1828 } 1829 1830 reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id)); 1831 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1832 wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF); 1833 vf->mdd_tx_events.count++; 1834 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1835 if (netif_msg_tx_err(pf)) 1836 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1837 vf->vf_id); 1838 } 1839 1840 reg = rd32(hw, VP_MDET_RX(vf->vf_id)); 1841 if (reg & VP_MDET_RX_VALID_M) { 1842 wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF); 1843 vf->mdd_rx_events.count++; 1844 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1845 if (netif_msg_rx_err(pf)) 1846 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1847 vf->vf_id); 1848 1849 /* Since the queue is disabled on VF Rx MDD events, the 1850 * PF can be configured to reset the VF through ethtool 1851 * private flag mdd-auto-reset-vf. 1852 */ 1853 if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 1854 /* VF MDD event counters will be cleared by 1855 * reset, so print the event prior to reset. 1856 */ 1857 ice_print_vf_rx_mdd_event(vf); 1858 ice_reset_vf(vf, ICE_VF_RESET_LOCK); 1859 } 1860 } 1861 } 1862 mutex_unlock(&pf->vfs.table_lock); 1863 1864 ice_print_vfs_mdd_events(pf); 1865 } 1866 1867 /** 1868 * ice_force_phys_link_state - Force the physical link state 1869 * @vsi: VSI to force the physical link state to up/down 1870 * @link_up: true/false indicates to set the physical link to up/down 1871 * 1872 * Force the physical link state by getting the current PHY capabilities from 1873 * hardware and setting the PHY config based on the determined capabilities. If 1874 * link changes a link event will be triggered because both the Enable Automatic 1875 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1876 * 1877 * Returns 0 on success, negative on failure 1878 */ 1879 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1880 { 1881 struct ice_aqc_get_phy_caps_data *pcaps; 1882 struct ice_aqc_set_phy_cfg_data *cfg; 1883 struct ice_port_info *pi; 1884 struct device *dev; 1885 int retcode; 1886 1887 if (!vsi || !vsi->port_info || !vsi->back) 1888 return -EINVAL; 1889 if (vsi->type != ICE_VSI_PF) 1890 return 0; 1891 1892 dev = ice_pf_to_dev(vsi->back); 1893 1894 pi = vsi->port_info; 1895 1896 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1897 if (!pcaps) 1898 return -ENOMEM; 1899 1900 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 1901 NULL); 1902 if (retcode) { 1903 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1904 vsi->vsi_num, retcode); 1905 retcode = -EIO; 1906 goto out; 1907 } 1908 1909 /* No change in link */ 1910 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1911 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1912 goto out; 1913 1914 /* Use the current user PHY configuration. The current user PHY 1915 * configuration is initialized during probe from PHY capabilities 1916 * software mode, and updated on set PHY configuration. 1917 */ 1918 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1919 if (!cfg) { 1920 retcode = -ENOMEM; 1921 goto out; 1922 } 1923 1924 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1925 if (link_up) 1926 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1927 else 1928 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1929 1930 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1931 if (retcode) { 1932 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1933 vsi->vsi_num, retcode); 1934 retcode = -EIO; 1935 } 1936 1937 kfree(cfg); 1938 out: 1939 kfree(pcaps); 1940 return retcode; 1941 } 1942 1943 /** 1944 * ice_init_nvm_phy_type - Initialize the NVM PHY type 1945 * @pi: port info structure 1946 * 1947 * Initialize nvm_phy_type_[low|high] for link lenient mode support 1948 */ 1949 static int ice_init_nvm_phy_type(struct ice_port_info *pi) 1950 { 1951 struct ice_aqc_get_phy_caps_data *pcaps; 1952 struct ice_pf *pf = pi->hw->back; 1953 int err; 1954 1955 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1956 if (!pcaps) 1957 return -ENOMEM; 1958 1959 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA, 1960 pcaps, NULL); 1961 1962 if (err) { 1963 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1964 goto out; 1965 } 1966 1967 pf->nvm_phy_type_hi = pcaps->phy_type_high; 1968 pf->nvm_phy_type_lo = pcaps->phy_type_low; 1969 1970 out: 1971 kfree(pcaps); 1972 return err; 1973 } 1974 1975 /** 1976 * ice_init_link_dflt_override - Initialize link default override 1977 * @pi: port info structure 1978 * 1979 * Initialize link default override and PHY total port shutdown during probe 1980 */ 1981 static void ice_init_link_dflt_override(struct ice_port_info *pi) 1982 { 1983 struct ice_link_default_override_tlv *ldo; 1984 struct ice_pf *pf = pi->hw->back; 1985 1986 ldo = &pf->link_dflt_override; 1987 if (ice_get_link_default_override(ldo, pi)) 1988 return; 1989 1990 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 1991 return; 1992 1993 /* Enable Total Port Shutdown (override/replace link-down-on-close 1994 * ethtool private flag) for ports with Port Disable bit set. 1995 */ 1996 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 1997 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 1998 } 1999 2000 /** 2001 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 2002 * @pi: port info structure 2003 * 2004 * If default override is enabled, initialize the user PHY cfg speed and FEC 2005 * settings using the default override mask from the NVM. 2006 * 2007 * The PHY should only be configured with the default override settings the 2008 * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state 2009 * is used to indicate that the user PHY cfg default override is initialized 2010 * and the PHY has not been configured with the default override settings. The 2011 * state is set here, and cleared in ice_configure_phy the first time the PHY is 2012 * configured. 2013 * 2014 * This function should be called only if the FW doesn't support default 2015 * configuration mode, as reported by ice_fw_supports_report_dflt_cfg. 2016 */ 2017 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 2018 { 2019 struct ice_link_default_override_tlv *ldo; 2020 struct ice_aqc_set_phy_cfg_data *cfg; 2021 struct ice_phy_info *phy = &pi->phy; 2022 struct ice_pf *pf = pi->hw->back; 2023 2024 ldo = &pf->link_dflt_override; 2025 2026 /* If link default override is enabled, use to mask NVM PHY capabilities 2027 * for speed and FEC default configuration. 2028 */ 2029 cfg = &phy->curr_user_phy_cfg; 2030 2031 if (ldo->phy_type_low || ldo->phy_type_high) { 2032 cfg->phy_type_low = pf->nvm_phy_type_lo & 2033 cpu_to_le64(ldo->phy_type_low); 2034 cfg->phy_type_high = pf->nvm_phy_type_hi & 2035 cpu_to_le64(ldo->phy_type_high); 2036 } 2037 cfg->link_fec_opt = ldo->fec_options; 2038 phy->curr_user_fec_req = ICE_FEC_AUTO; 2039 2040 set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 2041 } 2042 2043 /** 2044 * ice_init_phy_user_cfg - Initialize the PHY user configuration 2045 * @pi: port info structure 2046 * 2047 * Initialize the current user PHY configuration, speed, FEC, and FC requested 2048 * mode to default. The PHY defaults are from get PHY capabilities topology 2049 * with media so call when media is first available. An error is returned if 2050 * called when media is not available. The PHY initialization completed state is 2051 * set here. 2052 * 2053 * These configurations are used when setting PHY 2054 * configuration. The user PHY configuration is updated on set PHY 2055 * configuration. Returns 0 on success, negative on failure 2056 */ 2057 static int ice_init_phy_user_cfg(struct ice_port_info *pi) 2058 { 2059 struct ice_aqc_get_phy_caps_data *pcaps; 2060 struct ice_phy_info *phy = &pi->phy; 2061 struct ice_pf *pf = pi->hw->back; 2062 int err; 2063 2064 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2065 return -EIO; 2066 2067 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2068 if (!pcaps) 2069 return -ENOMEM; 2070 2071 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2072 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2073 pcaps, NULL); 2074 else 2075 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2076 pcaps, NULL); 2077 if (err) { 2078 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 2079 goto err_out; 2080 } 2081 2082 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 2083 2084 /* check if lenient mode is supported and enabled */ 2085 if (ice_fw_supports_link_override(pi->hw) && 2086 !(pcaps->module_compliance_enforcement & 2087 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 2088 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 2089 2090 /* if the FW supports default PHY configuration mode, then the driver 2091 * does not have to apply link override settings. If not, 2092 * initialize user PHY configuration with link override values 2093 */ 2094 if (!ice_fw_supports_report_dflt_cfg(pi->hw) && 2095 (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) { 2096 ice_init_phy_cfg_dflt_override(pi); 2097 goto out; 2098 } 2099 } 2100 2101 /* if link default override is not enabled, set user flow control and 2102 * FEC settings based on what get_phy_caps returned 2103 */ 2104 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 2105 pcaps->link_fec_options); 2106 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 2107 2108 out: 2109 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 2110 set_bit(ICE_PHY_INIT_COMPLETE, pf->state); 2111 err_out: 2112 kfree(pcaps); 2113 return err; 2114 } 2115 2116 /** 2117 * ice_configure_phy - configure PHY 2118 * @vsi: VSI of PHY 2119 * 2120 * Set the PHY configuration. If the current PHY configuration is the same as 2121 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 2122 * configure the based get PHY capabilities for topology with media. 2123 */ 2124 static int ice_configure_phy(struct ice_vsi *vsi) 2125 { 2126 struct device *dev = ice_pf_to_dev(vsi->back); 2127 struct ice_port_info *pi = vsi->port_info; 2128 struct ice_aqc_get_phy_caps_data *pcaps; 2129 struct ice_aqc_set_phy_cfg_data *cfg; 2130 struct ice_phy_info *phy = &pi->phy; 2131 struct ice_pf *pf = vsi->back; 2132 int err; 2133 2134 /* Ensure we have media as we cannot configure a medialess port */ 2135 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2136 return -EPERM; 2137 2138 ice_print_topo_conflict(vsi); 2139 2140 if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) && 2141 phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 2142 return -EPERM; 2143 2144 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) 2145 return ice_force_phys_link_state(vsi, true); 2146 2147 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2148 if (!pcaps) 2149 return -ENOMEM; 2150 2151 /* Get current PHY config */ 2152 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 2153 NULL); 2154 if (err) { 2155 dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n", 2156 vsi->vsi_num, err); 2157 goto done; 2158 } 2159 2160 /* If PHY enable link is configured and configuration has not changed, 2161 * there's nothing to do 2162 */ 2163 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 2164 ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg)) 2165 goto done; 2166 2167 /* Use PHY topology as baseline for configuration */ 2168 memset(pcaps, 0, sizeof(*pcaps)); 2169 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2170 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2171 pcaps, NULL); 2172 else 2173 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2174 pcaps, NULL); 2175 if (err) { 2176 dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n", 2177 vsi->vsi_num, err); 2178 goto done; 2179 } 2180 2181 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 2182 if (!cfg) { 2183 err = -ENOMEM; 2184 goto done; 2185 } 2186 2187 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 2188 2189 /* Speed - If default override pending, use curr_user_phy_cfg set in 2190 * ice_init_phy_user_cfg_ldo. 2191 */ 2192 if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, 2193 vsi->back->state)) { 2194 cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low; 2195 cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high; 2196 } else { 2197 u64 phy_low = 0, phy_high = 0; 2198 2199 ice_update_phy_type(&phy_low, &phy_high, 2200 pi->phy.curr_user_speed_req); 2201 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 2202 cfg->phy_type_high = pcaps->phy_type_high & 2203 cpu_to_le64(phy_high); 2204 } 2205 2206 /* Can't provide what was requested; use PHY capabilities */ 2207 if (!cfg->phy_type_low && !cfg->phy_type_high) { 2208 cfg->phy_type_low = pcaps->phy_type_low; 2209 cfg->phy_type_high = pcaps->phy_type_high; 2210 } 2211 2212 /* FEC */ 2213 ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req); 2214 2215 /* Can't provide what was requested; use PHY capabilities */ 2216 if (cfg->link_fec_opt != 2217 (cfg->link_fec_opt & pcaps->link_fec_options)) { 2218 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 2219 cfg->link_fec_opt = pcaps->link_fec_options; 2220 } 2221 2222 /* Flow Control - always supported; no need to check against 2223 * capabilities 2224 */ 2225 ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req); 2226 2227 /* Enable link and link update */ 2228 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 2229 2230 err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL); 2231 if (err) 2232 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 2233 vsi->vsi_num, err); 2234 2235 kfree(cfg); 2236 done: 2237 kfree(pcaps); 2238 return err; 2239 } 2240 2241 /** 2242 * ice_check_media_subtask - Check for media 2243 * @pf: pointer to PF struct 2244 * 2245 * If media is available, then initialize PHY user configuration if it is not 2246 * been, and configure the PHY if the interface is up. 2247 */ 2248 static void ice_check_media_subtask(struct ice_pf *pf) 2249 { 2250 struct ice_port_info *pi; 2251 struct ice_vsi *vsi; 2252 int err; 2253 2254 /* No need to check for media if it's already present */ 2255 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 2256 return; 2257 2258 vsi = ice_get_main_vsi(pf); 2259 if (!vsi) 2260 return; 2261 2262 /* Refresh link info and check if media is present */ 2263 pi = vsi->port_info; 2264 err = ice_update_link_info(pi); 2265 if (err) 2266 return; 2267 2268 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 2269 2270 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2271 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) 2272 ice_init_phy_user_cfg(pi); 2273 2274 /* PHY settings are reset on media insertion, reconfigure 2275 * PHY to preserve settings. 2276 */ 2277 if (test_bit(ICE_VSI_DOWN, vsi->state) && 2278 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 2279 return; 2280 2281 err = ice_configure_phy(vsi); 2282 if (!err) 2283 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 2284 2285 /* A Link Status Event will be generated; the event handler 2286 * will complete bringing the interface up 2287 */ 2288 } 2289 } 2290 2291 /** 2292 * ice_service_task - manage and run subtasks 2293 * @work: pointer to work_struct contained by the PF struct 2294 */ 2295 static void ice_service_task(struct work_struct *work) 2296 { 2297 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 2298 unsigned long start_time = jiffies; 2299 2300 /* subtasks */ 2301 2302 /* process reset requests first */ 2303 ice_reset_subtask(pf); 2304 2305 /* bail if a reset/recovery cycle is pending or rebuild failed */ 2306 if (ice_is_reset_in_progress(pf->state) || 2307 test_bit(ICE_SUSPENDED, pf->state) || 2308 test_bit(ICE_NEEDS_RESTART, pf->state)) { 2309 ice_service_task_complete(pf); 2310 return; 2311 } 2312 2313 if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) { 2314 struct iidc_event *event; 2315 2316 event = kzalloc(sizeof(*event), GFP_KERNEL); 2317 if (event) { 2318 set_bit(IIDC_EVENT_CRIT_ERR, event->type); 2319 /* report the entire OICR value to AUX driver */ 2320 swap(event->reg, pf->oicr_err_reg); 2321 ice_send_event_to_aux(pf, event); 2322 kfree(event); 2323 } 2324 } 2325 2326 /* unplug aux dev per request, if an unplug request came in 2327 * while processing a plug request, this will handle it 2328 */ 2329 if (test_and_clear_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags)) 2330 ice_unplug_aux_dev(pf); 2331 2332 /* Plug aux device per request */ 2333 if (test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags)) 2334 ice_plug_aux_dev(pf); 2335 2336 if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) { 2337 struct iidc_event *event; 2338 2339 event = kzalloc(sizeof(*event), GFP_KERNEL); 2340 if (event) { 2341 set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type); 2342 ice_send_event_to_aux(pf, event); 2343 kfree(event); 2344 } 2345 } 2346 2347 ice_clean_adminq_subtask(pf); 2348 ice_check_media_subtask(pf); 2349 ice_check_for_hang_subtask(pf); 2350 ice_sync_fltr_subtask(pf); 2351 ice_handle_mdd_event(pf); 2352 ice_watchdog_subtask(pf); 2353 2354 if (ice_is_safe_mode(pf)) { 2355 ice_service_task_complete(pf); 2356 return; 2357 } 2358 2359 ice_process_vflr_event(pf); 2360 ice_clean_mailboxq_subtask(pf); 2361 ice_clean_sbq_subtask(pf); 2362 ice_sync_arfs_fltrs(pf); 2363 ice_flush_fdir_ctx(pf); 2364 2365 /* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */ 2366 ice_service_task_complete(pf); 2367 2368 /* If the tasks have taken longer than one service timer period 2369 * or there is more work to be done, reset the service timer to 2370 * schedule the service task now. 2371 */ 2372 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 2373 test_bit(ICE_MDD_EVENT_PENDING, pf->state) || 2374 test_bit(ICE_VFLR_EVENT_PENDING, pf->state) || 2375 test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 2376 test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) || 2377 test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) || 2378 test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 2379 mod_timer(&pf->serv_tmr, jiffies); 2380 } 2381 2382 /** 2383 * ice_set_ctrlq_len - helper function to set controlq length 2384 * @hw: pointer to the HW instance 2385 */ 2386 static void ice_set_ctrlq_len(struct ice_hw *hw) 2387 { 2388 hw->adminq.num_rq_entries = ICE_AQ_LEN; 2389 hw->adminq.num_sq_entries = ICE_AQ_LEN; 2390 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 2391 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 2392 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 2393 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 2394 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2395 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2396 hw->sbq.num_rq_entries = ICE_SBQ_LEN; 2397 hw->sbq.num_sq_entries = ICE_SBQ_LEN; 2398 hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2399 hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2400 } 2401 2402 /** 2403 * ice_schedule_reset - schedule a reset 2404 * @pf: board private structure 2405 * @reset: reset being requested 2406 */ 2407 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 2408 { 2409 struct device *dev = ice_pf_to_dev(pf); 2410 2411 /* bail out if earlier reset has failed */ 2412 if (test_bit(ICE_RESET_FAILED, pf->state)) { 2413 dev_dbg(dev, "earlier reset has failed\n"); 2414 return -EIO; 2415 } 2416 /* bail if reset/recovery already in progress */ 2417 if (ice_is_reset_in_progress(pf->state)) { 2418 dev_dbg(dev, "Reset already in progress\n"); 2419 return -EBUSY; 2420 } 2421 2422 switch (reset) { 2423 case ICE_RESET_PFR: 2424 set_bit(ICE_PFR_REQ, pf->state); 2425 break; 2426 case ICE_RESET_CORER: 2427 set_bit(ICE_CORER_REQ, pf->state); 2428 break; 2429 case ICE_RESET_GLOBR: 2430 set_bit(ICE_GLOBR_REQ, pf->state); 2431 break; 2432 default: 2433 return -EINVAL; 2434 } 2435 2436 ice_service_task_schedule(pf); 2437 return 0; 2438 } 2439 2440 /** 2441 * ice_irq_affinity_notify - Callback for affinity changes 2442 * @notify: context as to what irq was changed 2443 * @mask: the new affinity mask 2444 * 2445 * This is a callback function used by the irq_set_affinity_notifier function 2446 * so that we may register to receive changes to the irq affinity masks. 2447 */ 2448 static void 2449 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 2450 const cpumask_t *mask) 2451 { 2452 struct ice_q_vector *q_vector = 2453 container_of(notify, struct ice_q_vector, affinity_notify); 2454 2455 cpumask_copy(&q_vector->affinity_mask, mask); 2456 } 2457 2458 /** 2459 * ice_irq_affinity_release - Callback for affinity notifier release 2460 * @ref: internal core kernel usage 2461 * 2462 * This is a callback function used by the irq_set_affinity_notifier function 2463 * to inform the current notification subscriber that they will no longer 2464 * receive notifications. 2465 */ 2466 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 2467 2468 /** 2469 * ice_vsi_ena_irq - Enable IRQ for the given VSI 2470 * @vsi: the VSI being configured 2471 */ 2472 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 2473 { 2474 struct ice_hw *hw = &vsi->back->hw; 2475 int i; 2476 2477 ice_for_each_q_vector(vsi, i) 2478 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 2479 2480 ice_flush(hw); 2481 return 0; 2482 } 2483 2484 /** 2485 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 2486 * @vsi: the VSI being configured 2487 * @basename: name for the vector 2488 */ 2489 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 2490 { 2491 int q_vectors = vsi->num_q_vectors; 2492 struct ice_pf *pf = vsi->back; 2493 struct device *dev; 2494 int rx_int_idx = 0; 2495 int tx_int_idx = 0; 2496 int vector, err; 2497 int irq_num; 2498 2499 dev = ice_pf_to_dev(pf); 2500 for (vector = 0; vector < q_vectors; vector++) { 2501 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 2502 2503 irq_num = q_vector->irq.virq; 2504 2505 if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) { 2506 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2507 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 2508 tx_int_idx++; 2509 } else if (q_vector->rx.rx_ring) { 2510 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2511 "%s-%s-%d", basename, "rx", rx_int_idx++); 2512 } else if (q_vector->tx.tx_ring) { 2513 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2514 "%s-%s-%d", basename, "tx", tx_int_idx++); 2515 } else { 2516 /* skip this unused q_vector */ 2517 continue; 2518 } 2519 if (vsi->type == ICE_VSI_CTRL && vsi->vf) 2520 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2521 IRQF_SHARED, q_vector->name, 2522 q_vector); 2523 else 2524 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2525 0, q_vector->name, q_vector); 2526 if (err) { 2527 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 2528 err); 2529 goto free_q_irqs; 2530 } 2531 2532 /* register for affinity change notifications */ 2533 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { 2534 struct irq_affinity_notify *affinity_notify; 2535 2536 affinity_notify = &q_vector->affinity_notify; 2537 affinity_notify->notify = ice_irq_affinity_notify; 2538 affinity_notify->release = ice_irq_affinity_release; 2539 irq_set_affinity_notifier(irq_num, affinity_notify); 2540 } 2541 2542 /* assign the mask for this irq */ 2543 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 2544 } 2545 2546 err = ice_set_cpu_rx_rmap(vsi); 2547 if (err) { 2548 netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n", 2549 vsi->vsi_num, ERR_PTR(err)); 2550 goto free_q_irqs; 2551 } 2552 2553 vsi->irqs_ready = true; 2554 return 0; 2555 2556 free_q_irqs: 2557 while (vector--) { 2558 irq_num = vsi->q_vectors[vector]->irq.virq; 2559 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) 2560 irq_set_affinity_notifier(irq_num, NULL); 2561 irq_set_affinity_hint(irq_num, NULL); 2562 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 2563 } 2564 return err; 2565 } 2566 2567 /** 2568 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 2569 * @vsi: VSI to setup Tx rings used by XDP 2570 * 2571 * Return 0 on success and negative value on error 2572 */ 2573 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 2574 { 2575 struct device *dev = ice_pf_to_dev(vsi->back); 2576 struct ice_tx_desc *tx_desc; 2577 int i, j; 2578 2579 ice_for_each_xdp_txq(vsi, i) { 2580 u16 xdp_q_idx = vsi->alloc_txq + i; 2581 struct ice_ring_stats *ring_stats; 2582 struct ice_tx_ring *xdp_ring; 2583 2584 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2585 if (!xdp_ring) 2586 goto free_xdp_rings; 2587 2588 ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL); 2589 if (!ring_stats) { 2590 ice_free_tx_ring(xdp_ring); 2591 goto free_xdp_rings; 2592 } 2593 2594 xdp_ring->ring_stats = ring_stats; 2595 xdp_ring->q_index = xdp_q_idx; 2596 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2597 xdp_ring->vsi = vsi; 2598 xdp_ring->netdev = NULL; 2599 xdp_ring->dev = dev; 2600 xdp_ring->count = vsi->num_tx_desc; 2601 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2602 if (ice_setup_tx_ring(xdp_ring)) 2603 goto free_xdp_rings; 2604 ice_set_ring_xdp(xdp_ring); 2605 spin_lock_init(&xdp_ring->tx_lock); 2606 for (j = 0; j < xdp_ring->count; j++) { 2607 tx_desc = ICE_TX_DESC(xdp_ring, j); 2608 tx_desc->cmd_type_offset_bsz = 0; 2609 } 2610 } 2611 2612 return 0; 2613 2614 free_xdp_rings: 2615 for (; i >= 0; i--) { 2616 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) { 2617 kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu); 2618 vsi->xdp_rings[i]->ring_stats = NULL; 2619 ice_free_tx_ring(vsi->xdp_rings[i]); 2620 } 2621 } 2622 return -ENOMEM; 2623 } 2624 2625 /** 2626 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2627 * @vsi: VSI to set the bpf prog on 2628 * @prog: the bpf prog pointer 2629 */ 2630 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2631 { 2632 struct bpf_prog *old_prog; 2633 int i; 2634 2635 old_prog = xchg(&vsi->xdp_prog, prog); 2636 ice_for_each_rxq(vsi, i) 2637 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2638 2639 if (old_prog) 2640 bpf_prog_put(old_prog); 2641 } 2642 2643 /** 2644 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2645 * @vsi: VSI to bring up Tx rings used by XDP 2646 * @prog: bpf program that will be assigned to VSI 2647 * 2648 * Return 0 on success and negative value on error 2649 */ 2650 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) 2651 { 2652 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2653 int xdp_rings_rem = vsi->num_xdp_txq; 2654 struct ice_pf *pf = vsi->back; 2655 struct ice_qs_cfg xdp_qs_cfg = { 2656 .qs_mutex = &pf->avail_q_mutex, 2657 .pf_map = pf->avail_txqs, 2658 .pf_map_size = pf->max_pf_txqs, 2659 .q_count = vsi->num_xdp_txq, 2660 .scatter_count = ICE_MAX_SCATTER_TXQS, 2661 .vsi_map = vsi->txq_map, 2662 .vsi_map_offset = vsi->alloc_txq, 2663 .mapping_mode = ICE_VSI_MAP_CONTIG 2664 }; 2665 struct device *dev; 2666 int i, v_idx; 2667 int status; 2668 2669 dev = ice_pf_to_dev(pf); 2670 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2671 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2672 if (!vsi->xdp_rings) 2673 return -ENOMEM; 2674 2675 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2676 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2677 goto err_map_xdp; 2678 2679 if (static_key_enabled(&ice_xdp_locking_key)) 2680 netdev_warn(vsi->netdev, 2681 "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n"); 2682 2683 if (ice_xdp_alloc_setup_rings(vsi)) 2684 goto clear_xdp_rings; 2685 2686 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2687 ice_for_each_q_vector(vsi, v_idx) { 2688 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2689 int xdp_rings_per_v, q_id, q_base; 2690 2691 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2692 vsi->num_q_vectors - v_idx); 2693 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2694 2695 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2696 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id]; 2697 2698 xdp_ring->q_vector = q_vector; 2699 xdp_ring->next = q_vector->tx.tx_ring; 2700 q_vector->tx.tx_ring = xdp_ring; 2701 } 2702 xdp_rings_rem -= xdp_rings_per_v; 2703 } 2704 2705 ice_for_each_rxq(vsi, i) { 2706 if (static_key_enabled(&ice_xdp_locking_key)) { 2707 vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq]; 2708 } else { 2709 struct ice_q_vector *q_vector = vsi->rx_rings[i]->q_vector; 2710 struct ice_tx_ring *ring; 2711 2712 ice_for_each_tx_ring(ring, q_vector->tx) { 2713 if (ice_ring_is_xdp(ring)) { 2714 vsi->rx_rings[i]->xdp_ring = ring; 2715 break; 2716 } 2717 } 2718 } 2719 ice_tx_xsk_pool(vsi, i); 2720 } 2721 2722 /* omit the scheduler update if in reset path; XDP queues will be 2723 * taken into account at the end of ice_vsi_rebuild, where 2724 * ice_cfg_vsi_lan is being called 2725 */ 2726 if (ice_is_reset_in_progress(pf->state)) 2727 return 0; 2728 2729 /* tell the Tx scheduler that right now we have 2730 * additional queues 2731 */ 2732 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2733 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2734 2735 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2736 max_txqs); 2737 if (status) { 2738 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n", 2739 status); 2740 goto clear_xdp_rings; 2741 } 2742 2743 /* assign the prog only when it's not already present on VSI; 2744 * this flow is a subject of both ethtool -L and ndo_bpf flows; 2745 * VSI rebuild that happens under ethtool -L can expose us to 2746 * the bpf_prog refcount issues as we would be swapping same 2747 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put 2748 * on it as it would be treated as an 'old_prog'; for ndo_bpf 2749 * this is not harmful as dev_xdp_install bumps the refcount 2750 * before calling the op exposed by the driver; 2751 */ 2752 if (!ice_is_xdp_ena_vsi(vsi)) 2753 ice_vsi_assign_bpf_prog(vsi, prog); 2754 2755 return 0; 2756 clear_xdp_rings: 2757 ice_for_each_xdp_txq(vsi, i) 2758 if (vsi->xdp_rings[i]) { 2759 kfree_rcu(vsi->xdp_rings[i], rcu); 2760 vsi->xdp_rings[i] = NULL; 2761 } 2762 2763 err_map_xdp: 2764 mutex_lock(&pf->avail_q_mutex); 2765 ice_for_each_xdp_txq(vsi, i) { 2766 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2767 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2768 } 2769 mutex_unlock(&pf->avail_q_mutex); 2770 2771 devm_kfree(dev, vsi->xdp_rings); 2772 return -ENOMEM; 2773 } 2774 2775 /** 2776 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2777 * @vsi: VSI to remove XDP rings 2778 * 2779 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2780 * resources 2781 */ 2782 int ice_destroy_xdp_rings(struct ice_vsi *vsi) 2783 { 2784 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2785 struct ice_pf *pf = vsi->back; 2786 int i, v_idx; 2787 2788 /* q_vectors are freed in reset path so there's no point in detaching 2789 * rings; in case of rebuild being triggered not from reset bits 2790 * in pf->state won't be set, so additionally check first q_vector 2791 * against NULL 2792 */ 2793 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2794 goto free_qmap; 2795 2796 ice_for_each_q_vector(vsi, v_idx) { 2797 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2798 struct ice_tx_ring *ring; 2799 2800 ice_for_each_tx_ring(ring, q_vector->tx) 2801 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2802 break; 2803 2804 /* restore the value of last node prior to XDP setup */ 2805 q_vector->tx.tx_ring = ring; 2806 } 2807 2808 free_qmap: 2809 mutex_lock(&pf->avail_q_mutex); 2810 ice_for_each_xdp_txq(vsi, i) { 2811 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2812 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2813 } 2814 mutex_unlock(&pf->avail_q_mutex); 2815 2816 ice_for_each_xdp_txq(vsi, i) 2817 if (vsi->xdp_rings[i]) { 2818 if (vsi->xdp_rings[i]->desc) { 2819 synchronize_rcu(); 2820 ice_free_tx_ring(vsi->xdp_rings[i]); 2821 } 2822 kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu); 2823 vsi->xdp_rings[i]->ring_stats = NULL; 2824 kfree_rcu(vsi->xdp_rings[i], rcu); 2825 vsi->xdp_rings[i] = NULL; 2826 } 2827 2828 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2829 vsi->xdp_rings = NULL; 2830 2831 if (static_key_enabled(&ice_xdp_locking_key)) 2832 static_branch_dec(&ice_xdp_locking_key); 2833 2834 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2835 return 0; 2836 2837 ice_vsi_assign_bpf_prog(vsi, NULL); 2838 2839 /* notify Tx scheduler that we destroyed XDP queues and bring 2840 * back the old number of child nodes 2841 */ 2842 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2843 max_txqs[i] = vsi->num_txq; 2844 2845 /* change number of XDP Tx queues to 0 */ 2846 vsi->num_xdp_txq = 0; 2847 2848 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2849 max_txqs); 2850 } 2851 2852 /** 2853 * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI 2854 * @vsi: VSI to schedule napi on 2855 */ 2856 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi) 2857 { 2858 int i; 2859 2860 ice_for_each_rxq(vsi, i) { 2861 struct ice_rx_ring *rx_ring = vsi->rx_rings[i]; 2862 2863 if (rx_ring->xsk_pool) 2864 napi_schedule(&rx_ring->q_vector->napi); 2865 } 2866 } 2867 2868 /** 2869 * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have 2870 * @vsi: VSI to determine the count of XDP Tx qs 2871 * 2872 * returns 0 if Tx qs count is higher than at least half of CPU count, 2873 * -ENOMEM otherwise 2874 */ 2875 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi) 2876 { 2877 u16 avail = ice_get_avail_txq_count(vsi->back); 2878 u16 cpus = num_possible_cpus(); 2879 2880 if (avail < cpus / 2) 2881 return -ENOMEM; 2882 2883 vsi->num_xdp_txq = min_t(u16, avail, cpus); 2884 2885 if (vsi->num_xdp_txq < cpus) 2886 static_branch_inc(&ice_xdp_locking_key); 2887 2888 return 0; 2889 } 2890 2891 /** 2892 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2893 * @vsi: Pointer to VSI structure 2894 */ 2895 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 2896 { 2897 if (test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 2898 return ICE_RXBUF_1664; 2899 else 2900 return ICE_RXBUF_3072; 2901 } 2902 2903 /** 2904 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2905 * @vsi: VSI to setup XDP for 2906 * @prog: XDP program 2907 * @extack: netlink extended ack 2908 */ 2909 static int 2910 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2911 struct netlink_ext_ack *extack) 2912 { 2913 unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2914 bool if_running = netif_running(vsi->netdev); 2915 int ret = 0, xdp_ring_err = 0; 2916 2917 if (prog && !prog->aux->xdp_has_frags) { 2918 if (frame_size > ice_max_xdp_frame_size(vsi)) { 2919 NL_SET_ERR_MSG_MOD(extack, 2920 "MTU is too large for linear frames and XDP prog does not support frags"); 2921 return -EOPNOTSUPP; 2922 } 2923 } 2924 2925 /* hot swap progs and avoid toggling link */ 2926 if (ice_is_xdp_ena_vsi(vsi) == !!prog) { 2927 ice_vsi_assign_bpf_prog(vsi, prog); 2928 return 0; 2929 } 2930 2931 /* need to stop netdev while setting up the program for Rx rings */ 2932 if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 2933 ret = ice_down(vsi); 2934 if (ret) { 2935 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 2936 return ret; 2937 } 2938 } 2939 2940 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 2941 xdp_ring_err = ice_vsi_determine_xdp_res(vsi); 2942 if (xdp_ring_err) { 2943 NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP"); 2944 } else { 2945 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); 2946 if (xdp_ring_err) 2947 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 2948 } 2949 xdp_features_set_redirect_target(vsi->netdev, true); 2950 /* reallocate Rx queues that are used for zero-copy */ 2951 xdp_ring_err = ice_realloc_zc_buf(vsi, true); 2952 if (xdp_ring_err) 2953 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed"); 2954 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 2955 xdp_features_clear_redirect_target(vsi->netdev); 2956 xdp_ring_err = ice_destroy_xdp_rings(vsi); 2957 if (xdp_ring_err) 2958 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 2959 /* reallocate Rx queues that were used for zero-copy */ 2960 xdp_ring_err = ice_realloc_zc_buf(vsi, false); 2961 if (xdp_ring_err) 2962 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed"); 2963 } 2964 2965 if (if_running) 2966 ret = ice_up(vsi); 2967 2968 if (!ret && prog) 2969 ice_vsi_rx_napi_schedule(vsi); 2970 2971 return (ret || xdp_ring_err) ? -ENOMEM : 0; 2972 } 2973 2974 /** 2975 * ice_xdp_safe_mode - XDP handler for safe mode 2976 * @dev: netdevice 2977 * @xdp: XDP command 2978 */ 2979 static int ice_xdp_safe_mode(struct net_device __always_unused *dev, 2980 struct netdev_bpf *xdp) 2981 { 2982 NL_SET_ERR_MSG_MOD(xdp->extack, 2983 "Please provide working DDP firmware package in order to use XDP\n" 2984 "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst"); 2985 return -EOPNOTSUPP; 2986 } 2987 2988 /** 2989 * ice_xdp - implements XDP handler 2990 * @dev: netdevice 2991 * @xdp: XDP command 2992 */ 2993 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2994 { 2995 struct ice_netdev_priv *np = netdev_priv(dev); 2996 struct ice_vsi *vsi = np->vsi; 2997 2998 if (vsi->type != ICE_VSI_PF) { 2999 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); 3000 return -EINVAL; 3001 } 3002 3003 switch (xdp->command) { 3004 case XDP_SETUP_PROG: 3005 return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 3006 case XDP_SETUP_XSK_POOL: 3007 return ice_xsk_pool_setup(vsi, xdp->xsk.pool, 3008 xdp->xsk.queue_id); 3009 default: 3010 return -EINVAL; 3011 } 3012 } 3013 3014 /** 3015 * ice_ena_misc_vector - enable the non-queue interrupts 3016 * @pf: board private structure 3017 */ 3018 static void ice_ena_misc_vector(struct ice_pf *pf) 3019 { 3020 struct ice_hw *hw = &pf->hw; 3021 u32 val; 3022 3023 /* Disable anti-spoof detection interrupt to prevent spurious event 3024 * interrupts during a function reset. Anti-spoof functionally is 3025 * still supported. 3026 */ 3027 val = rd32(hw, GL_MDCK_TX_TDPU); 3028 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 3029 wr32(hw, GL_MDCK_TX_TDPU, val); 3030 3031 /* clear things first */ 3032 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 3033 rd32(hw, PFINT_OICR); /* read to clear */ 3034 3035 val = (PFINT_OICR_ECC_ERR_M | 3036 PFINT_OICR_MAL_DETECT_M | 3037 PFINT_OICR_GRST_M | 3038 PFINT_OICR_PCI_EXCEPTION_M | 3039 PFINT_OICR_VFLR_M | 3040 PFINT_OICR_HMC_ERR_M | 3041 PFINT_OICR_PE_PUSH_M | 3042 PFINT_OICR_PE_CRITERR_M); 3043 3044 wr32(hw, PFINT_OICR_ENA, val); 3045 3046 /* SW_ITR_IDX = 0, but don't change INTENA */ 3047 wr32(hw, GLINT_DYN_CTL(pf->oicr_irq.index), 3048 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 3049 } 3050 3051 /** 3052 * ice_misc_intr - misc interrupt handler 3053 * @irq: interrupt number 3054 * @data: pointer to a q_vector 3055 */ 3056 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 3057 { 3058 struct ice_pf *pf = (struct ice_pf *)data; 3059 struct ice_hw *hw = &pf->hw; 3060 struct device *dev; 3061 u32 oicr, ena_mask; 3062 3063 dev = ice_pf_to_dev(pf); 3064 set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 3065 set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 3066 set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 3067 3068 oicr = rd32(hw, PFINT_OICR); 3069 ena_mask = rd32(hw, PFINT_OICR_ENA); 3070 3071 if (oicr & PFINT_OICR_SWINT_M) { 3072 ena_mask &= ~PFINT_OICR_SWINT_M; 3073 pf->sw_int_count++; 3074 } 3075 3076 if (oicr & PFINT_OICR_MAL_DETECT_M) { 3077 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 3078 set_bit(ICE_MDD_EVENT_PENDING, pf->state); 3079 } 3080 if (oicr & PFINT_OICR_VFLR_M) { 3081 /* disable any further VFLR event notifications */ 3082 if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) { 3083 u32 reg = rd32(hw, PFINT_OICR_ENA); 3084 3085 reg &= ~PFINT_OICR_VFLR_M; 3086 wr32(hw, PFINT_OICR_ENA, reg); 3087 } else { 3088 ena_mask &= ~PFINT_OICR_VFLR_M; 3089 set_bit(ICE_VFLR_EVENT_PENDING, pf->state); 3090 } 3091 } 3092 3093 if (oicr & PFINT_OICR_GRST_M) { 3094 u32 reset; 3095 3096 /* we have a reset warning */ 3097 ena_mask &= ~PFINT_OICR_GRST_M; 3098 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 3099 GLGEN_RSTAT_RESET_TYPE_S; 3100 3101 if (reset == ICE_RESET_CORER) 3102 pf->corer_count++; 3103 else if (reset == ICE_RESET_GLOBR) 3104 pf->globr_count++; 3105 else if (reset == ICE_RESET_EMPR) 3106 pf->empr_count++; 3107 else 3108 dev_dbg(dev, "Invalid reset type %d\n", reset); 3109 3110 /* If a reset cycle isn't already in progress, we set a bit in 3111 * pf->state so that the service task can start a reset/rebuild. 3112 */ 3113 if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) { 3114 if (reset == ICE_RESET_CORER) 3115 set_bit(ICE_CORER_RECV, pf->state); 3116 else if (reset == ICE_RESET_GLOBR) 3117 set_bit(ICE_GLOBR_RECV, pf->state); 3118 else 3119 set_bit(ICE_EMPR_RECV, pf->state); 3120 3121 /* There are couple of different bits at play here. 3122 * hw->reset_ongoing indicates whether the hardware is 3123 * in reset. This is set to true when a reset interrupt 3124 * is received and set back to false after the driver 3125 * has determined that the hardware is out of reset. 3126 * 3127 * ICE_RESET_OICR_RECV in pf->state indicates 3128 * that a post reset rebuild is required before the 3129 * driver is operational again. This is set above. 3130 * 3131 * As this is the start of the reset/rebuild cycle, set 3132 * both to indicate that. 3133 */ 3134 hw->reset_ongoing = true; 3135 } 3136 } 3137 3138 if (oicr & PFINT_OICR_TSYN_TX_M) { 3139 ena_mask &= ~PFINT_OICR_TSYN_TX_M; 3140 if (!hw->reset_ongoing) 3141 set_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread); 3142 } 3143 3144 if (oicr & PFINT_OICR_TSYN_EVNT_M) { 3145 u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; 3146 u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx)); 3147 3148 ena_mask &= ~PFINT_OICR_TSYN_EVNT_M; 3149 3150 if (hw->func_caps.ts_func_info.src_tmr_owned) { 3151 /* Save EVENTs from GLTSYN register */ 3152 pf->ptp.ext_ts_irq |= gltsyn_stat & 3153 (GLTSYN_STAT_EVENT0_M | 3154 GLTSYN_STAT_EVENT1_M | 3155 GLTSYN_STAT_EVENT2_M); 3156 3157 set_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread); 3158 } 3159 } 3160 3161 #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M) 3162 if (oicr & ICE_AUX_CRIT_ERR) { 3163 pf->oicr_err_reg |= oicr; 3164 set_bit(ICE_AUX_ERR_PENDING, pf->state); 3165 ena_mask &= ~ICE_AUX_CRIT_ERR; 3166 } 3167 3168 /* Report any remaining unexpected interrupts */ 3169 oicr &= ena_mask; 3170 if (oicr) { 3171 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 3172 /* If a critical error is pending there is no choice but to 3173 * reset the device. 3174 */ 3175 if (oicr & (PFINT_OICR_PCI_EXCEPTION_M | 3176 PFINT_OICR_ECC_ERR_M)) { 3177 set_bit(ICE_PFR_REQ, pf->state); 3178 } 3179 } 3180 3181 return IRQ_WAKE_THREAD; 3182 } 3183 3184 /** 3185 * ice_misc_intr_thread_fn - misc interrupt thread function 3186 * @irq: interrupt number 3187 * @data: pointer to a q_vector 3188 */ 3189 static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data) 3190 { 3191 struct ice_pf *pf = data; 3192 struct ice_hw *hw; 3193 3194 hw = &pf->hw; 3195 3196 if (ice_is_reset_in_progress(pf->state)) 3197 return IRQ_HANDLED; 3198 3199 ice_service_task_schedule(pf); 3200 3201 if (test_and_clear_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread)) 3202 ice_ptp_extts_event(pf); 3203 3204 if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) { 3205 /* Process outstanding Tx timestamps. If there is more work, 3206 * re-arm the interrupt to trigger again. 3207 */ 3208 if (ice_ptp_process_ts(pf) == ICE_TX_TSTAMP_WORK_PENDING) { 3209 wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M); 3210 ice_flush(hw); 3211 } 3212 } 3213 3214 ice_irq_dynamic_ena(hw, NULL, NULL); 3215 3216 return IRQ_HANDLED; 3217 } 3218 3219 /** 3220 * ice_dis_ctrlq_interrupts - disable control queue interrupts 3221 * @hw: pointer to HW structure 3222 */ 3223 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 3224 { 3225 /* disable Admin queue Interrupt causes */ 3226 wr32(hw, PFINT_FW_CTL, 3227 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 3228 3229 /* disable Mailbox queue Interrupt causes */ 3230 wr32(hw, PFINT_MBX_CTL, 3231 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 3232 3233 wr32(hw, PFINT_SB_CTL, 3234 rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M); 3235 3236 /* disable Control queue Interrupt causes */ 3237 wr32(hw, PFINT_OICR_CTL, 3238 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 3239 3240 ice_flush(hw); 3241 } 3242 3243 /** 3244 * ice_free_irq_msix_misc - Unroll misc vector setup 3245 * @pf: board private structure 3246 */ 3247 static void ice_free_irq_msix_misc(struct ice_pf *pf) 3248 { 3249 int misc_irq_num = pf->oicr_irq.virq; 3250 struct ice_hw *hw = &pf->hw; 3251 3252 ice_dis_ctrlq_interrupts(hw); 3253 3254 /* disable OICR interrupt */ 3255 wr32(hw, PFINT_OICR_ENA, 0); 3256 ice_flush(hw); 3257 3258 synchronize_irq(misc_irq_num); 3259 devm_free_irq(ice_pf_to_dev(pf), misc_irq_num, pf); 3260 3261 ice_free_irq(pf, pf->oicr_irq); 3262 } 3263 3264 /** 3265 * ice_ena_ctrlq_interrupts - enable control queue interrupts 3266 * @hw: pointer to HW structure 3267 * @reg_idx: HW vector index to associate the control queue interrupts with 3268 */ 3269 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 3270 { 3271 u32 val; 3272 3273 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 3274 PFINT_OICR_CTL_CAUSE_ENA_M); 3275 wr32(hw, PFINT_OICR_CTL, val); 3276 3277 /* enable Admin queue Interrupt causes */ 3278 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 3279 PFINT_FW_CTL_CAUSE_ENA_M); 3280 wr32(hw, PFINT_FW_CTL, val); 3281 3282 /* enable Mailbox queue Interrupt causes */ 3283 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 3284 PFINT_MBX_CTL_CAUSE_ENA_M); 3285 wr32(hw, PFINT_MBX_CTL, val); 3286 3287 /* This enables Sideband queue Interrupt causes */ 3288 val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) | 3289 PFINT_SB_CTL_CAUSE_ENA_M); 3290 wr32(hw, PFINT_SB_CTL, val); 3291 3292 ice_flush(hw); 3293 } 3294 3295 /** 3296 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 3297 * @pf: board private structure 3298 * 3299 * This sets up the handler for MSIX 0, which is used to manage the 3300 * non-queue interrupts, e.g. AdminQ and errors. This is not used 3301 * when in MSI or Legacy interrupt mode. 3302 */ 3303 static int ice_req_irq_msix_misc(struct ice_pf *pf) 3304 { 3305 struct device *dev = ice_pf_to_dev(pf); 3306 struct ice_hw *hw = &pf->hw; 3307 struct msi_map oicr_irq; 3308 int err = 0; 3309 3310 if (!pf->int_name[0]) 3311 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 3312 dev_driver_string(dev), dev_name(dev)); 3313 3314 /* Do not request IRQ but do enable OICR interrupt since settings are 3315 * lost during reset. Note that this function is called only during 3316 * rebuild path and not while reset is in progress. 3317 */ 3318 if (ice_is_reset_in_progress(pf->state)) 3319 goto skip_req_irq; 3320 3321 /* reserve one vector in irq_tracker for misc interrupts */ 3322 oicr_irq = ice_alloc_irq(pf, false); 3323 if (oicr_irq.index < 0) 3324 return oicr_irq.index; 3325 3326 pf->oicr_irq = oicr_irq; 3327 err = devm_request_threaded_irq(dev, pf->oicr_irq.virq, ice_misc_intr, 3328 ice_misc_intr_thread_fn, 0, 3329 pf->int_name, pf); 3330 if (err) { 3331 dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n", 3332 pf->int_name, err); 3333 ice_free_irq(pf, pf->oicr_irq); 3334 return err; 3335 } 3336 3337 skip_req_irq: 3338 ice_ena_misc_vector(pf); 3339 3340 ice_ena_ctrlq_interrupts(hw, pf->oicr_irq.index); 3341 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_irq.index), 3342 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 3343 3344 ice_flush(hw); 3345 ice_irq_dynamic_ena(hw, NULL, NULL); 3346 3347 return 0; 3348 } 3349 3350 /** 3351 * ice_napi_add - register NAPI handler for the VSI 3352 * @vsi: VSI for which NAPI handler is to be registered 3353 * 3354 * This function is only called in the driver's load path. Registering the NAPI 3355 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 3356 * reset/rebuild, etc.) 3357 */ 3358 static void ice_napi_add(struct ice_vsi *vsi) 3359 { 3360 int v_idx; 3361 3362 if (!vsi->netdev) 3363 return; 3364 3365 ice_for_each_q_vector(vsi, v_idx) 3366 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 3367 ice_napi_poll); 3368 } 3369 3370 /** 3371 * ice_set_ops - set netdev and ethtools ops for the given netdev 3372 * @vsi: the VSI associated with the new netdev 3373 */ 3374 static void ice_set_ops(struct ice_vsi *vsi) 3375 { 3376 struct net_device *netdev = vsi->netdev; 3377 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3378 3379 if (ice_is_safe_mode(pf)) { 3380 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 3381 ice_set_ethtool_safe_mode_ops(netdev); 3382 return; 3383 } 3384 3385 netdev->netdev_ops = &ice_netdev_ops; 3386 netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; 3387 ice_set_ethtool_ops(netdev); 3388 3389 if (vsi->type != ICE_VSI_PF) 3390 return; 3391 3392 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 3393 NETDEV_XDP_ACT_XSK_ZEROCOPY | 3394 NETDEV_XDP_ACT_RX_SG; 3395 netdev->xdp_zc_max_segs = ICE_MAX_BUF_TXD; 3396 } 3397 3398 /** 3399 * ice_set_netdev_features - set features for the given netdev 3400 * @netdev: netdev instance 3401 */ 3402 static void ice_set_netdev_features(struct net_device *netdev) 3403 { 3404 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3405 bool is_dvm_ena = ice_is_dvm_ena(&pf->hw); 3406 netdev_features_t csumo_features; 3407 netdev_features_t vlano_features; 3408 netdev_features_t dflt_features; 3409 netdev_features_t tso_features; 3410 3411 if (ice_is_safe_mode(pf)) { 3412 /* safe mode */ 3413 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 3414 netdev->hw_features = netdev->features; 3415 return; 3416 } 3417 3418 dflt_features = NETIF_F_SG | 3419 NETIF_F_HIGHDMA | 3420 NETIF_F_NTUPLE | 3421 NETIF_F_RXHASH; 3422 3423 csumo_features = NETIF_F_RXCSUM | 3424 NETIF_F_IP_CSUM | 3425 NETIF_F_SCTP_CRC | 3426 NETIF_F_IPV6_CSUM; 3427 3428 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 3429 NETIF_F_HW_VLAN_CTAG_TX | 3430 NETIF_F_HW_VLAN_CTAG_RX; 3431 3432 /* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */ 3433 if (is_dvm_ena) 3434 vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER; 3435 3436 tso_features = NETIF_F_TSO | 3437 NETIF_F_TSO_ECN | 3438 NETIF_F_TSO6 | 3439 NETIF_F_GSO_GRE | 3440 NETIF_F_GSO_UDP_TUNNEL | 3441 NETIF_F_GSO_GRE_CSUM | 3442 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3443 NETIF_F_GSO_PARTIAL | 3444 NETIF_F_GSO_IPXIP4 | 3445 NETIF_F_GSO_IPXIP6 | 3446 NETIF_F_GSO_UDP_L4; 3447 3448 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 3449 NETIF_F_GSO_GRE_CSUM; 3450 /* set features that user can change */ 3451 netdev->hw_features = dflt_features | csumo_features | 3452 vlano_features | tso_features; 3453 3454 /* add support for HW_CSUM on packets with MPLS header */ 3455 netdev->mpls_features = NETIF_F_HW_CSUM | 3456 NETIF_F_TSO | 3457 NETIF_F_TSO6; 3458 3459 /* enable features */ 3460 netdev->features |= netdev->hw_features; 3461 3462 netdev->hw_features |= NETIF_F_HW_TC; 3463 netdev->hw_features |= NETIF_F_LOOPBACK; 3464 3465 /* encap and VLAN devices inherit default, csumo and tso features */ 3466 netdev->hw_enc_features |= dflt_features | csumo_features | 3467 tso_features; 3468 netdev->vlan_features |= dflt_features | csumo_features | 3469 tso_features; 3470 3471 /* advertise support but don't enable by default since only one type of 3472 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one 3473 * type turns on the other has to be turned off. This is enforced by the 3474 * ice_fix_features() ndo callback. 3475 */ 3476 if (is_dvm_ena) 3477 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX | 3478 NETIF_F_HW_VLAN_STAG_TX; 3479 3480 /* Leave CRC / FCS stripping enabled by default, but allow the value to 3481 * be changed at runtime 3482 */ 3483 netdev->hw_features |= NETIF_F_RXFCS; 3484 3485 netif_set_tso_max_size(netdev, ICE_MAX_TSO_SIZE); 3486 } 3487 3488 /** 3489 * ice_fill_rss_lut - Fill the RSS lookup table with default values 3490 * @lut: Lookup table 3491 * @rss_table_size: Lookup table size 3492 * @rss_size: Range of queue number for hashing 3493 */ 3494 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 3495 { 3496 u16 i; 3497 3498 for (i = 0; i < rss_table_size; i++) 3499 lut[i] = i % rss_size; 3500 } 3501 3502 /** 3503 * ice_pf_vsi_setup - Set up a PF VSI 3504 * @pf: board private structure 3505 * @pi: pointer to the port_info instance 3506 * 3507 * Returns pointer to the successfully allocated VSI software struct 3508 * on success, otherwise returns NULL on failure. 3509 */ 3510 static struct ice_vsi * 3511 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3512 { 3513 struct ice_vsi_cfg_params params = {}; 3514 3515 params.type = ICE_VSI_PF; 3516 params.pi = pi; 3517 params.flags = ICE_VSI_FLAG_INIT; 3518 3519 return ice_vsi_setup(pf, ¶ms); 3520 } 3521 3522 static struct ice_vsi * 3523 ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, 3524 struct ice_channel *ch) 3525 { 3526 struct ice_vsi_cfg_params params = {}; 3527 3528 params.type = ICE_VSI_CHNL; 3529 params.pi = pi; 3530 params.ch = ch; 3531 params.flags = ICE_VSI_FLAG_INIT; 3532 3533 return ice_vsi_setup(pf, ¶ms); 3534 } 3535 3536 /** 3537 * ice_ctrl_vsi_setup - Set up a control VSI 3538 * @pf: board private structure 3539 * @pi: pointer to the port_info instance 3540 * 3541 * Returns pointer to the successfully allocated VSI software struct 3542 * on success, otherwise returns NULL on failure. 3543 */ 3544 static struct ice_vsi * 3545 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3546 { 3547 struct ice_vsi_cfg_params params = {}; 3548 3549 params.type = ICE_VSI_CTRL; 3550 params.pi = pi; 3551 params.flags = ICE_VSI_FLAG_INIT; 3552 3553 return ice_vsi_setup(pf, ¶ms); 3554 } 3555 3556 /** 3557 * ice_lb_vsi_setup - Set up a loopback VSI 3558 * @pf: board private structure 3559 * @pi: pointer to the port_info instance 3560 * 3561 * Returns pointer to the successfully allocated VSI software struct 3562 * on success, otherwise returns NULL on failure. 3563 */ 3564 struct ice_vsi * 3565 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3566 { 3567 struct ice_vsi_cfg_params params = {}; 3568 3569 params.type = ICE_VSI_LB; 3570 params.pi = pi; 3571 params.flags = ICE_VSI_FLAG_INIT; 3572 3573 return ice_vsi_setup(pf, ¶ms); 3574 } 3575 3576 /** 3577 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 3578 * @netdev: network interface to be adjusted 3579 * @proto: VLAN TPID 3580 * @vid: VLAN ID to be added 3581 * 3582 * net_device_ops implementation for adding VLAN IDs 3583 */ 3584 static int 3585 ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 3586 { 3587 struct ice_netdev_priv *np = netdev_priv(netdev); 3588 struct ice_vsi_vlan_ops *vlan_ops; 3589 struct ice_vsi *vsi = np->vsi; 3590 struct ice_vlan vlan; 3591 int ret; 3592 3593 /* VLAN 0 is added by default during load/reset */ 3594 if (!vid) 3595 return 0; 3596 3597 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3598 usleep_range(1000, 2000); 3599 3600 /* Add multicast promisc rule for the VLAN ID to be added if 3601 * all-multicast is currently enabled. 3602 */ 3603 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3604 ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3605 ICE_MCAST_VLAN_PROMISC_BITS, 3606 vid); 3607 if (ret) 3608 goto finish; 3609 } 3610 3611 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3612 3613 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 3614 * packets aren't pruned by the device's internal switch on Rx 3615 */ 3616 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3617 ret = vlan_ops->add_vlan(vsi, &vlan); 3618 if (ret) 3619 goto finish; 3620 3621 /* If all-multicast is currently enabled and this VLAN ID is only one 3622 * besides VLAN-0 we have to update look-up type of multicast promisc 3623 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN. 3624 */ 3625 if ((vsi->current_netdev_flags & IFF_ALLMULTI) && 3626 ice_vsi_num_non_zero_vlans(vsi) == 1) { 3627 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3628 ICE_MCAST_PROMISC_BITS, 0); 3629 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3630 ICE_MCAST_VLAN_PROMISC_BITS, 0); 3631 } 3632 3633 finish: 3634 clear_bit(ICE_CFG_BUSY, vsi->state); 3635 3636 return ret; 3637 } 3638 3639 /** 3640 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 3641 * @netdev: network interface to be adjusted 3642 * @proto: VLAN TPID 3643 * @vid: VLAN ID to be removed 3644 * 3645 * net_device_ops implementation for removing VLAN IDs 3646 */ 3647 static int 3648 ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 3649 { 3650 struct ice_netdev_priv *np = netdev_priv(netdev); 3651 struct ice_vsi_vlan_ops *vlan_ops; 3652 struct ice_vsi *vsi = np->vsi; 3653 struct ice_vlan vlan; 3654 int ret; 3655 3656 /* don't allow removal of VLAN 0 */ 3657 if (!vid) 3658 return 0; 3659 3660 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3661 usleep_range(1000, 2000); 3662 3663 ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3664 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3665 if (ret) { 3666 netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n", 3667 vsi->vsi_num); 3668 vsi->current_netdev_flags |= IFF_ALLMULTI; 3669 } 3670 3671 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3672 3673 /* Make sure VLAN delete is successful before updating VLAN 3674 * information 3675 */ 3676 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3677 ret = vlan_ops->del_vlan(vsi, &vlan); 3678 if (ret) 3679 goto finish; 3680 3681 /* Remove multicast promisc rule for the removed VLAN ID if 3682 * all-multicast is enabled. 3683 */ 3684 if (vsi->current_netdev_flags & IFF_ALLMULTI) 3685 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3686 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3687 3688 if (!ice_vsi_has_non_zero_vlans(vsi)) { 3689 /* Update look-up type of multicast promisc rule for VLAN 0 3690 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when 3691 * all-multicast is enabled and VLAN 0 is the only VLAN rule. 3692 */ 3693 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3694 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3695 ICE_MCAST_VLAN_PROMISC_BITS, 3696 0); 3697 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3698 ICE_MCAST_PROMISC_BITS, 0); 3699 } 3700 } 3701 3702 finish: 3703 clear_bit(ICE_CFG_BUSY, vsi->state); 3704 3705 return ret; 3706 } 3707 3708 /** 3709 * ice_rep_indr_tc_block_unbind 3710 * @cb_priv: indirection block private data 3711 */ 3712 static void ice_rep_indr_tc_block_unbind(void *cb_priv) 3713 { 3714 struct ice_indr_block_priv *indr_priv = cb_priv; 3715 3716 list_del(&indr_priv->list); 3717 kfree(indr_priv); 3718 } 3719 3720 /** 3721 * ice_tc_indir_block_unregister - Unregister TC indirect block notifications 3722 * @vsi: VSI struct which has the netdev 3723 */ 3724 static void ice_tc_indir_block_unregister(struct ice_vsi *vsi) 3725 { 3726 struct ice_netdev_priv *np = netdev_priv(vsi->netdev); 3727 3728 flow_indr_dev_unregister(ice_indr_setup_tc_cb, np, 3729 ice_rep_indr_tc_block_unbind); 3730 } 3731 3732 /** 3733 * ice_tc_indir_block_register - Register TC indirect block notifications 3734 * @vsi: VSI struct which has the netdev 3735 * 3736 * Returns 0 on success, negative value on failure 3737 */ 3738 static int ice_tc_indir_block_register(struct ice_vsi *vsi) 3739 { 3740 struct ice_netdev_priv *np; 3741 3742 if (!vsi || !vsi->netdev) 3743 return -EINVAL; 3744 3745 np = netdev_priv(vsi->netdev); 3746 3747 INIT_LIST_HEAD(&np->tc_indr_block_priv_list); 3748 return flow_indr_dev_register(ice_indr_setup_tc_cb, np); 3749 } 3750 3751 /** 3752 * ice_get_avail_q_count - Get count of queues in use 3753 * @pf_qmap: bitmap to get queue use count from 3754 * @lock: pointer to a mutex that protects access to pf_qmap 3755 * @size: size of the bitmap 3756 */ 3757 static u16 3758 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 3759 { 3760 unsigned long bit; 3761 u16 count = 0; 3762 3763 mutex_lock(lock); 3764 for_each_clear_bit(bit, pf_qmap, size) 3765 count++; 3766 mutex_unlock(lock); 3767 3768 return count; 3769 } 3770 3771 /** 3772 * ice_get_avail_txq_count - Get count of Tx queues in use 3773 * @pf: pointer to an ice_pf instance 3774 */ 3775 u16 ice_get_avail_txq_count(struct ice_pf *pf) 3776 { 3777 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3778 pf->max_pf_txqs); 3779 } 3780 3781 /** 3782 * ice_get_avail_rxq_count - Get count of Rx queues in use 3783 * @pf: pointer to an ice_pf instance 3784 */ 3785 u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3786 { 3787 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3788 pf->max_pf_rxqs); 3789 } 3790 3791 /** 3792 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3793 * @pf: board private structure to initialize 3794 */ 3795 static void ice_deinit_pf(struct ice_pf *pf) 3796 { 3797 ice_service_task_stop(pf); 3798 mutex_destroy(&pf->adev_mutex); 3799 mutex_destroy(&pf->sw_mutex); 3800 mutex_destroy(&pf->tc_mutex); 3801 mutex_destroy(&pf->avail_q_mutex); 3802 mutex_destroy(&pf->vfs.table_lock); 3803 3804 if (pf->avail_txqs) { 3805 bitmap_free(pf->avail_txqs); 3806 pf->avail_txqs = NULL; 3807 } 3808 3809 if (pf->avail_rxqs) { 3810 bitmap_free(pf->avail_rxqs); 3811 pf->avail_rxqs = NULL; 3812 } 3813 3814 if (pf->ptp.clock) 3815 ptp_clock_unregister(pf->ptp.clock); 3816 } 3817 3818 /** 3819 * ice_set_pf_caps - set PFs capability flags 3820 * @pf: pointer to the PF instance 3821 */ 3822 static void ice_set_pf_caps(struct ice_pf *pf) 3823 { 3824 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3825 3826 clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3827 if (func_caps->common_cap.rdma) 3828 set_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3829 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3830 if (func_caps->common_cap.dcb) 3831 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3832 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3833 if (func_caps->common_cap.sr_iov_1_1) { 3834 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3835 pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs, 3836 ICE_MAX_SRIOV_VFS); 3837 } 3838 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 3839 if (func_caps->common_cap.rss_table_size) 3840 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 3841 3842 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 3843 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 3844 u16 unused; 3845 3846 /* ctrl_vsi_idx will be set to a valid value when flow director 3847 * is setup by ice_init_fdir 3848 */ 3849 pf->ctrl_vsi_idx = ICE_NO_VSI; 3850 set_bit(ICE_FLAG_FD_ENA, pf->flags); 3851 /* force guaranteed filter pool for PF */ 3852 ice_alloc_fd_guar_item(&pf->hw, &unused, 3853 func_caps->fd_fltr_guar); 3854 /* force shared filter pool for PF */ 3855 ice_alloc_fd_shrd_item(&pf->hw, &unused, 3856 func_caps->fd_fltr_best_effort); 3857 } 3858 3859 clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 3860 if (func_caps->common_cap.ieee_1588) 3861 set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 3862 3863 pf->max_pf_txqs = func_caps->common_cap.num_txq; 3864 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 3865 } 3866 3867 /** 3868 * ice_init_pf - Initialize general software structures (struct ice_pf) 3869 * @pf: board private structure to initialize 3870 */ 3871 static int ice_init_pf(struct ice_pf *pf) 3872 { 3873 ice_set_pf_caps(pf); 3874 3875 mutex_init(&pf->sw_mutex); 3876 mutex_init(&pf->tc_mutex); 3877 mutex_init(&pf->adev_mutex); 3878 3879 INIT_HLIST_HEAD(&pf->aq_wait_list); 3880 spin_lock_init(&pf->aq_wait_lock); 3881 init_waitqueue_head(&pf->aq_wait_queue); 3882 3883 init_waitqueue_head(&pf->reset_wait_queue); 3884 3885 /* setup service timer and periodic service task */ 3886 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 3887 pf->serv_tmr_period = HZ; 3888 INIT_WORK(&pf->serv_task, ice_service_task); 3889 clear_bit(ICE_SERVICE_SCHED, pf->state); 3890 3891 mutex_init(&pf->avail_q_mutex); 3892 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 3893 if (!pf->avail_txqs) 3894 return -ENOMEM; 3895 3896 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 3897 if (!pf->avail_rxqs) { 3898 bitmap_free(pf->avail_txqs); 3899 pf->avail_txqs = NULL; 3900 return -ENOMEM; 3901 } 3902 3903 mutex_init(&pf->vfs.table_lock); 3904 hash_init(pf->vfs.table); 3905 ice_mbx_init_snapshot(&pf->hw); 3906 3907 return 0; 3908 } 3909 3910 /** 3911 * ice_is_wol_supported - check if WoL is supported 3912 * @hw: pointer to hardware info 3913 * 3914 * Check if WoL is supported based on the HW configuration. 3915 * Returns true if NVM supports and enables WoL for this port, false otherwise 3916 */ 3917 bool ice_is_wol_supported(struct ice_hw *hw) 3918 { 3919 u16 wol_ctrl; 3920 3921 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 3922 * word) indicates WoL is not supported on the corresponding PF ID. 3923 */ 3924 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 3925 return false; 3926 3927 return !(BIT(hw->port_info->lport) & wol_ctrl); 3928 } 3929 3930 /** 3931 * ice_vsi_recfg_qs - Change the number of queues on a VSI 3932 * @vsi: VSI being changed 3933 * @new_rx: new number of Rx queues 3934 * @new_tx: new number of Tx queues 3935 * @locked: is adev device_lock held 3936 * 3937 * Only change the number of queues if new_tx, or new_rx is non-0. 3938 * 3939 * Returns 0 on success. 3940 */ 3941 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked) 3942 { 3943 struct ice_pf *pf = vsi->back; 3944 int err = 0, timeout = 50; 3945 3946 if (!new_rx && !new_tx) 3947 return -EINVAL; 3948 3949 while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) { 3950 timeout--; 3951 if (!timeout) 3952 return -EBUSY; 3953 usleep_range(1000, 2000); 3954 } 3955 3956 if (new_tx) 3957 vsi->req_txq = (u16)new_tx; 3958 if (new_rx) 3959 vsi->req_rxq = (u16)new_rx; 3960 3961 /* set for the next time the netdev is started */ 3962 if (!netif_running(vsi->netdev)) { 3963 ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 3964 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 3965 goto done; 3966 } 3967 3968 ice_vsi_close(vsi); 3969 ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 3970 ice_pf_dcb_recfg(pf, locked); 3971 ice_vsi_open(vsi); 3972 done: 3973 clear_bit(ICE_CFG_BUSY, pf->state); 3974 return err; 3975 } 3976 3977 /** 3978 * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode 3979 * @pf: PF to configure 3980 * 3981 * No VLAN offloads/filtering are advertised in safe mode so make sure the PF 3982 * VSI can still Tx/Rx VLAN tagged packets. 3983 */ 3984 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf) 3985 { 3986 struct ice_vsi *vsi = ice_get_main_vsi(pf); 3987 struct ice_vsi_ctx *ctxt; 3988 struct ice_hw *hw; 3989 int status; 3990 3991 if (!vsi) 3992 return; 3993 3994 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 3995 if (!ctxt) 3996 return; 3997 3998 hw = &pf->hw; 3999 ctxt->info = vsi->info; 4000 4001 ctxt->info.valid_sections = 4002 cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID | 4003 ICE_AQ_VSI_PROP_SECURITY_VALID | 4004 ICE_AQ_VSI_PROP_SW_VALID); 4005 4006 /* disable VLAN anti-spoof */ 4007 ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << 4008 ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); 4009 4010 /* disable VLAN pruning and keep all other settings */ 4011 ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; 4012 4013 /* allow all VLANs on Tx and don't strip on Rx */ 4014 ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL | 4015 ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING; 4016 4017 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 4018 if (status) { 4019 dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n", 4020 status, ice_aq_str(hw->adminq.sq_last_status)); 4021 } else { 4022 vsi->info.sec_flags = ctxt->info.sec_flags; 4023 vsi->info.sw_flags2 = ctxt->info.sw_flags2; 4024 vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags; 4025 } 4026 4027 kfree(ctxt); 4028 } 4029 4030 /** 4031 * ice_log_pkg_init - log result of DDP package load 4032 * @hw: pointer to hardware info 4033 * @state: state of package load 4034 */ 4035 static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state) 4036 { 4037 struct ice_pf *pf = hw->back; 4038 struct device *dev; 4039 4040 dev = ice_pf_to_dev(pf); 4041 4042 switch (state) { 4043 case ICE_DDP_PKG_SUCCESS: 4044 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 4045 hw->active_pkg_name, 4046 hw->active_pkg_ver.major, 4047 hw->active_pkg_ver.minor, 4048 hw->active_pkg_ver.update, 4049 hw->active_pkg_ver.draft); 4050 break; 4051 case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED: 4052 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 4053 hw->active_pkg_name, 4054 hw->active_pkg_ver.major, 4055 hw->active_pkg_ver.minor, 4056 hw->active_pkg_ver.update, 4057 hw->active_pkg_ver.draft); 4058 break; 4059 case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED: 4060 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 4061 hw->active_pkg_name, 4062 hw->active_pkg_ver.major, 4063 hw->active_pkg_ver.minor, 4064 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4065 break; 4066 case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED: 4067 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 4068 hw->active_pkg_name, 4069 hw->active_pkg_ver.major, 4070 hw->active_pkg_ver.minor, 4071 hw->active_pkg_ver.update, 4072 hw->active_pkg_ver.draft, 4073 hw->pkg_name, 4074 hw->pkg_ver.major, 4075 hw->pkg_ver.minor, 4076 hw->pkg_ver.update, 4077 hw->pkg_ver.draft); 4078 break; 4079 case ICE_DDP_PKG_FW_MISMATCH: 4080 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 4081 break; 4082 case ICE_DDP_PKG_INVALID_FILE: 4083 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 4084 break; 4085 case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH: 4086 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 4087 break; 4088 case ICE_DDP_PKG_FILE_VERSION_TOO_LOW: 4089 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 4090 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4091 break; 4092 case ICE_DDP_PKG_FILE_SIGNATURE_INVALID: 4093 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 4094 break; 4095 case ICE_DDP_PKG_FILE_REVISION_TOO_LOW: 4096 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 4097 break; 4098 case ICE_DDP_PKG_LOAD_ERROR: 4099 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 4100 /* poll for reset to complete */ 4101 if (ice_check_reset(hw)) 4102 dev_err(dev, "Error resetting device. Please reload the driver\n"); 4103 break; 4104 case ICE_DDP_PKG_ERR: 4105 default: 4106 dev_err(dev, "An unknown error occurred when loading the DDP package. Entering Safe Mode.\n"); 4107 break; 4108 } 4109 } 4110 4111 /** 4112 * ice_load_pkg - load/reload the DDP Package file 4113 * @firmware: firmware structure when firmware requested or NULL for reload 4114 * @pf: pointer to the PF instance 4115 * 4116 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 4117 * initialize HW tables. 4118 */ 4119 static void 4120 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 4121 { 4122 enum ice_ddp_state state = ICE_DDP_PKG_ERR; 4123 struct device *dev = ice_pf_to_dev(pf); 4124 struct ice_hw *hw = &pf->hw; 4125 4126 /* Load DDP Package */ 4127 if (firmware && !hw->pkg_copy) { 4128 state = ice_copy_and_init_pkg(hw, firmware->data, 4129 firmware->size); 4130 ice_log_pkg_init(hw, state); 4131 } else if (!firmware && hw->pkg_copy) { 4132 /* Reload package during rebuild after CORER/GLOBR reset */ 4133 state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 4134 ice_log_pkg_init(hw, state); 4135 } else { 4136 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 4137 } 4138 4139 if (!ice_is_init_pkg_successful(state)) { 4140 /* Safe Mode */ 4141 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4142 return; 4143 } 4144 4145 /* Successful download package is the precondition for advanced 4146 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 4147 */ 4148 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4149 } 4150 4151 /** 4152 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 4153 * @pf: pointer to the PF structure 4154 * 4155 * There is no error returned here because the driver should be able to handle 4156 * 128 Byte cache lines, so we only print a warning in case issues are seen, 4157 * specifically with Tx. 4158 */ 4159 static void ice_verify_cacheline_size(struct ice_pf *pf) 4160 { 4161 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 4162 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 4163 ICE_CACHE_LINE_BYTES); 4164 } 4165 4166 /** 4167 * ice_send_version - update firmware with driver version 4168 * @pf: PF struct 4169 * 4170 * Returns 0 on success, else error code 4171 */ 4172 static int ice_send_version(struct ice_pf *pf) 4173 { 4174 struct ice_driver_ver dv; 4175 4176 dv.major_ver = 0xff; 4177 dv.minor_ver = 0xff; 4178 dv.build_ver = 0xff; 4179 dv.subbuild_ver = 0; 4180 strscpy((char *)dv.driver_string, UTS_RELEASE, 4181 sizeof(dv.driver_string)); 4182 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 4183 } 4184 4185 /** 4186 * ice_init_fdir - Initialize flow director VSI and configuration 4187 * @pf: pointer to the PF instance 4188 * 4189 * returns 0 on success, negative on error 4190 */ 4191 static int ice_init_fdir(struct ice_pf *pf) 4192 { 4193 struct device *dev = ice_pf_to_dev(pf); 4194 struct ice_vsi *ctrl_vsi; 4195 int err; 4196 4197 /* Side Band Flow Director needs to have a control VSI. 4198 * Allocate it and store it in the PF. 4199 */ 4200 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 4201 if (!ctrl_vsi) { 4202 dev_dbg(dev, "could not create control VSI\n"); 4203 return -ENOMEM; 4204 } 4205 4206 err = ice_vsi_open_ctrl(ctrl_vsi); 4207 if (err) { 4208 dev_dbg(dev, "could not open control VSI\n"); 4209 goto err_vsi_open; 4210 } 4211 4212 mutex_init(&pf->hw.fdir_fltr_lock); 4213 4214 err = ice_fdir_create_dflt_rules(pf); 4215 if (err) 4216 goto err_fdir_rule; 4217 4218 return 0; 4219 4220 err_fdir_rule: 4221 ice_fdir_release_flows(&pf->hw); 4222 ice_vsi_close(ctrl_vsi); 4223 err_vsi_open: 4224 ice_vsi_release(ctrl_vsi); 4225 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 4226 pf->vsi[pf->ctrl_vsi_idx] = NULL; 4227 pf->ctrl_vsi_idx = ICE_NO_VSI; 4228 } 4229 return err; 4230 } 4231 4232 static void ice_deinit_fdir(struct ice_pf *pf) 4233 { 4234 struct ice_vsi *vsi = ice_get_ctrl_vsi(pf); 4235 4236 if (!vsi) 4237 return; 4238 4239 ice_vsi_manage_fdir(vsi, false); 4240 ice_vsi_release(vsi); 4241 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 4242 pf->vsi[pf->ctrl_vsi_idx] = NULL; 4243 pf->ctrl_vsi_idx = ICE_NO_VSI; 4244 } 4245 4246 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 4247 } 4248 4249 /** 4250 * ice_get_opt_fw_name - return optional firmware file name or NULL 4251 * @pf: pointer to the PF instance 4252 */ 4253 static char *ice_get_opt_fw_name(struct ice_pf *pf) 4254 { 4255 /* Optional firmware name same as default with additional dash 4256 * followed by a EUI-64 identifier (PCIe Device Serial Number) 4257 */ 4258 struct pci_dev *pdev = pf->pdev; 4259 char *opt_fw_filename; 4260 u64 dsn; 4261 4262 /* Determine the name of the optional file using the DSN (two 4263 * dwords following the start of the DSN Capability). 4264 */ 4265 dsn = pci_get_dsn(pdev); 4266 if (!dsn) 4267 return NULL; 4268 4269 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 4270 if (!opt_fw_filename) 4271 return NULL; 4272 4273 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 4274 ICE_DDP_PKG_PATH, dsn); 4275 4276 return opt_fw_filename; 4277 } 4278 4279 /** 4280 * ice_request_fw - Device initialization routine 4281 * @pf: pointer to the PF instance 4282 */ 4283 static void ice_request_fw(struct ice_pf *pf) 4284 { 4285 char *opt_fw_filename = ice_get_opt_fw_name(pf); 4286 const struct firmware *firmware = NULL; 4287 struct device *dev = ice_pf_to_dev(pf); 4288 int err = 0; 4289 4290 /* optional device-specific DDP (if present) overrides the default DDP 4291 * package file. kernel logs a debug message if the file doesn't exist, 4292 * and warning messages for other errors. 4293 */ 4294 if (opt_fw_filename) { 4295 err = firmware_request_nowarn(&firmware, opt_fw_filename, dev); 4296 if (err) { 4297 kfree(opt_fw_filename); 4298 goto dflt_pkg_load; 4299 } 4300 4301 /* request for firmware was successful. Download to device */ 4302 ice_load_pkg(firmware, pf); 4303 kfree(opt_fw_filename); 4304 release_firmware(firmware); 4305 return; 4306 } 4307 4308 dflt_pkg_load: 4309 err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev); 4310 if (err) { 4311 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 4312 return; 4313 } 4314 4315 /* request for firmware was successful. Download to device */ 4316 ice_load_pkg(firmware, pf); 4317 release_firmware(firmware); 4318 } 4319 4320 /** 4321 * ice_print_wake_reason - show the wake up cause in the log 4322 * @pf: pointer to the PF struct 4323 */ 4324 static void ice_print_wake_reason(struct ice_pf *pf) 4325 { 4326 u32 wus = pf->wakeup_reason; 4327 const char *wake_str; 4328 4329 /* if no wake event, nothing to print */ 4330 if (!wus) 4331 return; 4332 4333 if (wus & PFPM_WUS_LNKC_M) 4334 wake_str = "Link\n"; 4335 else if (wus & PFPM_WUS_MAG_M) 4336 wake_str = "Magic Packet\n"; 4337 else if (wus & PFPM_WUS_MNG_M) 4338 wake_str = "Management\n"; 4339 else if (wus & PFPM_WUS_FW_RST_WK_M) 4340 wake_str = "Firmware Reset\n"; 4341 else 4342 wake_str = "Unknown\n"; 4343 4344 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 4345 } 4346 4347 /** 4348 * ice_register_netdev - register netdev 4349 * @vsi: pointer to the VSI struct 4350 */ 4351 static int ice_register_netdev(struct ice_vsi *vsi) 4352 { 4353 int err; 4354 4355 if (!vsi || !vsi->netdev) 4356 return -EIO; 4357 4358 err = register_netdev(vsi->netdev); 4359 if (err) 4360 return err; 4361 4362 set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4363 netif_carrier_off(vsi->netdev); 4364 netif_tx_stop_all_queues(vsi->netdev); 4365 4366 return 0; 4367 } 4368 4369 static void ice_unregister_netdev(struct ice_vsi *vsi) 4370 { 4371 if (!vsi || !vsi->netdev) 4372 return; 4373 4374 unregister_netdev(vsi->netdev); 4375 clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4376 } 4377 4378 /** 4379 * ice_cfg_netdev - Allocate, configure and register a netdev 4380 * @vsi: the VSI associated with the new netdev 4381 * 4382 * Returns 0 on success, negative value on failure 4383 */ 4384 static int ice_cfg_netdev(struct ice_vsi *vsi) 4385 { 4386 struct ice_netdev_priv *np; 4387 struct net_device *netdev; 4388 u8 mac_addr[ETH_ALEN]; 4389 4390 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 4391 vsi->alloc_rxq); 4392 if (!netdev) 4393 return -ENOMEM; 4394 4395 set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 4396 vsi->netdev = netdev; 4397 np = netdev_priv(netdev); 4398 np->vsi = vsi; 4399 4400 ice_set_netdev_features(netdev); 4401 ice_set_ops(vsi); 4402 4403 if (vsi->type == ICE_VSI_PF) { 4404 SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back)); 4405 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 4406 eth_hw_addr_set(netdev, mac_addr); 4407 } 4408 4409 netdev->priv_flags |= IFF_UNICAST_FLT; 4410 4411 /* Setup netdev TC information */ 4412 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 4413 4414 netdev->max_mtu = ICE_MAX_MTU; 4415 4416 return 0; 4417 } 4418 4419 static void ice_decfg_netdev(struct ice_vsi *vsi) 4420 { 4421 clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 4422 free_netdev(vsi->netdev); 4423 vsi->netdev = NULL; 4424 } 4425 4426 static int ice_start_eth(struct ice_vsi *vsi) 4427 { 4428 int err; 4429 4430 err = ice_init_mac_fltr(vsi->back); 4431 if (err) 4432 return err; 4433 4434 err = ice_vsi_open(vsi); 4435 if (err) 4436 ice_fltr_remove_all(vsi); 4437 4438 return err; 4439 } 4440 4441 static void ice_stop_eth(struct ice_vsi *vsi) 4442 { 4443 ice_fltr_remove_all(vsi); 4444 ice_vsi_close(vsi); 4445 } 4446 4447 static int ice_init_eth(struct ice_pf *pf) 4448 { 4449 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4450 int err; 4451 4452 if (!vsi) 4453 return -EINVAL; 4454 4455 /* init channel list */ 4456 INIT_LIST_HEAD(&vsi->ch_list); 4457 4458 err = ice_cfg_netdev(vsi); 4459 if (err) 4460 return err; 4461 /* Setup DCB netlink interface */ 4462 ice_dcbnl_setup(vsi); 4463 4464 err = ice_init_mac_fltr(pf); 4465 if (err) 4466 goto err_init_mac_fltr; 4467 4468 err = ice_devlink_create_pf_port(pf); 4469 if (err) 4470 goto err_devlink_create_pf_port; 4471 4472 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port); 4473 4474 err = ice_register_netdev(vsi); 4475 if (err) 4476 goto err_register_netdev; 4477 4478 err = ice_tc_indir_block_register(vsi); 4479 if (err) 4480 goto err_tc_indir_block_register; 4481 4482 ice_napi_add(vsi); 4483 4484 return 0; 4485 4486 err_tc_indir_block_register: 4487 ice_unregister_netdev(vsi); 4488 err_register_netdev: 4489 ice_devlink_destroy_pf_port(pf); 4490 err_devlink_create_pf_port: 4491 err_init_mac_fltr: 4492 ice_decfg_netdev(vsi); 4493 return err; 4494 } 4495 4496 static void ice_deinit_eth(struct ice_pf *pf) 4497 { 4498 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4499 4500 if (!vsi) 4501 return; 4502 4503 ice_vsi_close(vsi); 4504 ice_unregister_netdev(vsi); 4505 ice_devlink_destroy_pf_port(pf); 4506 ice_tc_indir_block_unregister(vsi); 4507 ice_decfg_netdev(vsi); 4508 } 4509 4510 static int ice_init_dev(struct ice_pf *pf) 4511 { 4512 struct device *dev = ice_pf_to_dev(pf); 4513 struct ice_hw *hw = &pf->hw; 4514 int err; 4515 4516 err = ice_init_hw(hw); 4517 if (err) { 4518 dev_err(dev, "ice_init_hw failed: %d\n", err); 4519 return err; 4520 } 4521 4522 ice_init_feature_support(pf); 4523 4524 ice_request_fw(pf); 4525 4526 /* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be 4527 * set in pf->state, which will cause ice_is_safe_mode to return 4528 * true 4529 */ 4530 if (ice_is_safe_mode(pf)) { 4531 /* we already got function/device capabilities but these don't 4532 * reflect what the driver needs to do in safe mode. Instead of 4533 * adding conditional logic everywhere to ignore these 4534 * device/function capabilities, override them. 4535 */ 4536 ice_set_safe_mode_caps(hw); 4537 } 4538 4539 err = ice_init_pf(pf); 4540 if (err) { 4541 dev_err(dev, "ice_init_pf failed: %d\n", err); 4542 goto err_init_pf; 4543 } 4544 4545 pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port; 4546 pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port; 4547 pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 4548 pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared; 4549 if (pf->hw.tnl.valid_count[TNL_VXLAN]) { 4550 pf->hw.udp_tunnel_nic.tables[0].n_entries = 4551 pf->hw.tnl.valid_count[TNL_VXLAN]; 4552 pf->hw.udp_tunnel_nic.tables[0].tunnel_types = 4553 UDP_TUNNEL_TYPE_VXLAN; 4554 } 4555 if (pf->hw.tnl.valid_count[TNL_GENEVE]) { 4556 pf->hw.udp_tunnel_nic.tables[1].n_entries = 4557 pf->hw.tnl.valid_count[TNL_GENEVE]; 4558 pf->hw.udp_tunnel_nic.tables[1].tunnel_types = 4559 UDP_TUNNEL_TYPE_GENEVE; 4560 } 4561 4562 err = ice_init_interrupt_scheme(pf); 4563 if (err) { 4564 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 4565 err = -EIO; 4566 goto err_init_interrupt_scheme; 4567 } 4568 4569 /* In case of MSIX we are going to setup the misc vector right here 4570 * to handle admin queue events etc. In case of legacy and MSI 4571 * the misc functionality and queue processing is combined in 4572 * the same vector and that gets setup at open. 4573 */ 4574 err = ice_req_irq_msix_misc(pf); 4575 if (err) { 4576 dev_err(dev, "setup of misc vector failed: %d\n", err); 4577 goto err_req_irq_msix_misc; 4578 } 4579 4580 return 0; 4581 4582 err_req_irq_msix_misc: 4583 ice_clear_interrupt_scheme(pf); 4584 err_init_interrupt_scheme: 4585 ice_deinit_pf(pf); 4586 err_init_pf: 4587 ice_deinit_hw(hw); 4588 return err; 4589 } 4590 4591 static void ice_deinit_dev(struct ice_pf *pf) 4592 { 4593 ice_free_irq_msix_misc(pf); 4594 ice_deinit_pf(pf); 4595 ice_deinit_hw(&pf->hw); 4596 4597 /* Service task is already stopped, so call reset directly. */ 4598 ice_reset(&pf->hw, ICE_RESET_PFR); 4599 pci_wait_for_pending_transaction(pf->pdev); 4600 ice_clear_interrupt_scheme(pf); 4601 } 4602 4603 static void ice_init_features(struct ice_pf *pf) 4604 { 4605 struct device *dev = ice_pf_to_dev(pf); 4606 4607 if (ice_is_safe_mode(pf)) 4608 return; 4609 4610 /* initialize DDP driven features */ 4611 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 4612 ice_ptp_init(pf); 4613 4614 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 4615 ice_gnss_init(pf); 4616 4617 /* Note: Flow director init failure is non-fatal to load */ 4618 if (ice_init_fdir(pf)) 4619 dev_err(dev, "could not initialize flow director\n"); 4620 4621 /* Note: DCB init failure is non-fatal to load */ 4622 if (ice_init_pf_dcb(pf, false)) { 4623 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 4624 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 4625 } else { 4626 ice_cfg_lldp_mib_change(&pf->hw, true); 4627 } 4628 4629 if (ice_init_lag(pf)) 4630 dev_warn(dev, "Failed to init link aggregation support\n"); 4631 } 4632 4633 static void ice_deinit_features(struct ice_pf *pf) 4634 { 4635 ice_deinit_lag(pf); 4636 if (test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags)) 4637 ice_cfg_lldp_mib_change(&pf->hw, false); 4638 ice_deinit_fdir(pf); 4639 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 4640 ice_gnss_exit(pf); 4641 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 4642 ice_ptp_release(pf); 4643 } 4644 4645 static void ice_init_wakeup(struct ice_pf *pf) 4646 { 4647 /* Save wakeup reason register for later use */ 4648 pf->wakeup_reason = rd32(&pf->hw, PFPM_WUS); 4649 4650 /* check for a power management event */ 4651 ice_print_wake_reason(pf); 4652 4653 /* clear wake status, all bits */ 4654 wr32(&pf->hw, PFPM_WUS, U32_MAX); 4655 4656 /* Disable WoL at init, wait for user to enable */ 4657 device_set_wakeup_enable(ice_pf_to_dev(pf), false); 4658 } 4659 4660 static int ice_init_link(struct ice_pf *pf) 4661 { 4662 struct device *dev = ice_pf_to_dev(pf); 4663 int err; 4664 4665 err = ice_init_link_events(pf->hw.port_info); 4666 if (err) { 4667 dev_err(dev, "ice_init_link_events failed: %d\n", err); 4668 return err; 4669 } 4670 4671 /* not a fatal error if this fails */ 4672 err = ice_init_nvm_phy_type(pf->hw.port_info); 4673 if (err) 4674 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 4675 4676 /* not a fatal error if this fails */ 4677 err = ice_update_link_info(pf->hw.port_info); 4678 if (err) 4679 dev_err(dev, "ice_update_link_info failed: %d\n", err); 4680 4681 ice_init_link_dflt_override(pf->hw.port_info); 4682 4683 ice_check_link_cfg_err(pf, 4684 pf->hw.port_info->phy.link_info.link_cfg_err); 4685 4686 /* if media available, initialize PHY settings */ 4687 if (pf->hw.port_info->phy.link_info.link_info & 4688 ICE_AQ_MEDIA_AVAILABLE) { 4689 /* not a fatal error if this fails */ 4690 err = ice_init_phy_user_cfg(pf->hw.port_info); 4691 if (err) 4692 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 4693 4694 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 4695 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4696 4697 if (vsi) 4698 ice_configure_phy(vsi); 4699 } 4700 } else { 4701 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 4702 } 4703 4704 return err; 4705 } 4706 4707 static int ice_init_pf_sw(struct ice_pf *pf) 4708 { 4709 bool dvm = ice_is_dvm_ena(&pf->hw); 4710 struct ice_vsi *vsi; 4711 int err; 4712 4713 /* create switch struct for the switch element created by FW on boot */ 4714 pf->first_sw = kzalloc(sizeof(*pf->first_sw), GFP_KERNEL); 4715 if (!pf->first_sw) 4716 return -ENOMEM; 4717 4718 if (pf->hw.evb_veb) 4719 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 4720 else 4721 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 4722 4723 pf->first_sw->pf = pf; 4724 4725 /* record the sw_id available for later use */ 4726 pf->first_sw->sw_id = pf->hw.port_info->sw_id; 4727 4728 err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 4729 if (err) 4730 goto err_aq_set_port_params; 4731 4732 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 4733 if (!vsi) { 4734 err = -ENOMEM; 4735 goto err_pf_vsi_setup; 4736 } 4737 4738 return 0; 4739 4740 err_pf_vsi_setup: 4741 err_aq_set_port_params: 4742 kfree(pf->first_sw); 4743 return err; 4744 } 4745 4746 static void ice_deinit_pf_sw(struct ice_pf *pf) 4747 { 4748 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4749 4750 if (!vsi) 4751 return; 4752 4753 ice_vsi_release(vsi); 4754 kfree(pf->first_sw); 4755 } 4756 4757 static int ice_alloc_vsis(struct ice_pf *pf) 4758 { 4759 struct device *dev = ice_pf_to_dev(pf); 4760 4761 pf->num_alloc_vsi = pf->hw.func_caps.guar_num_vsi; 4762 if (!pf->num_alloc_vsi) 4763 return -EIO; 4764 4765 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 4766 dev_warn(dev, 4767 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 4768 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 4769 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 4770 } 4771 4772 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 4773 GFP_KERNEL); 4774 if (!pf->vsi) 4775 return -ENOMEM; 4776 4777 pf->vsi_stats = devm_kcalloc(dev, pf->num_alloc_vsi, 4778 sizeof(*pf->vsi_stats), GFP_KERNEL); 4779 if (!pf->vsi_stats) { 4780 devm_kfree(dev, pf->vsi); 4781 return -ENOMEM; 4782 } 4783 4784 return 0; 4785 } 4786 4787 static void ice_dealloc_vsis(struct ice_pf *pf) 4788 { 4789 devm_kfree(ice_pf_to_dev(pf), pf->vsi_stats); 4790 pf->vsi_stats = NULL; 4791 4792 pf->num_alloc_vsi = 0; 4793 devm_kfree(ice_pf_to_dev(pf), pf->vsi); 4794 pf->vsi = NULL; 4795 } 4796 4797 static int ice_init_devlink(struct ice_pf *pf) 4798 { 4799 int err; 4800 4801 err = ice_devlink_register_params(pf); 4802 if (err) 4803 return err; 4804 4805 ice_devlink_init_regions(pf); 4806 ice_devlink_register(pf); 4807 4808 return 0; 4809 } 4810 4811 static void ice_deinit_devlink(struct ice_pf *pf) 4812 { 4813 ice_devlink_unregister(pf); 4814 ice_devlink_destroy_regions(pf); 4815 ice_devlink_unregister_params(pf); 4816 } 4817 4818 static int ice_init(struct ice_pf *pf) 4819 { 4820 int err; 4821 4822 err = ice_init_dev(pf); 4823 if (err) 4824 return err; 4825 4826 err = ice_alloc_vsis(pf); 4827 if (err) 4828 goto err_alloc_vsis; 4829 4830 err = ice_init_pf_sw(pf); 4831 if (err) 4832 goto err_init_pf_sw; 4833 4834 ice_init_wakeup(pf); 4835 4836 err = ice_init_link(pf); 4837 if (err) 4838 goto err_init_link; 4839 4840 err = ice_send_version(pf); 4841 if (err) 4842 goto err_init_link; 4843 4844 ice_verify_cacheline_size(pf); 4845 4846 if (ice_is_safe_mode(pf)) 4847 ice_set_safe_mode_vlan_cfg(pf); 4848 else 4849 /* print PCI link speed and width */ 4850 pcie_print_link_status(pf->pdev); 4851 4852 /* ready to go, so clear down state bit */ 4853 clear_bit(ICE_DOWN, pf->state); 4854 clear_bit(ICE_SERVICE_DIS, pf->state); 4855 4856 /* since everything is good, start the service timer */ 4857 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4858 4859 return 0; 4860 4861 err_init_link: 4862 ice_deinit_pf_sw(pf); 4863 err_init_pf_sw: 4864 ice_dealloc_vsis(pf); 4865 err_alloc_vsis: 4866 ice_deinit_dev(pf); 4867 return err; 4868 } 4869 4870 static void ice_deinit(struct ice_pf *pf) 4871 { 4872 set_bit(ICE_SERVICE_DIS, pf->state); 4873 set_bit(ICE_DOWN, pf->state); 4874 4875 ice_deinit_pf_sw(pf); 4876 ice_dealloc_vsis(pf); 4877 ice_deinit_dev(pf); 4878 } 4879 4880 /** 4881 * ice_load - load pf by init hw and starting VSI 4882 * @pf: pointer to the pf instance 4883 */ 4884 int ice_load(struct ice_pf *pf) 4885 { 4886 struct ice_vsi_cfg_params params = {}; 4887 struct ice_vsi *vsi; 4888 int err; 4889 4890 err = ice_init_dev(pf); 4891 if (err) 4892 return err; 4893 4894 vsi = ice_get_main_vsi(pf); 4895 4896 params = ice_vsi_to_params(vsi); 4897 params.flags = ICE_VSI_FLAG_INIT; 4898 4899 rtnl_lock(); 4900 err = ice_vsi_cfg(vsi, ¶ms); 4901 if (err) 4902 goto err_vsi_cfg; 4903 4904 err = ice_start_eth(ice_get_main_vsi(pf)); 4905 if (err) 4906 goto err_start_eth; 4907 rtnl_unlock(); 4908 4909 err = ice_init_rdma(pf); 4910 if (err) 4911 goto err_init_rdma; 4912 4913 ice_init_features(pf); 4914 ice_service_task_restart(pf); 4915 4916 clear_bit(ICE_DOWN, pf->state); 4917 4918 return 0; 4919 4920 err_init_rdma: 4921 ice_vsi_close(ice_get_main_vsi(pf)); 4922 rtnl_lock(); 4923 err_start_eth: 4924 ice_vsi_decfg(ice_get_main_vsi(pf)); 4925 err_vsi_cfg: 4926 rtnl_unlock(); 4927 ice_deinit_dev(pf); 4928 return err; 4929 } 4930 4931 /** 4932 * ice_unload - unload pf by stopping VSI and deinit hw 4933 * @pf: pointer to the pf instance 4934 */ 4935 void ice_unload(struct ice_pf *pf) 4936 { 4937 ice_deinit_features(pf); 4938 ice_deinit_rdma(pf); 4939 rtnl_lock(); 4940 ice_stop_eth(ice_get_main_vsi(pf)); 4941 ice_vsi_decfg(ice_get_main_vsi(pf)); 4942 rtnl_unlock(); 4943 ice_deinit_dev(pf); 4944 } 4945 4946 /** 4947 * ice_probe - Device initialization routine 4948 * @pdev: PCI device information struct 4949 * @ent: entry in ice_pci_tbl 4950 * 4951 * Returns 0 on success, negative on failure 4952 */ 4953 static int 4954 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 4955 { 4956 struct device *dev = &pdev->dev; 4957 struct ice_pf *pf; 4958 struct ice_hw *hw; 4959 int err; 4960 4961 if (pdev->is_virtfn) { 4962 dev_err(dev, "can't probe a virtual function\n"); 4963 return -EINVAL; 4964 } 4965 4966 /* this driver uses devres, see 4967 * Documentation/driver-api/driver-model/devres.rst 4968 */ 4969 err = pcim_enable_device(pdev); 4970 if (err) 4971 return err; 4972 4973 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev)); 4974 if (err) { 4975 dev_err(dev, "BAR0 I/O map error %d\n", err); 4976 return err; 4977 } 4978 4979 pf = ice_allocate_pf(dev); 4980 if (!pf) 4981 return -ENOMEM; 4982 4983 /* initialize Auxiliary index to invalid value */ 4984 pf->aux_idx = -1; 4985 4986 /* set up for high or low DMA */ 4987 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 4988 if (err) { 4989 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 4990 return err; 4991 } 4992 4993 pci_set_master(pdev); 4994 4995 pf->pdev = pdev; 4996 pci_set_drvdata(pdev, pf); 4997 set_bit(ICE_DOWN, pf->state); 4998 /* Disable service task until DOWN bit is cleared */ 4999 set_bit(ICE_SERVICE_DIS, pf->state); 5000 5001 hw = &pf->hw; 5002 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 5003 pci_save_state(pdev); 5004 5005 hw->back = pf; 5006 hw->port_info = NULL; 5007 hw->vendor_id = pdev->vendor; 5008 hw->device_id = pdev->device; 5009 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 5010 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5011 hw->subsystem_device_id = pdev->subsystem_device; 5012 hw->bus.device = PCI_SLOT(pdev->devfn); 5013 hw->bus.func = PCI_FUNC(pdev->devfn); 5014 ice_set_ctrlq_len(hw); 5015 5016 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 5017 5018 #ifndef CONFIG_DYNAMIC_DEBUG 5019 if (debug < -1) 5020 hw->debug_mask = debug; 5021 #endif 5022 5023 err = ice_init(pf); 5024 if (err) 5025 goto err_init; 5026 5027 err = ice_init_eth(pf); 5028 if (err) 5029 goto err_init_eth; 5030 5031 err = ice_init_rdma(pf); 5032 if (err) 5033 goto err_init_rdma; 5034 5035 err = ice_init_devlink(pf); 5036 if (err) 5037 goto err_init_devlink; 5038 5039 ice_init_features(pf); 5040 5041 return 0; 5042 5043 err_init_devlink: 5044 ice_deinit_rdma(pf); 5045 err_init_rdma: 5046 ice_deinit_eth(pf); 5047 err_init_eth: 5048 ice_deinit(pf); 5049 err_init: 5050 pci_disable_device(pdev); 5051 return err; 5052 } 5053 5054 /** 5055 * ice_set_wake - enable or disable Wake on LAN 5056 * @pf: pointer to the PF struct 5057 * 5058 * Simple helper for WoL control 5059 */ 5060 static void ice_set_wake(struct ice_pf *pf) 5061 { 5062 struct ice_hw *hw = &pf->hw; 5063 bool wol = pf->wol_ena; 5064 5065 /* clear wake state, otherwise new wake events won't fire */ 5066 wr32(hw, PFPM_WUS, U32_MAX); 5067 5068 /* enable / disable APM wake up, no RMW needed */ 5069 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 5070 5071 /* set magic packet filter enabled */ 5072 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 5073 } 5074 5075 /** 5076 * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet 5077 * @pf: pointer to the PF struct 5078 * 5079 * Issue firmware command to enable multicast magic wake, making 5080 * sure that any locally administered address (LAA) is used for 5081 * wake, and that PF reset doesn't undo the LAA. 5082 */ 5083 static void ice_setup_mc_magic_wake(struct ice_pf *pf) 5084 { 5085 struct device *dev = ice_pf_to_dev(pf); 5086 struct ice_hw *hw = &pf->hw; 5087 u8 mac_addr[ETH_ALEN]; 5088 struct ice_vsi *vsi; 5089 int status; 5090 u8 flags; 5091 5092 if (!pf->wol_ena) 5093 return; 5094 5095 vsi = ice_get_main_vsi(pf); 5096 if (!vsi) 5097 return; 5098 5099 /* Get current MAC address in case it's an LAA */ 5100 if (vsi->netdev) 5101 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 5102 else 5103 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 5104 5105 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 5106 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 5107 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 5108 5109 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 5110 if (status) 5111 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n", 5112 status, ice_aq_str(hw->adminq.sq_last_status)); 5113 } 5114 5115 /** 5116 * ice_remove - Device removal routine 5117 * @pdev: PCI device information struct 5118 */ 5119 static void ice_remove(struct pci_dev *pdev) 5120 { 5121 struct ice_pf *pf = pci_get_drvdata(pdev); 5122 int i; 5123 5124 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 5125 if (!ice_is_reset_in_progress(pf->state)) 5126 break; 5127 msleep(100); 5128 } 5129 5130 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 5131 set_bit(ICE_VF_RESETS_DISABLED, pf->state); 5132 ice_free_vfs(pf); 5133 } 5134 5135 ice_service_task_stop(pf); 5136 ice_aq_cancel_waiting_tasks(pf); 5137 set_bit(ICE_DOWN, pf->state); 5138 5139 if (!ice_is_safe_mode(pf)) 5140 ice_remove_arfs(pf); 5141 ice_deinit_features(pf); 5142 ice_deinit_devlink(pf); 5143 ice_deinit_rdma(pf); 5144 ice_deinit_eth(pf); 5145 ice_deinit(pf); 5146 5147 ice_vsi_release_all(pf); 5148 5149 ice_setup_mc_magic_wake(pf); 5150 ice_set_wake(pf); 5151 5152 pci_disable_device(pdev); 5153 } 5154 5155 /** 5156 * ice_shutdown - PCI callback for shutting down device 5157 * @pdev: PCI device information struct 5158 */ 5159 static void ice_shutdown(struct pci_dev *pdev) 5160 { 5161 struct ice_pf *pf = pci_get_drvdata(pdev); 5162 5163 ice_remove(pdev); 5164 5165 if (system_state == SYSTEM_POWER_OFF) { 5166 pci_wake_from_d3(pdev, pf->wol_ena); 5167 pci_set_power_state(pdev, PCI_D3hot); 5168 } 5169 } 5170 5171 #ifdef CONFIG_PM 5172 /** 5173 * ice_prepare_for_shutdown - prep for PCI shutdown 5174 * @pf: board private structure 5175 * 5176 * Inform or close all dependent features in prep for PCI device shutdown 5177 */ 5178 static void ice_prepare_for_shutdown(struct ice_pf *pf) 5179 { 5180 struct ice_hw *hw = &pf->hw; 5181 u32 v; 5182 5183 /* Notify VFs of impending reset */ 5184 if (ice_check_sq_alive(hw, &hw->mailboxq)) 5185 ice_vc_notify_reset(pf); 5186 5187 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 5188 5189 /* disable the VSIs and their queues that are not already DOWN */ 5190 ice_pf_dis_all_vsi(pf, false); 5191 5192 ice_for_each_vsi(pf, v) 5193 if (pf->vsi[v]) 5194 pf->vsi[v]->vsi_num = 0; 5195 5196 ice_shutdown_all_ctrlq(hw); 5197 } 5198 5199 /** 5200 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 5201 * @pf: board private structure to reinitialize 5202 * 5203 * This routine reinitialize interrupt scheme that was cleared during 5204 * power management suspend callback. 5205 * 5206 * This should be called during resume routine to re-allocate the q_vectors 5207 * and reacquire interrupts. 5208 */ 5209 static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 5210 { 5211 struct device *dev = ice_pf_to_dev(pf); 5212 int ret, v; 5213 5214 /* Since we clear MSIX flag during suspend, we need to 5215 * set it back during resume... 5216 */ 5217 5218 ret = ice_init_interrupt_scheme(pf); 5219 if (ret) { 5220 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 5221 return ret; 5222 } 5223 5224 /* Remap vectors and rings, after successful re-init interrupts */ 5225 ice_for_each_vsi(pf, v) { 5226 if (!pf->vsi[v]) 5227 continue; 5228 5229 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 5230 if (ret) 5231 goto err_reinit; 5232 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 5233 } 5234 5235 ret = ice_req_irq_msix_misc(pf); 5236 if (ret) { 5237 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 5238 ret); 5239 goto err_reinit; 5240 } 5241 5242 return 0; 5243 5244 err_reinit: 5245 while (v--) 5246 if (pf->vsi[v]) 5247 ice_vsi_free_q_vectors(pf->vsi[v]); 5248 5249 return ret; 5250 } 5251 5252 /** 5253 * ice_suspend 5254 * @dev: generic device information structure 5255 * 5256 * Power Management callback to quiesce the device and prepare 5257 * for D3 transition. 5258 */ 5259 static int __maybe_unused ice_suspend(struct device *dev) 5260 { 5261 struct pci_dev *pdev = to_pci_dev(dev); 5262 struct ice_pf *pf; 5263 int disabled, v; 5264 5265 pf = pci_get_drvdata(pdev); 5266 5267 if (!ice_pf_state_is_nominal(pf)) { 5268 dev_err(dev, "Device is not ready, no need to suspend it\n"); 5269 return -EBUSY; 5270 } 5271 5272 /* Stop watchdog tasks until resume completion. 5273 * Even though it is most likely that the service task is 5274 * disabled if the device is suspended or down, the service task's 5275 * state is controlled by a different state bit, and we should 5276 * store and honor whatever state that bit is in at this point. 5277 */ 5278 disabled = ice_service_task_stop(pf); 5279 5280 ice_unplug_aux_dev(pf); 5281 5282 /* Already suspended?, then there is nothing to do */ 5283 if (test_and_set_bit(ICE_SUSPENDED, pf->state)) { 5284 if (!disabled) 5285 ice_service_task_restart(pf); 5286 return 0; 5287 } 5288 5289 if (test_bit(ICE_DOWN, pf->state) || 5290 ice_is_reset_in_progress(pf->state)) { 5291 dev_err(dev, "can't suspend device in reset or already down\n"); 5292 if (!disabled) 5293 ice_service_task_restart(pf); 5294 return 0; 5295 } 5296 5297 ice_setup_mc_magic_wake(pf); 5298 5299 ice_prepare_for_shutdown(pf); 5300 5301 ice_set_wake(pf); 5302 5303 /* Free vectors, clear the interrupt scheme and release IRQs 5304 * for proper hibernation, especially with large number of CPUs. 5305 * Otherwise hibernation might fail when mapping all the vectors back 5306 * to CPU0. 5307 */ 5308 ice_free_irq_msix_misc(pf); 5309 ice_for_each_vsi(pf, v) { 5310 if (!pf->vsi[v]) 5311 continue; 5312 ice_vsi_free_q_vectors(pf->vsi[v]); 5313 } 5314 ice_clear_interrupt_scheme(pf); 5315 5316 pci_save_state(pdev); 5317 pci_wake_from_d3(pdev, pf->wol_ena); 5318 pci_set_power_state(pdev, PCI_D3hot); 5319 return 0; 5320 } 5321 5322 /** 5323 * ice_resume - PM callback for waking up from D3 5324 * @dev: generic device information structure 5325 */ 5326 static int __maybe_unused ice_resume(struct device *dev) 5327 { 5328 struct pci_dev *pdev = to_pci_dev(dev); 5329 enum ice_reset_req reset_type; 5330 struct ice_pf *pf; 5331 struct ice_hw *hw; 5332 int ret; 5333 5334 pci_set_power_state(pdev, PCI_D0); 5335 pci_restore_state(pdev); 5336 pci_save_state(pdev); 5337 5338 if (!pci_device_is_present(pdev)) 5339 return -ENODEV; 5340 5341 ret = pci_enable_device_mem(pdev); 5342 if (ret) { 5343 dev_err(dev, "Cannot enable device after suspend\n"); 5344 return ret; 5345 } 5346 5347 pf = pci_get_drvdata(pdev); 5348 hw = &pf->hw; 5349 5350 pf->wakeup_reason = rd32(hw, PFPM_WUS); 5351 ice_print_wake_reason(pf); 5352 5353 /* We cleared the interrupt scheme when we suspended, so we need to 5354 * restore it now to resume device functionality. 5355 */ 5356 ret = ice_reinit_interrupt_scheme(pf); 5357 if (ret) 5358 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 5359 5360 clear_bit(ICE_DOWN, pf->state); 5361 /* Now perform PF reset and rebuild */ 5362 reset_type = ICE_RESET_PFR; 5363 /* re-enable service task for reset, but allow reset to schedule it */ 5364 clear_bit(ICE_SERVICE_DIS, pf->state); 5365 5366 if (ice_schedule_reset(pf, reset_type)) 5367 dev_err(dev, "Reset during resume failed.\n"); 5368 5369 clear_bit(ICE_SUSPENDED, pf->state); 5370 ice_service_task_restart(pf); 5371 5372 /* Restart the service task */ 5373 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5374 5375 return 0; 5376 } 5377 #endif /* CONFIG_PM */ 5378 5379 /** 5380 * ice_pci_err_detected - warning that PCI error has been detected 5381 * @pdev: PCI device information struct 5382 * @err: the type of PCI error 5383 * 5384 * Called to warn that something happened on the PCI bus and the error handling 5385 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 5386 */ 5387 static pci_ers_result_t 5388 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err) 5389 { 5390 struct ice_pf *pf = pci_get_drvdata(pdev); 5391 5392 if (!pf) { 5393 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 5394 __func__, err); 5395 return PCI_ERS_RESULT_DISCONNECT; 5396 } 5397 5398 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5399 ice_service_task_stop(pf); 5400 5401 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5402 set_bit(ICE_PFR_REQ, pf->state); 5403 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5404 } 5405 } 5406 5407 return PCI_ERS_RESULT_NEED_RESET; 5408 } 5409 5410 /** 5411 * ice_pci_err_slot_reset - a PCI slot reset has just happened 5412 * @pdev: PCI device information struct 5413 * 5414 * Called to determine if the driver can recover from the PCI slot reset by 5415 * using a register read to determine if the device is recoverable. 5416 */ 5417 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 5418 { 5419 struct ice_pf *pf = pci_get_drvdata(pdev); 5420 pci_ers_result_t result; 5421 int err; 5422 u32 reg; 5423 5424 err = pci_enable_device_mem(pdev); 5425 if (err) { 5426 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 5427 err); 5428 result = PCI_ERS_RESULT_DISCONNECT; 5429 } else { 5430 pci_set_master(pdev); 5431 pci_restore_state(pdev); 5432 pci_save_state(pdev); 5433 pci_wake_from_d3(pdev, false); 5434 5435 /* Check for life */ 5436 reg = rd32(&pf->hw, GLGEN_RTRIG); 5437 if (!reg) 5438 result = PCI_ERS_RESULT_RECOVERED; 5439 else 5440 result = PCI_ERS_RESULT_DISCONNECT; 5441 } 5442 5443 return result; 5444 } 5445 5446 /** 5447 * ice_pci_err_resume - restart operations after PCI error recovery 5448 * @pdev: PCI device information struct 5449 * 5450 * Called to allow the driver to bring things back up after PCI error and/or 5451 * reset recovery have finished 5452 */ 5453 static void ice_pci_err_resume(struct pci_dev *pdev) 5454 { 5455 struct ice_pf *pf = pci_get_drvdata(pdev); 5456 5457 if (!pf) { 5458 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 5459 __func__); 5460 return; 5461 } 5462 5463 if (test_bit(ICE_SUSPENDED, pf->state)) { 5464 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 5465 __func__); 5466 return; 5467 } 5468 5469 ice_restore_all_vfs_msi_state(pdev); 5470 5471 ice_do_reset(pf, ICE_RESET_PFR); 5472 ice_service_task_restart(pf); 5473 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5474 } 5475 5476 /** 5477 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 5478 * @pdev: PCI device information struct 5479 */ 5480 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 5481 { 5482 struct ice_pf *pf = pci_get_drvdata(pdev); 5483 5484 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5485 ice_service_task_stop(pf); 5486 5487 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5488 set_bit(ICE_PFR_REQ, pf->state); 5489 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5490 } 5491 } 5492 } 5493 5494 /** 5495 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 5496 * @pdev: PCI device information struct 5497 */ 5498 static void ice_pci_err_reset_done(struct pci_dev *pdev) 5499 { 5500 ice_pci_err_resume(pdev); 5501 } 5502 5503 /* ice_pci_tbl - PCI Device ID Table 5504 * 5505 * Wildcard entries (PCI_ANY_ID) should come last 5506 * Last entry must be all 0s 5507 * 5508 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 5509 * Class, Class Mask, private data (not used) } 5510 */ 5511 static const struct pci_device_id ice_pci_tbl[] = { 5512 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 5513 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 5514 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 5515 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE), 0 }, 5516 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP), 0 }, 5517 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 }, 5518 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 }, 5519 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 }, 5520 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 }, 5521 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 }, 5522 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 }, 5523 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 }, 5524 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 }, 5525 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 }, 5526 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 }, 5527 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 }, 5528 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 }, 5529 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 }, 5530 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 }, 5531 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 }, 5532 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 }, 5533 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 }, 5534 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 }, 5535 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 }, 5536 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 }, 5537 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT), 0 }, 5538 /* required last entry */ 5539 { 0, } 5540 }; 5541 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 5542 5543 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 5544 5545 static const struct pci_error_handlers ice_pci_err_handler = { 5546 .error_detected = ice_pci_err_detected, 5547 .slot_reset = ice_pci_err_slot_reset, 5548 .reset_prepare = ice_pci_err_reset_prepare, 5549 .reset_done = ice_pci_err_reset_done, 5550 .resume = ice_pci_err_resume 5551 }; 5552 5553 static struct pci_driver ice_driver = { 5554 .name = KBUILD_MODNAME, 5555 .id_table = ice_pci_tbl, 5556 .probe = ice_probe, 5557 .remove = ice_remove, 5558 #ifdef CONFIG_PM 5559 .driver.pm = &ice_pm_ops, 5560 #endif /* CONFIG_PM */ 5561 .shutdown = ice_shutdown, 5562 .sriov_configure = ice_sriov_configure, 5563 .err_handler = &ice_pci_err_handler 5564 }; 5565 5566 /** 5567 * ice_module_init - Driver registration routine 5568 * 5569 * ice_module_init is the first routine called when the driver is 5570 * loaded. All it does is register with the PCI subsystem. 5571 */ 5572 static int __init ice_module_init(void) 5573 { 5574 int status; 5575 5576 pr_info("%s\n", ice_driver_string); 5577 pr_info("%s\n", ice_copyright); 5578 5579 ice_wq = alloc_workqueue("%s", 0, 0, KBUILD_MODNAME); 5580 if (!ice_wq) { 5581 pr_err("Failed to create workqueue\n"); 5582 return -ENOMEM; 5583 } 5584 5585 status = pci_register_driver(&ice_driver); 5586 if (status) { 5587 pr_err("failed to register PCI driver, err %d\n", status); 5588 destroy_workqueue(ice_wq); 5589 } 5590 5591 return status; 5592 } 5593 module_init(ice_module_init); 5594 5595 /** 5596 * ice_module_exit - Driver exit cleanup routine 5597 * 5598 * ice_module_exit is called just before the driver is removed 5599 * from memory. 5600 */ 5601 static void __exit ice_module_exit(void) 5602 { 5603 pci_unregister_driver(&ice_driver); 5604 destroy_workqueue(ice_wq); 5605 pr_info("module unloaded\n"); 5606 } 5607 module_exit(ice_module_exit); 5608 5609 /** 5610 * ice_set_mac_address - NDO callback to set MAC address 5611 * @netdev: network interface device structure 5612 * @pi: pointer to an address structure 5613 * 5614 * Returns 0 on success, negative on failure 5615 */ 5616 static int ice_set_mac_address(struct net_device *netdev, void *pi) 5617 { 5618 struct ice_netdev_priv *np = netdev_priv(netdev); 5619 struct ice_vsi *vsi = np->vsi; 5620 struct ice_pf *pf = vsi->back; 5621 struct ice_hw *hw = &pf->hw; 5622 struct sockaddr *addr = pi; 5623 u8 old_mac[ETH_ALEN]; 5624 u8 flags = 0; 5625 u8 *mac; 5626 int err; 5627 5628 mac = (u8 *)addr->sa_data; 5629 5630 if (!is_valid_ether_addr(mac)) 5631 return -EADDRNOTAVAIL; 5632 5633 if (test_bit(ICE_DOWN, pf->state) || 5634 ice_is_reset_in_progress(pf->state)) { 5635 netdev_err(netdev, "can't set mac %pM. device not ready\n", 5636 mac); 5637 return -EBUSY; 5638 } 5639 5640 if (ice_chnl_dmac_fltr_cnt(pf)) { 5641 netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n", 5642 mac); 5643 return -EAGAIN; 5644 } 5645 5646 netif_addr_lock_bh(netdev); 5647 ether_addr_copy(old_mac, netdev->dev_addr); 5648 /* change the netdev's MAC address */ 5649 eth_hw_addr_set(netdev, mac); 5650 netif_addr_unlock_bh(netdev); 5651 5652 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 5653 err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI); 5654 if (err && err != -ENOENT) { 5655 err = -EADDRNOTAVAIL; 5656 goto err_update_filters; 5657 } 5658 5659 /* Add filter for new MAC. If filter exists, return success */ 5660 err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 5661 if (err == -EEXIST) { 5662 /* Although this MAC filter is already present in hardware it's 5663 * possible in some cases (e.g. bonding) that dev_addr was 5664 * modified outside of the driver and needs to be restored back 5665 * to this value. 5666 */ 5667 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 5668 5669 return 0; 5670 } else if (err) { 5671 /* error if the new filter addition failed */ 5672 err = -EADDRNOTAVAIL; 5673 } 5674 5675 err_update_filters: 5676 if (err) { 5677 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 5678 mac); 5679 netif_addr_lock_bh(netdev); 5680 eth_hw_addr_set(netdev, old_mac); 5681 netif_addr_unlock_bh(netdev); 5682 return err; 5683 } 5684 5685 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 5686 netdev->dev_addr); 5687 5688 /* write new MAC address to the firmware */ 5689 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 5690 err = ice_aq_manage_mac_write(hw, mac, flags, NULL); 5691 if (err) { 5692 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n", 5693 mac, err); 5694 } 5695 return 0; 5696 } 5697 5698 /** 5699 * ice_set_rx_mode - NDO callback to set the netdev filters 5700 * @netdev: network interface device structure 5701 */ 5702 static void ice_set_rx_mode(struct net_device *netdev) 5703 { 5704 struct ice_netdev_priv *np = netdev_priv(netdev); 5705 struct ice_vsi *vsi = np->vsi; 5706 5707 if (!vsi || ice_is_switchdev_running(vsi->back)) 5708 return; 5709 5710 /* Set the flags to synchronize filters 5711 * ndo_set_rx_mode may be triggered even without a change in netdev 5712 * flags 5713 */ 5714 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 5715 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 5716 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 5717 5718 /* schedule our worker thread which will take care of 5719 * applying the new filter changes 5720 */ 5721 ice_service_task_schedule(vsi->back); 5722 } 5723 5724 /** 5725 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 5726 * @netdev: network interface device structure 5727 * @queue_index: Queue ID 5728 * @maxrate: maximum bandwidth in Mbps 5729 */ 5730 static int 5731 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 5732 { 5733 struct ice_netdev_priv *np = netdev_priv(netdev); 5734 struct ice_vsi *vsi = np->vsi; 5735 u16 q_handle; 5736 int status; 5737 u8 tc; 5738 5739 /* Validate maxrate requested is within permitted range */ 5740 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 5741 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 5742 maxrate, queue_index); 5743 return -EINVAL; 5744 } 5745 5746 q_handle = vsi->tx_rings[queue_index]->q_handle; 5747 tc = ice_dcb_get_tc(vsi, queue_index); 5748 5749 vsi = ice_locate_vsi_using_queue(vsi, queue_index); 5750 if (!vsi) { 5751 netdev_err(netdev, "Invalid VSI for given queue %d\n", 5752 queue_index); 5753 return -EINVAL; 5754 } 5755 5756 /* Set BW back to default, when user set maxrate to 0 */ 5757 if (!maxrate) 5758 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 5759 q_handle, ICE_MAX_BW); 5760 else 5761 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 5762 q_handle, ICE_MAX_BW, maxrate * 1000); 5763 if (status) 5764 netdev_err(netdev, "Unable to set Tx max rate, error %d\n", 5765 status); 5766 5767 return status; 5768 } 5769 5770 /** 5771 * ice_fdb_add - add an entry to the hardware database 5772 * @ndm: the input from the stack 5773 * @tb: pointer to array of nladdr (unused) 5774 * @dev: the net device pointer 5775 * @addr: the MAC address entry being added 5776 * @vid: VLAN ID 5777 * @flags: instructions from stack about fdb operation 5778 * @extack: netlink extended ack 5779 */ 5780 static int 5781 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 5782 struct net_device *dev, const unsigned char *addr, u16 vid, 5783 u16 flags, struct netlink_ext_ack __always_unused *extack) 5784 { 5785 int err; 5786 5787 if (vid) { 5788 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 5789 return -EINVAL; 5790 } 5791 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 5792 netdev_err(dev, "FDB only supports static addresses\n"); 5793 return -EINVAL; 5794 } 5795 5796 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 5797 err = dev_uc_add_excl(dev, addr); 5798 else if (is_multicast_ether_addr(addr)) 5799 err = dev_mc_add_excl(dev, addr); 5800 else 5801 err = -EINVAL; 5802 5803 /* Only return duplicate errors if NLM_F_EXCL is set */ 5804 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 5805 err = 0; 5806 5807 return err; 5808 } 5809 5810 /** 5811 * ice_fdb_del - delete an entry from the hardware database 5812 * @ndm: the input from the stack 5813 * @tb: pointer to array of nladdr (unused) 5814 * @dev: the net device pointer 5815 * @addr: the MAC address entry being added 5816 * @vid: VLAN ID 5817 * @extack: netlink extended ack 5818 */ 5819 static int 5820 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 5821 struct net_device *dev, const unsigned char *addr, 5822 __always_unused u16 vid, struct netlink_ext_ack *extack) 5823 { 5824 int err; 5825 5826 if (ndm->ndm_state & NUD_PERMANENT) { 5827 netdev_err(dev, "FDB only supports static addresses\n"); 5828 return -EINVAL; 5829 } 5830 5831 if (is_unicast_ether_addr(addr)) 5832 err = dev_uc_del(dev, addr); 5833 else if (is_multicast_ether_addr(addr)) 5834 err = dev_mc_del(dev, addr); 5835 else 5836 err = -EINVAL; 5837 5838 return err; 5839 } 5840 5841 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 5842 NETIF_F_HW_VLAN_CTAG_TX | \ 5843 NETIF_F_HW_VLAN_STAG_RX | \ 5844 NETIF_F_HW_VLAN_STAG_TX) 5845 5846 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 5847 NETIF_F_HW_VLAN_STAG_RX) 5848 5849 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ 5850 NETIF_F_HW_VLAN_STAG_FILTER) 5851 5852 /** 5853 * ice_fix_features - fix the netdev features flags based on device limitations 5854 * @netdev: ptr to the netdev that flags are being fixed on 5855 * @features: features that need to be checked and possibly fixed 5856 * 5857 * Make sure any fixups are made to features in this callback. This enables the 5858 * driver to not have to check unsupported configurations throughout the driver 5859 * because that's the responsiblity of this callback. 5860 * 5861 * Single VLAN Mode (SVM) Supported Features: 5862 * NETIF_F_HW_VLAN_CTAG_FILTER 5863 * NETIF_F_HW_VLAN_CTAG_RX 5864 * NETIF_F_HW_VLAN_CTAG_TX 5865 * 5866 * Double VLAN Mode (DVM) Supported Features: 5867 * NETIF_F_HW_VLAN_CTAG_FILTER 5868 * NETIF_F_HW_VLAN_CTAG_RX 5869 * NETIF_F_HW_VLAN_CTAG_TX 5870 * 5871 * NETIF_F_HW_VLAN_STAG_FILTER 5872 * NETIF_HW_VLAN_STAG_RX 5873 * NETIF_HW_VLAN_STAG_TX 5874 * 5875 * Features that need fixing: 5876 * Cannot simultaneously enable CTAG and STAG stripping and/or insertion. 5877 * These are mutually exlusive as the VSI context cannot support multiple 5878 * VLAN ethertypes simultaneously for stripping and/or insertion. If this 5879 * is not done, then default to clearing the requested STAG offload 5880 * settings. 5881 * 5882 * All supported filtering has to be enabled or disabled together. For 5883 * example, in DVM, CTAG and STAG filtering have to be enabled and disabled 5884 * together. If this is not done, then default to VLAN filtering disabled. 5885 * These are mutually exclusive as there is currently no way to 5886 * enable/disable VLAN filtering based on VLAN ethertype when using VLAN 5887 * prune rules. 5888 */ 5889 static netdev_features_t 5890 ice_fix_features(struct net_device *netdev, netdev_features_t features) 5891 { 5892 struct ice_netdev_priv *np = netdev_priv(netdev); 5893 netdev_features_t req_vlan_fltr, cur_vlan_fltr; 5894 bool cur_ctag, cur_stag, req_ctag, req_stag; 5895 5896 cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES; 5897 cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 5898 cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 5899 5900 req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES; 5901 req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 5902 req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 5903 5904 if (req_vlan_fltr != cur_vlan_fltr) { 5905 if (ice_is_dvm_ena(&np->vsi->back->hw)) { 5906 if (req_ctag && req_stag) { 5907 features |= NETIF_VLAN_FILTERING_FEATURES; 5908 } else if (!req_ctag && !req_stag) { 5909 features &= ~NETIF_VLAN_FILTERING_FEATURES; 5910 } else if ((!cur_ctag && req_ctag && !cur_stag) || 5911 (!cur_stag && req_stag && !cur_ctag)) { 5912 features |= NETIF_VLAN_FILTERING_FEATURES; 5913 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n"); 5914 } else if ((cur_ctag && !req_ctag && cur_stag) || 5915 (cur_stag && !req_stag && cur_ctag)) { 5916 features &= ~NETIF_VLAN_FILTERING_FEATURES; 5917 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n"); 5918 } 5919 } else { 5920 if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER) 5921 netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n"); 5922 5923 if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER) 5924 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5925 } 5926 } 5927 5928 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 5929 (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) { 5930 netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 5931 features &= ~(NETIF_F_HW_VLAN_STAG_RX | 5932 NETIF_F_HW_VLAN_STAG_TX); 5933 } 5934 5935 if (!(netdev->features & NETIF_F_RXFCS) && 5936 (features & NETIF_F_RXFCS) && 5937 (features & NETIF_VLAN_STRIPPING_FEATURES) && 5938 !ice_vsi_has_non_zero_vlans(np->vsi)) { 5939 netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n"); 5940 features &= ~NETIF_VLAN_STRIPPING_FEATURES; 5941 } 5942 5943 return features; 5944 } 5945 5946 /** 5947 * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI 5948 * @vsi: PF's VSI 5949 * @features: features used to determine VLAN offload settings 5950 * 5951 * First, determine the vlan_ethertype based on the VLAN offload bits in 5952 * features. Then determine if stripping and insertion should be enabled or 5953 * disabled. Finally enable or disable VLAN stripping and insertion. 5954 */ 5955 static int 5956 ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features) 5957 { 5958 bool enable_stripping = true, enable_insertion = true; 5959 struct ice_vsi_vlan_ops *vlan_ops; 5960 int strip_err = 0, insert_err = 0; 5961 u16 vlan_ethertype = 0; 5962 5963 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 5964 5965 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 5966 vlan_ethertype = ETH_P_8021AD; 5967 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 5968 vlan_ethertype = ETH_P_8021Q; 5969 5970 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 5971 enable_stripping = false; 5972 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 5973 enable_insertion = false; 5974 5975 if (enable_stripping) 5976 strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype); 5977 else 5978 strip_err = vlan_ops->dis_stripping(vsi); 5979 5980 if (enable_insertion) 5981 insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype); 5982 else 5983 insert_err = vlan_ops->dis_insertion(vsi); 5984 5985 if (strip_err || insert_err) 5986 return -EIO; 5987 5988 return 0; 5989 } 5990 5991 /** 5992 * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI 5993 * @vsi: PF's VSI 5994 * @features: features used to determine VLAN filtering settings 5995 * 5996 * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the 5997 * features. 5998 */ 5999 static int 6000 ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features) 6001 { 6002 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 6003 int err = 0; 6004 6005 /* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking 6006 * if either bit is set 6007 */ 6008 if (features & 6009 (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER)) 6010 err = vlan_ops->ena_rx_filtering(vsi); 6011 else 6012 err = vlan_ops->dis_rx_filtering(vsi); 6013 6014 return err; 6015 } 6016 6017 /** 6018 * ice_set_vlan_features - set VLAN settings based on suggested feature set 6019 * @netdev: ptr to the netdev being adjusted 6020 * @features: the feature set that the stack is suggesting 6021 * 6022 * Only update VLAN settings if the requested_vlan_features are different than 6023 * the current_vlan_features. 6024 */ 6025 static int 6026 ice_set_vlan_features(struct net_device *netdev, netdev_features_t features) 6027 { 6028 netdev_features_t current_vlan_features, requested_vlan_features; 6029 struct ice_netdev_priv *np = netdev_priv(netdev); 6030 struct ice_vsi *vsi = np->vsi; 6031 int err; 6032 6033 current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES; 6034 requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES; 6035 if (current_vlan_features ^ requested_vlan_features) { 6036 if ((features & NETIF_F_RXFCS) && 6037 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 6038 dev_err(ice_pf_to_dev(vsi->back), 6039 "To enable VLAN stripping, you must first enable FCS/CRC stripping\n"); 6040 return -EIO; 6041 } 6042 6043 err = ice_set_vlan_offload_features(vsi, features); 6044 if (err) 6045 return err; 6046 } 6047 6048 current_vlan_features = netdev->features & 6049 NETIF_VLAN_FILTERING_FEATURES; 6050 requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES; 6051 if (current_vlan_features ^ requested_vlan_features) { 6052 err = ice_set_vlan_filtering_features(vsi, features); 6053 if (err) 6054 return err; 6055 } 6056 6057 return 0; 6058 } 6059 6060 /** 6061 * ice_set_loopback - turn on/off loopback mode on underlying PF 6062 * @vsi: ptr to VSI 6063 * @ena: flag to indicate the on/off setting 6064 */ 6065 static int ice_set_loopback(struct ice_vsi *vsi, bool ena) 6066 { 6067 bool if_running = netif_running(vsi->netdev); 6068 int ret; 6069 6070 if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 6071 ret = ice_down(vsi); 6072 if (ret) { 6073 netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n"); 6074 return ret; 6075 } 6076 } 6077 ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 6078 if (ret) 6079 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 6080 if (if_running) 6081 ret = ice_up(vsi); 6082 6083 return ret; 6084 } 6085 6086 /** 6087 * ice_set_features - set the netdev feature flags 6088 * @netdev: ptr to the netdev being adjusted 6089 * @features: the feature set that the stack is suggesting 6090 */ 6091 static int 6092 ice_set_features(struct net_device *netdev, netdev_features_t features) 6093 { 6094 netdev_features_t changed = netdev->features ^ features; 6095 struct ice_netdev_priv *np = netdev_priv(netdev); 6096 struct ice_vsi *vsi = np->vsi; 6097 struct ice_pf *pf = vsi->back; 6098 int ret = 0; 6099 6100 /* Don't set any netdev advanced features with device in Safe Mode */ 6101 if (ice_is_safe_mode(pf)) { 6102 dev_err(ice_pf_to_dev(pf), 6103 "Device is in Safe Mode - not enabling advanced netdev features\n"); 6104 return ret; 6105 } 6106 6107 /* Do not change setting during reset */ 6108 if (ice_is_reset_in_progress(pf->state)) { 6109 dev_err(ice_pf_to_dev(pf), 6110 "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 6111 return -EBUSY; 6112 } 6113 6114 /* Multiple features can be changed in one call so keep features in 6115 * separate if/else statements to guarantee each feature is checked 6116 */ 6117 if (changed & NETIF_F_RXHASH) 6118 ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH)); 6119 6120 ret = ice_set_vlan_features(netdev, features); 6121 if (ret) 6122 return ret; 6123 6124 /* Turn on receive of FCS aka CRC, and after setting this 6125 * flag the packet data will have the 4 byte CRC appended 6126 */ 6127 if (changed & NETIF_F_RXFCS) { 6128 if ((features & NETIF_F_RXFCS) && 6129 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 6130 dev_err(ice_pf_to_dev(vsi->back), 6131 "To disable FCS/CRC stripping, you must first disable VLAN stripping\n"); 6132 return -EIO; 6133 } 6134 6135 ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS)); 6136 ret = ice_down_up(vsi); 6137 if (ret) 6138 return ret; 6139 } 6140 6141 if (changed & NETIF_F_NTUPLE) { 6142 bool ena = !!(features & NETIF_F_NTUPLE); 6143 6144 ice_vsi_manage_fdir(vsi, ena); 6145 ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi); 6146 } 6147 6148 /* don't turn off hw_tc_offload when ADQ is already enabled */ 6149 if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) { 6150 dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n"); 6151 return -EACCES; 6152 } 6153 6154 if (changed & NETIF_F_HW_TC) { 6155 bool ena = !!(features & NETIF_F_HW_TC); 6156 6157 ena ? set_bit(ICE_FLAG_CLS_FLOWER, pf->flags) : 6158 clear_bit(ICE_FLAG_CLS_FLOWER, pf->flags); 6159 } 6160 6161 if (changed & NETIF_F_LOOPBACK) 6162 ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 6163 6164 return ret; 6165 } 6166 6167 /** 6168 * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI 6169 * @vsi: VSI to setup VLAN properties for 6170 */ 6171 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 6172 { 6173 int err; 6174 6175 err = ice_set_vlan_offload_features(vsi, vsi->netdev->features); 6176 if (err) 6177 return err; 6178 6179 err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features); 6180 if (err) 6181 return err; 6182 6183 return ice_vsi_add_vlan_zero(vsi); 6184 } 6185 6186 /** 6187 * ice_vsi_cfg_lan - Setup the VSI lan related config 6188 * @vsi: the VSI being configured 6189 * 6190 * Return 0 on success and negative value on error 6191 */ 6192 int ice_vsi_cfg_lan(struct ice_vsi *vsi) 6193 { 6194 int err; 6195 6196 if (vsi->netdev && vsi->type == ICE_VSI_PF) { 6197 ice_set_rx_mode(vsi->netdev); 6198 6199 err = ice_vsi_vlan_setup(vsi); 6200 if (err) 6201 return err; 6202 } 6203 ice_vsi_cfg_dcb_rings(vsi); 6204 6205 err = ice_vsi_cfg_lan_txqs(vsi); 6206 if (!err && ice_is_xdp_ena_vsi(vsi)) 6207 err = ice_vsi_cfg_xdp_txqs(vsi); 6208 if (!err) 6209 err = ice_vsi_cfg_rxqs(vsi); 6210 6211 return err; 6212 } 6213 6214 /* THEORY OF MODERATION: 6215 * The ice driver hardware works differently than the hardware that DIMLIB was 6216 * originally made for. ice hardware doesn't have packet count limits that 6217 * can trigger an interrupt, but it *does* have interrupt rate limit support, 6218 * which is hard-coded to a limit of 250,000 ints/second. 6219 * If not using dynamic moderation, the INTRL value can be modified 6220 * by ethtool rx-usecs-high. 6221 */ 6222 struct ice_dim { 6223 /* the throttle rate for interrupts, basically worst case delay before 6224 * an initial interrupt fires, value is stored in microseconds. 6225 */ 6226 u16 itr; 6227 }; 6228 6229 /* Make a different profile for Rx that doesn't allow quite so aggressive 6230 * moderation at the high end (it maxes out at 126us or about 8k interrupts a 6231 * second. 6232 */ 6233 static const struct ice_dim rx_profile[] = { 6234 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6235 {8}, /* 125,000 ints/s */ 6236 {16}, /* 62,500 ints/s */ 6237 {62}, /* 16,129 ints/s */ 6238 {126} /* 7,936 ints/s */ 6239 }; 6240 6241 /* The transmit profile, which has the same sorts of values 6242 * as the previous struct 6243 */ 6244 static const struct ice_dim tx_profile[] = { 6245 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6246 {8}, /* 125,000 ints/s */ 6247 {40}, /* 16,125 ints/s */ 6248 {128}, /* 7,812 ints/s */ 6249 {256} /* 3,906 ints/s */ 6250 }; 6251 6252 static void ice_tx_dim_work(struct work_struct *work) 6253 { 6254 struct ice_ring_container *rc; 6255 struct dim *dim; 6256 u16 itr; 6257 6258 dim = container_of(work, struct dim, work); 6259 rc = dim->priv; 6260 6261 WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile)); 6262 6263 /* look up the values in our local table */ 6264 itr = tx_profile[dim->profile_ix].itr; 6265 6266 ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim); 6267 ice_write_itr(rc, itr); 6268 6269 dim->state = DIM_START_MEASURE; 6270 } 6271 6272 static void ice_rx_dim_work(struct work_struct *work) 6273 { 6274 struct ice_ring_container *rc; 6275 struct dim *dim; 6276 u16 itr; 6277 6278 dim = container_of(work, struct dim, work); 6279 rc = dim->priv; 6280 6281 WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile)); 6282 6283 /* look up the values in our local table */ 6284 itr = rx_profile[dim->profile_ix].itr; 6285 6286 ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim); 6287 ice_write_itr(rc, itr); 6288 6289 dim->state = DIM_START_MEASURE; 6290 } 6291 6292 #define ICE_DIM_DEFAULT_PROFILE_IX 1 6293 6294 /** 6295 * ice_init_moderation - set up interrupt moderation 6296 * @q_vector: the vector containing rings to be configured 6297 * 6298 * Set up interrupt moderation registers, with the intent to do the right thing 6299 * when called from reset or from probe, and whether or not dynamic moderation 6300 * is enabled or not. Take special care to write all the registers in both 6301 * dynamic moderation mode or not in order to make sure hardware is in a known 6302 * state. 6303 */ 6304 static void ice_init_moderation(struct ice_q_vector *q_vector) 6305 { 6306 struct ice_ring_container *rc; 6307 bool tx_dynamic, rx_dynamic; 6308 6309 rc = &q_vector->tx; 6310 INIT_WORK(&rc->dim.work, ice_tx_dim_work); 6311 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6312 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6313 rc->dim.priv = rc; 6314 tx_dynamic = ITR_IS_DYNAMIC(rc); 6315 6316 /* set the initial TX ITR to match the above */ 6317 ice_write_itr(rc, tx_dynamic ? 6318 tx_profile[rc->dim.profile_ix].itr : rc->itr_setting); 6319 6320 rc = &q_vector->rx; 6321 INIT_WORK(&rc->dim.work, ice_rx_dim_work); 6322 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6323 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6324 rc->dim.priv = rc; 6325 rx_dynamic = ITR_IS_DYNAMIC(rc); 6326 6327 /* set the initial RX ITR to match the above */ 6328 ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr : 6329 rc->itr_setting); 6330 6331 ice_set_q_vector_intrl(q_vector); 6332 } 6333 6334 /** 6335 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 6336 * @vsi: the VSI being configured 6337 */ 6338 static void ice_napi_enable_all(struct ice_vsi *vsi) 6339 { 6340 int q_idx; 6341 6342 if (!vsi->netdev) 6343 return; 6344 6345 ice_for_each_q_vector(vsi, q_idx) { 6346 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 6347 6348 ice_init_moderation(q_vector); 6349 6350 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 6351 napi_enable(&q_vector->napi); 6352 } 6353 } 6354 6355 /** 6356 * ice_up_complete - Finish the last steps of bringing up a connection 6357 * @vsi: The VSI being configured 6358 * 6359 * Return 0 on success and negative value on error 6360 */ 6361 static int ice_up_complete(struct ice_vsi *vsi) 6362 { 6363 struct ice_pf *pf = vsi->back; 6364 int err; 6365 6366 ice_vsi_cfg_msix(vsi); 6367 6368 /* Enable only Rx rings, Tx rings were enabled by the FW when the 6369 * Tx queue group list was configured and the context bits were 6370 * programmed using ice_vsi_cfg_txqs 6371 */ 6372 err = ice_vsi_start_all_rx_rings(vsi); 6373 if (err) 6374 return err; 6375 6376 clear_bit(ICE_VSI_DOWN, vsi->state); 6377 ice_napi_enable_all(vsi); 6378 ice_vsi_ena_irq(vsi); 6379 6380 if (vsi->port_info && 6381 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 6382 vsi->netdev && vsi->type == ICE_VSI_PF) { 6383 ice_print_link_msg(vsi, true); 6384 netif_tx_start_all_queues(vsi->netdev); 6385 netif_carrier_on(vsi->netdev); 6386 ice_ptp_link_change(pf, pf->hw.pf_id, true); 6387 } 6388 6389 /* Perform an initial read of the statistics registers now to 6390 * set the baseline so counters are ready when interface is up 6391 */ 6392 ice_update_eth_stats(vsi); 6393 6394 if (vsi->type == ICE_VSI_PF) 6395 ice_service_task_schedule(pf); 6396 6397 return 0; 6398 } 6399 6400 /** 6401 * ice_up - Bring the connection back up after being down 6402 * @vsi: VSI being configured 6403 */ 6404 int ice_up(struct ice_vsi *vsi) 6405 { 6406 int err; 6407 6408 err = ice_vsi_cfg_lan(vsi); 6409 if (!err) 6410 err = ice_up_complete(vsi); 6411 6412 return err; 6413 } 6414 6415 /** 6416 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 6417 * @syncp: pointer to u64_stats_sync 6418 * @stats: stats that pkts and bytes count will be taken from 6419 * @pkts: packets stats counter 6420 * @bytes: bytes stats counter 6421 * 6422 * This function fetches stats from the ring considering the atomic operations 6423 * that needs to be performed to read u64 values in 32 bit machine. 6424 */ 6425 void 6426 ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, 6427 struct ice_q_stats stats, u64 *pkts, u64 *bytes) 6428 { 6429 unsigned int start; 6430 6431 do { 6432 start = u64_stats_fetch_begin(syncp); 6433 *pkts = stats.pkts; 6434 *bytes = stats.bytes; 6435 } while (u64_stats_fetch_retry(syncp, start)); 6436 } 6437 6438 /** 6439 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 6440 * @vsi: the VSI to be updated 6441 * @vsi_stats: the stats struct to be updated 6442 * @rings: rings to work on 6443 * @count: number of rings 6444 */ 6445 static void 6446 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, 6447 struct rtnl_link_stats64 *vsi_stats, 6448 struct ice_tx_ring **rings, u16 count) 6449 { 6450 u16 i; 6451 6452 for (i = 0; i < count; i++) { 6453 struct ice_tx_ring *ring; 6454 u64 pkts = 0, bytes = 0; 6455 6456 ring = READ_ONCE(rings[i]); 6457 if (!ring || !ring->ring_stats) 6458 continue; 6459 ice_fetch_u64_stats_per_ring(&ring->ring_stats->syncp, 6460 ring->ring_stats->stats, &pkts, 6461 &bytes); 6462 vsi_stats->tx_packets += pkts; 6463 vsi_stats->tx_bytes += bytes; 6464 vsi->tx_restart += ring->ring_stats->tx_stats.restart_q; 6465 vsi->tx_busy += ring->ring_stats->tx_stats.tx_busy; 6466 vsi->tx_linearize += ring->ring_stats->tx_stats.tx_linearize; 6467 } 6468 } 6469 6470 /** 6471 * ice_update_vsi_ring_stats - Update VSI stats counters 6472 * @vsi: the VSI to be updated 6473 */ 6474 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 6475 { 6476 struct rtnl_link_stats64 *net_stats, *stats_prev; 6477 struct rtnl_link_stats64 *vsi_stats; 6478 u64 pkts, bytes; 6479 int i; 6480 6481 vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC); 6482 if (!vsi_stats) 6483 return; 6484 6485 /* reset non-netdev (extended) stats */ 6486 vsi->tx_restart = 0; 6487 vsi->tx_busy = 0; 6488 vsi->tx_linearize = 0; 6489 vsi->rx_buf_failed = 0; 6490 vsi->rx_page_failed = 0; 6491 6492 rcu_read_lock(); 6493 6494 /* update Tx rings counters */ 6495 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings, 6496 vsi->num_txq); 6497 6498 /* update Rx rings counters */ 6499 ice_for_each_rxq(vsi, i) { 6500 struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]); 6501 struct ice_ring_stats *ring_stats; 6502 6503 ring_stats = ring->ring_stats; 6504 ice_fetch_u64_stats_per_ring(&ring_stats->syncp, 6505 ring_stats->stats, &pkts, 6506 &bytes); 6507 vsi_stats->rx_packets += pkts; 6508 vsi_stats->rx_bytes += bytes; 6509 vsi->rx_buf_failed += ring_stats->rx_stats.alloc_buf_failed; 6510 vsi->rx_page_failed += ring_stats->rx_stats.alloc_page_failed; 6511 } 6512 6513 /* update XDP Tx rings counters */ 6514 if (ice_is_xdp_ena_vsi(vsi)) 6515 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings, 6516 vsi->num_xdp_txq); 6517 6518 rcu_read_unlock(); 6519 6520 net_stats = &vsi->net_stats; 6521 stats_prev = &vsi->net_stats_prev; 6522 6523 /* clear prev counters after reset */ 6524 if (vsi_stats->tx_packets < stats_prev->tx_packets || 6525 vsi_stats->rx_packets < stats_prev->rx_packets) { 6526 stats_prev->tx_packets = 0; 6527 stats_prev->tx_bytes = 0; 6528 stats_prev->rx_packets = 0; 6529 stats_prev->rx_bytes = 0; 6530 } 6531 6532 /* update netdev counters */ 6533 net_stats->tx_packets += vsi_stats->tx_packets - stats_prev->tx_packets; 6534 net_stats->tx_bytes += vsi_stats->tx_bytes - stats_prev->tx_bytes; 6535 net_stats->rx_packets += vsi_stats->rx_packets - stats_prev->rx_packets; 6536 net_stats->rx_bytes += vsi_stats->rx_bytes - stats_prev->rx_bytes; 6537 6538 stats_prev->tx_packets = vsi_stats->tx_packets; 6539 stats_prev->tx_bytes = vsi_stats->tx_bytes; 6540 stats_prev->rx_packets = vsi_stats->rx_packets; 6541 stats_prev->rx_bytes = vsi_stats->rx_bytes; 6542 6543 kfree(vsi_stats); 6544 } 6545 6546 /** 6547 * ice_update_vsi_stats - Update VSI stats counters 6548 * @vsi: the VSI to be updated 6549 */ 6550 void ice_update_vsi_stats(struct ice_vsi *vsi) 6551 { 6552 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 6553 struct ice_eth_stats *cur_es = &vsi->eth_stats; 6554 struct ice_pf *pf = vsi->back; 6555 6556 if (test_bit(ICE_VSI_DOWN, vsi->state) || 6557 test_bit(ICE_CFG_BUSY, pf->state)) 6558 return; 6559 6560 /* get stats as recorded by Tx/Rx rings */ 6561 ice_update_vsi_ring_stats(vsi); 6562 6563 /* get VSI stats as recorded by the hardware */ 6564 ice_update_eth_stats(vsi); 6565 6566 cur_ns->tx_errors = cur_es->tx_errors; 6567 cur_ns->rx_dropped = cur_es->rx_discards; 6568 cur_ns->tx_dropped = cur_es->tx_discards; 6569 cur_ns->multicast = cur_es->rx_multicast; 6570 6571 /* update some more netdev stats if this is main VSI */ 6572 if (vsi->type == ICE_VSI_PF) { 6573 cur_ns->rx_crc_errors = pf->stats.crc_errors; 6574 cur_ns->rx_errors = pf->stats.crc_errors + 6575 pf->stats.illegal_bytes + 6576 pf->stats.rx_len_errors + 6577 pf->stats.rx_undersize + 6578 pf->hw_csum_rx_error + 6579 pf->stats.rx_jabber + 6580 pf->stats.rx_fragments + 6581 pf->stats.rx_oversize; 6582 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 6583 /* record drops from the port level */ 6584 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 6585 } 6586 } 6587 6588 /** 6589 * ice_update_pf_stats - Update PF port stats counters 6590 * @pf: PF whose stats needs to be updated 6591 */ 6592 void ice_update_pf_stats(struct ice_pf *pf) 6593 { 6594 struct ice_hw_port_stats *prev_ps, *cur_ps; 6595 struct ice_hw *hw = &pf->hw; 6596 u16 fd_ctr_base; 6597 u8 port; 6598 6599 port = hw->port_info->lport; 6600 prev_ps = &pf->stats_prev; 6601 cur_ps = &pf->stats; 6602 6603 if (ice_is_reset_in_progress(pf->state)) 6604 pf->stat_prev_loaded = false; 6605 6606 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 6607 &prev_ps->eth.rx_bytes, 6608 &cur_ps->eth.rx_bytes); 6609 6610 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 6611 &prev_ps->eth.rx_unicast, 6612 &cur_ps->eth.rx_unicast); 6613 6614 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 6615 &prev_ps->eth.rx_multicast, 6616 &cur_ps->eth.rx_multicast); 6617 6618 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 6619 &prev_ps->eth.rx_broadcast, 6620 &cur_ps->eth.rx_broadcast); 6621 6622 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 6623 &prev_ps->eth.rx_discards, 6624 &cur_ps->eth.rx_discards); 6625 6626 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 6627 &prev_ps->eth.tx_bytes, 6628 &cur_ps->eth.tx_bytes); 6629 6630 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 6631 &prev_ps->eth.tx_unicast, 6632 &cur_ps->eth.tx_unicast); 6633 6634 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 6635 &prev_ps->eth.tx_multicast, 6636 &cur_ps->eth.tx_multicast); 6637 6638 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 6639 &prev_ps->eth.tx_broadcast, 6640 &cur_ps->eth.tx_broadcast); 6641 6642 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 6643 &prev_ps->tx_dropped_link_down, 6644 &cur_ps->tx_dropped_link_down); 6645 6646 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 6647 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 6648 6649 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 6650 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 6651 6652 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 6653 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 6654 6655 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 6656 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 6657 6658 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 6659 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 6660 6661 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 6662 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 6663 6664 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 6665 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 6666 6667 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 6668 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 6669 6670 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 6671 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 6672 6673 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 6674 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 6675 6676 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 6677 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 6678 6679 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 6680 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 6681 6682 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 6683 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 6684 6685 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 6686 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 6687 6688 fd_ctr_base = hw->fd_ctr_base; 6689 6690 ice_stat_update40(hw, 6691 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 6692 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 6693 &cur_ps->fd_sb_match); 6694 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 6695 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 6696 6697 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 6698 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 6699 6700 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 6701 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 6702 6703 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 6704 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 6705 6706 ice_update_dcb_stats(pf); 6707 6708 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 6709 &prev_ps->crc_errors, &cur_ps->crc_errors); 6710 6711 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 6712 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 6713 6714 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 6715 &prev_ps->mac_local_faults, 6716 &cur_ps->mac_local_faults); 6717 6718 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 6719 &prev_ps->mac_remote_faults, 6720 &cur_ps->mac_remote_faults); 6721 6722 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 6723 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 6724 6725 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 6726 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 6727 6728 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 6729 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 6730 6731 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 6732 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 6733 6734 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 6735 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 6736 6737 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 6738 6739 pf->stat_prev_loaded = true; 6740 } 6741 6742 /** 6743 * ice_get_stats64 - get statistics for network device structure 6744 * @netdev: network interface device structure 6745 * @stats: main device statistics structure 6746 */ 6747 static 6748 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 6749 { 6750 struct ice_netdev_priv *np = netdev_priv(netdev); 6751 struct rtnl_link_stats64 *vsi_stats; 6752 struct ice_vsi *vsi = np->vsi; 6753 6754 vsi_stats = &vsi->net_stats; 6755 6756 if (!vsi->num_txq || !vsi->num_rxq) 6757 return; 6758 6759 /* netdev packet/byte stats come from ring counter. These are obtained 6760 * by summing up ring counters (done by ice_update_vsi_ring_stats). 6761 * But, only call the update routine and read the registers if VSI is 6762 * not down. 6763 */ 6764 if (!test_bit(ICE_VSI_DOWN, vsi->state)) 6765 ice_update_vsi_ring_stats(vsi); 6766 stats->tx_packets = vsi_stats->tx_packets; 6767 stats->tx_bytes = vsi_stats->tx_bytes; 6768 stats->rx_packets = vsi_stats->rx_packets; 6769 stats->rx_bytes = vsi_stats->rx_bytes; 6770 6771 /* The rest of the stats can be read from the hardware but instead we 6772 * just return values that the watchdog task has already obtained from 6773 * the hardware. 6774 */ 6775 stats->multicast = vsi_stats->multicast; 6776 stats->tx_errors = vsi_stats->tx_errors; 6777 stats->tx_dropped = vsi_stats->tx_dropped; 6778 stats->rx_errors = vsi_stats->rx_errors; 6779 stats->rx_dropped = vsi_stats->rx_dropped; 6780 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 6781 stats->rx_length_errors = vsi_stats->rx_length_errors; 6782 } 6783 6784 /** 6785 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 6786 * @vsi: VSI having NAPI disabled 6787 */ 6788 static void ice_napi_disable_all(struct ice_vsi *vsi) 6789 { 6790 int q_idx; 6791 6792 if (!vsi->netdev) 6793 return; 6794 6795 ice_for_each_q_vector(vsi, q_idx) { 6796 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 6797 6798 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 6799 napi_disable(&q_vector->napi); 6800 6801 cancel_work_sync(&q_vector->tx.dim.work); 6802 cancel_work_sync(&q_vector->rx.dim.work); 6803 } 6804 } 6805 6806 /** 6807 * ice_down - Shutdown the connection 6808 * @vsi: The VSI being stopped 6809 * 6810 * Caller of this function is expected to set the vsi->state ICE_DOWN bit 6811 */ 6812 int ice_down(struct ice_vsi *vsi) 6813 { 6814 int i, tx_err, rx_err, vlan_err = 0; 6815 6816 WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state)); 6817 6818 if (vsi->netdev && vsi->type == ICE_VSI_PF) { 6819 vlan_err = ice_vsi_del_vlan_zero(vsi); 6820 ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false); 6821 netif_carrier_off(vsi->netdev); 6822 netif_tx_disable(vsi->netdev); 6823 } else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) { 6824 ice_eswitch_stop_all_tx_queues(vsi->back); 6825 } 6826 6827 ice_vsi_dis_irq(vsi); 6828 6829 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 6830 if (tx_err) 6831 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 6832 vsi->vsi_num, tx_err); 6833 if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { 6834 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 6835 if (tx_err) 6836 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 6837 vsi->vsi_num, tx_err); 6838 } 6839 6840 rx_err = ice_vsi_stop_all_rx_rings(vsi); 6841 if (rx_err) 6842 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 6843 vsi->vsi_num, rx_err); 6844 6845 ice_napi_disable_all(vsi); 6846 6847 ice_for_each_txq(vsi, i) 6848 ice_clean_tx_ring(vsi->tx_rings[i]); 6849 6850 if (ice_is_xdp_ena_vsi(vsi)) 6851 ice_for_each_xdp_txq(vsi, i) 6852 ice_clean_tx_ring(vsi->xdp_rings[i]); 6853 6854 ice_for_each_rxq(vsi, i) 6855 ice_clean_rx_ring(vsi->rx_rings[i]); 6856 6857 if (tx_err || rx_err || vlan_err) { 6858 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 6859 vsi->vsi_num, vsi->vsw->sw_id); 6860 return -EIO; 6861 } 6862 6863 return 0; 6864 } 6865 6866 /** 6867 * ice_down_up - shutdown the VSI connection and bring it up 6868 * @vsi: the VSI to be reconnected 6869 */ 6870 int ice_down_up(struct ice_vsi *vsi) 6871 { 6872 int ret; 6873 6874 /* if DOWN already set, nothing to do */ 6875 if (test_and_set_bit(ICE_VSI_DOWN, vsi->state)) 6876 return 0; 6877 6878 ret = ice_down(vsi); 6879 if (ret) 6880 return ret; 6881 6882 ret = ice_up(vsi); 6883 if (ret) { 6884 netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n"); 6885 return ret; 6886 } 6887 6888 return 0; 6889 } 6890 6891 /** 6892 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 6893 * @vsi: VSI having resources allocated 6894 * 6895 * Return 0 on success, negative on failure 6896 */ 6897 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 6898 { 6899 int i, err = 0; 6900 6901 if (!vsi->num_txq) { 6902 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 6903 vsi->vsi_num); 6904 return -EINVAL; 6905 } 6906 6907 ice_for_each_txq(vsi, i) { 6908 struct ice_tx_ring *ring = vsi->tx_rings[i]; 6909 6910 if (!ring) 6911 return -EINVAL; 6912 6913 if (vsi->netdev) 6914 ring->netdev = vsi->netdev; 6915 err = ice_setup_tx_ring(ring); 6916 if (err) 6917 break; 6918 } 6919 6920 return err; 6921 } 6922 6923 /** 6924 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 6925 * @vsi: VSI having resources allocated 6926 * 6927 * Return 0 on success, negative on failure 6928 */ 6929 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 6930 { 6931 int i, err = 0; 6932 6933 if (!vsi->num_rxq) { 6934 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 6935 vsi->vsi_num); 6936 return -EINVAL; 6937 } 6938 6939 ice_for_each_rxq(vsi, i) { 6940 struct ice_rx_ring *ring = vsi->rx_rings[i]; 6941 6942 if (!ring) 6943 return -EINVAL; 6944 6945 if (vsi->netdev) 6946 ring->netdev = vsi->netdev; 6947 err = ice_setup_rx_ring(ring); 6948 if (err) 6949 break; 6950 } 6951 6952 return err; 6953 } 6954 6955 /** 6956 * ice_vsi_open_ctrl - open control VSI for use 6957 * @vsi: the VSI to open 6958 * 6959 * Initialization of the Control VSI 6960 * 6961 * Returns 0 on success, negative value on error 6962 */ 6963 int ice_vsi_open_ctrl(struct ice_vsi *vsi) 6964 { 6965 char int_name[ICE_INT_NAME_STR_LEN]; 6966 struct ice_pf *pf = vsi->back; 6967 struct device *dev; 6968 int err; 6969 6970 dev = ice_pf_to_dev(pf); 6971 /* allocate descriptors */ 6972 err = ice_vsi_setup_tx_rings(vsi); 6973 if (err) 6974 goto err_setup_tx; 6975 6976 err = ice_vsi_setup_rx_rings(vsi); 6977 if (err) 6978 goto err_setup_rx; 6979 6980 err = ice_vsi_cfg_lan(vsi); 6981 if (err) 6982 goto err_setup_rx; 6983 6984 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 6985 dev_driver_string(dev), dev_name(dev)); 6986 err = ice_vsi_req_irq_msix(vsi, int_name); 6987 if (err) 6988 goto err_setup_rx; 6989 6990 ice_vsi_cfg_msix(vsi); 6991 6992 err = ice_vsi_start_all_rx_rings(vsi); 6993 if (err) 6994 goto err_up_complete; 6995 6996 clear_bit(ICE_VSI_DOWN, vsi->state); 6997 ice_vsi_ena_irq(vsi); 6998 6999 return 0; 7000 7001 err_up_complete: 7002 ice_down(vsi); 7003 err_setup_rx: 7004 ice_vsi_free_rx_rings(vsi); 7005 err_setup_tx: 7006 ice_vsi_free_tx_rings(vsi); 7007 7008 return err; 7009 } 7010 7011 /** 7012 * ice_vsi_open - Called when a network interface is made active 7013 * @vsi: the VSI to open 7014 * 7015 * Initialization of the VSI 7016 * 7017 * Returns 0 on success, negative value on error 7018 */ 7019 int ice_vsi_open(struct ice_vsi *vsi) 7020 { 7021 char int_name[ICE_INT_NAME_STR_LEN]; 7022 struct ice_pf *pf = vsi->back; 7023 int err; 7024 7025 /* allocate descriptors */ 7026 err = ice_vsi_setup_tx_rings(vsi); 7027 if (err) 7028 goto err_setup_tx; 7029 7030 err = ice_vsi_setup_rx_rings(vsi); 7031 if (err) 7032 goto err_setup_rx; 7033 7034 err = ice_vsi_cfg_lan(vsi); 7035 if (err) 7036 goto err_setup_rx; 7037 7038 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7039 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 7040 err = ice_vsi_req_irq_msix(vsi, int_name); 7041 if (err) 7042 goto err_setup_rx; 7043 7044 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 7045 7046 if (vsi->type == ICE_VSI_PF) { 7047 /* Notify the stack of the actual queue counts. */ 7048 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 7049 if (err) 7050 goto err_set_qs; 7051 7052 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 7053 if (err) 7054 goto err_set_qs; 7055 } 7056 7057 err = ice_up_complete(vsi); 7058 if (err) 7059 goto err_up_complete; 7060 7061 return 0; 7062 7063 err_up_complete: 7064 ice_down(vsi); 7065 err_set_qs: 7066 ice_vsi_free_irq(vsi); 7067 err_setup_rx: 7068 ice_vsi_free_rx_rings(vsi); 7069 err_setup_tx: 7070 ice_vsi_free_tx_rings(vsi); 7071 7072 return err; 7073 } 7074 7075 /** 7076 * ice_vsi_release_all - Delete all VSIs 7077 * @pf: PF from which all VSIs are being removed 7078 */ 7079 static void ice_vsi_release_all(struct ice_pf *pf) 7080 { 7081 int err, i; 7082 7083 if (!pf->vsi) 7084 return; 7085 7086 ice_for_each_vsi(pf, i) { 7087 if (!pf->vsi[i]) 7088 continue; 7089 7090 if (pf->vsi[i]->type == ICE_VSI_CHNL) 7091 continue; 7092 7093 err = ice_vsi_release(pf->vsi[i]); 7094 if (err) 7095 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 7096 i, err, pf->vsi[i]->vsi_num); 7097 } 7098 } 7099 7100 /** 7101 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 7102 * @pf: pointer to the PF instance 7103 * @type: VSI type to rebuild 7104 * 7105 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 7106 */ 7107 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 7108 { 7109 struct device *dev = ice_pf_to_dev(pf); 7110 int i, err; 7111 7112 ice_for_each_vsi(pf, i) { 7113 struct ice_vsi *vsi = pf->vsi[i]; 7114 7115 if (!vsi || vsi->type != type) 7116 continue; 7117 7118 /* rebuild the VSI */ 7119 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT); 7120 if (err) { 7121 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 7122 err, vsi->idx, ice_vsi_type_str(type)); 7123 return err; 7124 } 7125 7126 /* replay filters for the VSI */ 7127 err = ice_replay_vsi(&pf->hw, vsi->idx); 7128 if (err) { 7129 dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n", 7130 err, vsi->idx, ice_vsi_type_str(type)); 7131 return err; 7132 } 7133 7134 /* Re-map HW VSI number, using VSI handle that has been 7135 * previously validated in ice_replay_vsi() call above 7136 */ 7137 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 7138 7139 /* enable the VSI */ 7140 err = ice_ena_vsi(vsi, false); 7141 if (err) { 7142 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 7143 err, vsi->idx, ice_vsi_type_str(type)); 7144 return err; 7145 } 7146 7147 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 7148 ice_vsi_type_str(type)); 7149 } 7150 7151 return 0; 7152 } 7153 7154 /** 7155 * ice_update_pf_netdev_link - Update PF netdev link status 7156 * @pf: pointer to the PF instance 7157 */ 7158 static void ice_update_pf_netdev_link(struct ice_pf *pf) 7159 { 7160 bool link_up; 7161 int i; 7162 7163 ice_for_each_vsi(pf, i) { 7164 struct ice_vsi *vsi = pf->vsi[i]; 7165 7166 if (!vsi || vsi->type != ICE_VSI_PF) 7167 return; 7168 7169 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 7170 if (link_up) { 7171 netif_carrier_on(pf->vsi[i]->netdev); 7172 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 7173 } else { 7174 netif_carrier_off(pf->vsi[i]->netdev); 7175 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 7176 } 7177 } 7178 } 7179 7180 /** 7181 * ice_rebuild - rebuild after reset 7182 * @pf: PF to rebuild 7183 * @reset_type: type of reset 7184 * 7185 * Do not rebuild VF VSI in this flow because that is already handled via 7186 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 7187 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 7188 * to reset/rebuild all the VF VSI twice. 7189 */ 7190 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 7191 { 7192 struct device *dev = ice_pf_to_dev(pf); 7193 struct ice_hw *hw = &pf->hw; 7194 bool dvm; 7195 int err; 7196 7197 if (test_bit(ICE_DOWN, pf->state)) 7198 goto clear_recovery; 7199 7200 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 7201 7202 #define ICE_EMP_RESET_SLEEP_MS 5000 7203 if (reset_type == ICE_RESET_EMPR) { 7204 /* If an EMP reset has occurred, any previously pending flash 7205 * update will have completed. We no longer know whether or 7206 * not the NVM update EMP reset is restricted. 7207 */ 7208 pf->fw_emp_reset_disabled = false; 7209 7210 msleep(ICE_EMP_RESET_SLEEP_MS); 7211 } 7212 7213 err = ice_init_all_ctrlq(hw); 7214 if (err) { 7215 dev_err(dev, "control queues init failed %d\n", err); 7216 goto err_init_ctrlq; 7217 } 7218 7219 /* if DDP was previously loaded successfully */ 7220 if (!ice_is_safe_mode(pf)) { 7221 /* reload the SW DB of filter tables */ 7222 if (reset_type == ICE_RESET_PFR) 7223 ice_fill_blk_tbls(hw); 7224 else 7225 /* Reload DDP Package after CORER/GLOBR reset */ 7226 ice_load_pkg(NULL, pf); 7227 } 7228 7229 err = ice_clear_pf_cfg(hw); 7230 if (err) { 7231 dev_err(dev, "clear PF configuration failed %d\n", err); 7232 goto err_init_ctrlq; 7233 } 7234 7235 ice_clear_pxe_mode(hw); 7236 7237 err = ice_init_nvm(hw); 7238 if (err) { 7239 dev_err(dev, "ice_init_nvm failed %d\n", err); 7240 goto err_init_ctrlq; 7241 } 7242 7243 err = ice_get_caps(hw); 7244 if (err) { 7245 dev_err(dev, "ice_get_caps failed %d\n", err); 7246 goto err_init_ctrlq; 7247 } 7248 7249 err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 7250 if (err) { 7251 dev_err(dev, "set_mac_cfg failed %d\n", err); 7252 goto err_init_ctrlq; 7253 } 7254 7255 dvm = ice_is_dvm_ena(hw); 7256 7257 err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 7258 if (err) 7259 goto err_init_ctrlq; 7260 7261 err = ice_sched_init_port(hw->port_info); 7262 if (err) 7263 goto err_sched_init_port; 7264 7265 /* start misc vector */ 7266 err = ice_req_irq_msix_misc(pf); 7267 if (err) { 7268 dev_err(dev, "misc vector setup failed: %d\n", err); 7269 goto err_sched_init_port; 7270 } 7271 7272 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7273 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 7274 if (!rd32(hw, PFQF_FD_SIZE)) { 7275 u16 unused, guar, b_effort; 7276 7277 guar = hw->func_caps.fd_fltr_guar; 7278 b_effort = hw->func_caps.fd_fltr_best_effort; 7279 7280 /* force guaranteed filter pool for PF */ 7281 ice_alloc_fd_guar_item(hw, &unused, guar); 7282 /* force shared filter pool for PF */ 7283 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 7284 } 7285 } 7286 7287 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 7288 ice_dcb_rebuild(pf); 7289 7290 /* If the PF previously had enabled PTP, PTP init needs to happen before 7291 * the VSI rebuild. If not, this causes the PTP link status events to 7292 * fail. 7293 */ 7294 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 7295 ice_ptp_reset(pf); 7296 7297 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 7298 ice_gnss_init(pf); 7299 7300 /* rebuild PF VSI */ 7301 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 7302 if (err) { 7303 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 7304 goto err_vsi_rebuild; 7305 } 7306 7307 /* configure PTP timestamping after VSI rebuild */ 7308 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 7309 ice_ptp_cfg_timestamp(pf, false); 7310 7311 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_SWITCHDEV_CTRL); 7312 if (err) { 7313 dev_err(dev, "Switchdev CTRL VSI rebuild failed: %d\n", err); 7314 goto err_vsi_rebuild; 7315 } 7316 7317 if (reset_type == ICE_RESET_PFR) { 7318 err = ice_rebuild_channels(pf); 7319 if (err) { 7320 dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n", 7321 err); 7322 goto err_vsi_rebuild; 7323 } 7324 } 7325 7326 /* If Flow Director is active */ 7327 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7328 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 7329 if (err) { 7330 dev_err(dev, "control VSI rebuild failed: %d\n", err); 7331 goto err_vsi_rebuild; 7332 } 7333 7334 /* replay HW Flow Director recipes */ 7335 if (hw->fdir_prof) 7336 ice_fdir_replay_flows(hw); 7337 7338 /* replay Flow Director filters */ 7339 ice_fdir_replay_fltrs(pf); 7340 7341 ice_rebuild_arfs(pf); 7342 } 7343 7344 ice_update_pf_netdev_link(pf); 7345 7346 /* tell the firmware we are up */ 7347 err = ice_send_version(pf); 7348 if (err) { 7349 dev_err(dev, "Rebuild failed due to error sending driver version: %d\n", 7350 err); 7351 goto err_vsi_rebuild; 7352 } 7353 7354 ice_replay_post(hw); 7355 7356 /* if we get here, reset flow is successful */ 7357 clear_bit(ICE_RESET_FAILED, pf->state); 7358 7359 ice_plug_aux_dev(pf); 7360 return; 7361 7362 err_vsi_rebuild: 7363 err_sched_init_port: 7364 ice_sched_cleanup_all(hw); 7365 err_init_ctrlq: 7366 ice_shutdown_all_ctrlq(hw); 7367 set_bit(ICE_RESET_FAILED, pf->state); 7368 clear_recovery: 7369 /* set this bit in PF state to control service task scheduling */ 7370 set_bit(ICE_NEEDS_RESTART, pf->state); 7371 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 7372 } 7373 7374 /** 7375 * ice_change_mtu - NDO callback to change the MTU 7376 * @netdev: network interface device structure 7377 * @new_mtu: new value for maximum frame size 7378 * 7379 * Returns 0 on success, negative on failure 7380 */ 7381 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 7382 { 7383 struct ice_netdev_priv *np = netdev_priv(netdev); 7384 struct ice_vsi *vsi = np->vsi; 7385 struct ice_pf *pf = vsi->back; 7386 struct bpf_prog *prog; 7387 u8 count = 0; 7388 int err = 0; 7389 7390 if (new_mtu == (int)netdev->mtu) { 7391 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 7392 return 0; 7393 } 7394 7395 prog = vsi->xdp_prog; 7396 if (prog && !prog->aux->xdp_has_frags) { 7397 int frame_size = ice_max_xdp_frame_size(vsi); 7398 7399 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 7400 netdev_err(netdev, "max MTU for XDP usage is %d\n", 7401 frame_size - ICE_ETH_PKT_HDR_PAD); 7402 return -EINVAL; 7403 } 7404 } else if (test_bit(ICE_FLAG_LEGACY_RX, pf->flags)) { 7405 if (new_mtu + ICE_ETH_PKT_HDR_PAD > ICE_MAX_FRAME_LEGACY_RX) { 7406 netdev_err(netdev, "Too big MTU for legacy-rx; Max is %d\n", 7407 ICE_MAX_FRAME_LEGACY_RX - ICE_ETH_PKT_HDR_PAD); 7408 return -EINVAL; 7409 } 7410 } 7411 7412 /* if a reset is in progress, wait for some time for it to complete */ 7413 do { 7414 if (ice_is_reset_in_progress(pf->state)) { 7415 count++; 7416 usleep_range(1000, 2000); 7417 } else { 7418 break; 7419 } 7420 7421 } while (count < 100); 7422 7423 if (count == 100) { 7424 netdev_err(netdev, "can't change MTU. Device is busy\n"); 7425 return -EBUSY; 7426 } 7427 7428 netdev->mtu = (unsigned int)new_mtu; 7429 err = ice_down_up(vsi); 7430 if (err) 7431 return err; 7432 7433 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 7434 set_bit(ICE_FLAG_MTU_CHANGED, pf->flags); 7435 7436 return err; 7437 } 7438 7439 /** 7440 * ice_eth_ioctl - Access the hwtstamp interface 7441 * @netdev: network interface device structure 7442 * @ifr: interface request data 7443 * @cmd: ioctl command 7444 */ 7445 static int ice_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 7446 { 7447 struct ice_netdev_priv *np = netdev_priv(netdev); 7448 struct ice_pf *pf = np->vsi->back; 7449 7450 switch (cmd) { 7451 case SIOCGHWTSTAMP: 7452 return ice_ptp_get_ts_config(pf, ifr); 7453 case SIOCSHWTSTAMP: 7454 return ice_ptp_set_ts_config(pf, ifr); 7455 default: 7456 return -EOPNOTSUPP; 7457 } 7458 } 7459 7460 /** 7461 * ice_aq_str - convert AQ err code to a string 7462 * @aq_err: the AQ error code to convert 7463 */ 7464 const char *ice_aq_str(enum ice_aq_err aq_err) 7465 { 7466 switch (aq_err) { 7467 case ICE_AQ_RC_OK: 7468 return "OK"; 7469 case ICE_AQ_RC_EPERM: 7470 return "ICE_AQ_RC_EPERM"; 7471 case ICE_AQ_RC_ENOENT: 7472 return "ICE_AQ_RC_ENOENT"; 7473 case ICE_AQ_RC_ENOMEM: 7474 return "ICE_AQ_RC_ENOMEM"; 7475 case ICE_AQ_RC_EBUSY: 7476 return "ICE_AQ_RC_EBUSY"; 7477 case ICE_AQ_RC_EEXIST: 7478 return "ICE_AQ_RC_EEXIST"; 7479 case ICE_AQ_RC_EINVAL: 7480 return "ICE_AQ_RC_EINVAL"; 7481 case ICE_AQ_RC_ENOSPC: 7482 return "ICE_AQ_RC_ENOSPC"; 7483 case ICE_AQ_RC_ENOSYS: 7484 return "ICE_AQ_RC_ENOSYS"; 7485 case ICE_AQ_RC_EMODE: 7486 return "ICE_AQ_RC_EMODE"; 7487 case ICE_AQ_RC_ENOSEC: 7488 return "ICE_AQ_RC_ENOSEC"; 7489 case ICE_AQ_RC_EBADSIG: 7490 return "ICE_AQ_RC_EBADSIG"; 7491 case ICE_AQ_RC_ESVN: 7492 return "ICE_AQ_RC_ESVN"; 7493 case ICE_AQ_RC_EBADMAN: 7494 return "ICE_AQ_RC_EBADMAN"; 7495 case ICE_AQ_RC_EBADBUF: 7496 return "ICE_AQ_RC_EBADBUF"; 7497 } 7498 7499 return "ICE_AQ_RC_UNKNOWN"; 7500 } 7501 7502 /** 7503 * ice_set_rss_lut - Set RSS LUT 7504 * @vsi: Pointer to VSI structure 7505 * @lut: Lookup table 7506 * @lut_size: Lookup table size 7507 * 7508 * Returns 0 on success, negative on failure 7509 */ 7510 int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7511 { 7512 struct ice_aq_get_set_rss_lut_params params = {}; 7513 struct ice_hw *hw = &vsi->back->hw; 7514 int status; 7515 7516 if (!lut) 7517 return -EINVAL; 7518 7519 params.vsi_handle = vsi->idx; 7520 params.lut_size = lut_size; 7521 params.lut_type = vsi->rss_lut_type; 7522 params.lut = lut; 7523 7524 status = ice_aq_set_rss_lut(hw, ¶ms); 7525 if (status) 7526 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n", 7527 status, ice_aq_str(hw->adminq.sq_last_status)); 7528 7529 return status; 7530 } 7531 7532 /** 7533 * ice_set_rss_key - Set RSS key 7534 * @vsi: Pointer to the VSI structure 7535 * @seed: RSS hash seed 7536 * 7537 * Returns 0 on success, negative on failure 7538 */ 7539 int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed) 7540 { 7541 struct ice_hw *hw = &vsi->back->hw; 7542 int status; 7543 7544 if (!seed) 7545 return -EINVAL; 7546 7547 status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7548 if (status) 7549 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n", 7550 status, ice_aq_str(hw->adminq.sq_last_status)); 7551 7552 return status; 7553 } 7554 7555 /** 7556 * ice_get_rss_lut - Get RSS LUT 7557 * @vsi: Pointer to VSI structure 7558 * @lut: Buffer to store the lookup table entries 7559 * @lut_size: Size of buffer to store the lookup table entries 7560 * 7561 * Returns 0 on success, negative on failure 7562 */ 7563 int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7564 { 7565 struct ice_aq_get_set_rss_lut_params params = {}; 7566 struct ice_hw *hw = &vsi->back->hw; 7567 int status; 7568 7569 if (!lut) 7570 return -EINVAL; 7571 7572 params.vsi_handle = vsi->idx; 7573 params.lut_size = lut_size; 7574 params.lut_type = vsi->rss_lut_type; 7575 params.lut = lut; 7576 7577 status = ice_aq_get_rss_lut(hw, ¶ms); 7578 if (status) 7579 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n", 7580 status, ice_aq_str(hw->adminq.sq_last_status)); 7581 7582 return status; 7583 } 7584 7585 /** 7586 * ice_get_rss_key - Get RSS key 7587 * @vsi: Pointer to VSI structure 7588 * @seed: Buffer to store the key in 7589 * 7590 * Returns 0 on success, negative on failure 7591 */ 7592 int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed) 7593 { 7594 struct ice_hw *hw = &vsi->back->hw; 7595 int status; 7596 7597 if (!seed) 7598 return -EINVAL; 7599 7600 status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7601 if (status) 7602 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n", 7603 status, ice_aq_str(hw->adminq.sq_last_status)); 7604 7605 return status; 7606 } 7607 7608 /** 7609 * ice_bridge_getlink - Get the hardware bridge mode 7610 * @skb: skb buff 7611 * @pid: process ID 7612 * @seq: RTNL message seq 7613 * @dev: the netdev being configured 7614 * @filter_mask: filter mask passed in 7615 * @nlflags: netlink flags passed in 7616 * 7617 * Return the bridge mode (VEB/VEPA) 7618 */ 7619 static int 7620 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 7621 struct net_device *dev, u32 filter_mask, int nlflags) 7622 { 7623 struct ice_netdev_priv *np = netdev_priv(dev); 7624 struct ice_vsi *vsi = np->vsi; 7625 struct ice_pf *pf = vsi->back; 7626 u16 bmode; 7627 7628 bmode = pf->first_sw->bridge_mode; 7629 7630 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 7631 filter_mask, NULL); 7632 } 7633 7634 /** 7635 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 7636 * @vsi: Pointer to VSI structure 7637 * @bmode: Hardware bridge mode (VEB/VEPA) 7638 * 7639 * Returns 0 on success, negative on failure 7640 */ 7641 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 7642 { 7643 struct ice_aqc_vsi_props *vsi_props; 7644 struct ice_hw *hw = &vsi->back->hw; 7645 struct ice_vsi_ctx *ctxt; 7646 int ret; 7647 7648 vsi_props = &vsi->info; 7649 7650 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 7651 if (!ctxt) 7652 return -ENOMEM; 7653 7654 ctxt->info = vsi->info; 7655 7656 if (bmode == BRIDGE_MODE_VEB) 7657 /* change from VEPA to VEB mode */ 7658 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 7659 else 7660 /* change from VEB to VEPA mode */ 7661 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 7662 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 7663 7664 ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 7665 if (ret) { 7666 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n", 7667 bmode, ret, ice_aq_str(hw->adminq.sq_last_status)); 7668 goto out; 7669 } 7670 /* Update sw flags for book keeping */ 7671 vsi_props->sw_flags = ctxt->info.sw_flags; 7672 7673 out: 7674 kfree(ctxt); 7675 return ret; 7676 } 7677 7678 /** 7679 * ice_bridge_setlink - Set the hardware bridge mode 7680 * @dev: the netdev being configured 7681 * @nlh: RTNL message 7682 * @flags: bridge setlink flags 7683 * @extack: netlink extended ack 7684 * 7685 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 7686 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 7687 * not already set for all VSIs connected to this switch. And also update the 7688 * unicast switch filter rules for the corresponding switch of the netdev. 7689 */ 7690 static int 7691 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 7692 u16 __always_unused flags, 7693 struct netlink_ext_ack __always_unused *extack) 7694 { 7695 struct ice_netdev_priv *np = netdev_priv(dev); 7696 struct ice_pf *pf = np->vsi->back; 7697 struct nlattr *attr, *br_spec; 7698 struct ice_hw *hw = &pf->hw; 7699 struct ice_sw *pf_sw; 7700 int rem, v, err = 0; 7701 7702 pf_sw = pf->first_sw; 7703 /* find the attribute in the netlink message */ 7704 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 7705 7706 nla_for_each_nested(attr, br_spec, rem) { 7707 __u16 mode; 7708 7709 if (nla_type(attr) != IFLA_BRIDGE_MODE) 7710 continue; 7711 mode = nla_get_u16(attr); 7712 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 7713 return -EINVAL; 7714 /* Continue if bridge mode is not being flipped */ 7715 if (mode == pf_sw->bridge_mode) 7716 continue; 7717 /* Iterates through the PF VSI list and update the loopback 7718 * mode of the VSI 7719 */ 7720 ice_for_each_vsi(pf, v) { 7721 if (!pf->vsi[v]) 7722 continue; 7723 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 7724 if (err) 7725 return err; 7726 } 7727 7728 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 7729 /* Update the unicast switch filter rules for the corresponding 7730 * switch of the netdev 7731 */ 7732 err = ice_update_sw_rule_bridge_mode(hw); 7733 if (err) { 7734 netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n", 7735 mode, err, 7736 ice_aq_str(hw->adminq.sq_last_status)); 7737 /* revert hw->evb_veb */ 7738 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 7739 return err; 7740 } 7741 7742 pf_sw->bridge_mode = mode; 7743 } 7744 7745 return 0; 7746 } 7747 7748 /** 7749 * ice_tx_timeout - Respond to a Tx Hang 7750 * @netdev: network interface device structure 7751 * @txqueue: Tx queue 7752 */ 7753 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 7754 { 7755 struct ice_netdev_priv *np = netdev_priv(netdev); 7756 struct ice_tx_ring *tx_ring = NULL; 7757 struct ice_vsi *vsi = np->vsi; 7758 struct ice_pf *pf = vsi->back; 7759 u32 i; 7760 7761 pf->tx_timeout_count++; 7762 7763 /* Check if PFC is enabled for the TC to which the queue belongs 7764 * to. If yes then Tx timeout is not caused by a hung queue, no 7765 * need to reset and rebuild 7766 */ 7767 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 7768 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 7769 txqueue); 7770 return; 7771 } 7772 7773 /* now that we have an index, find the tx_ring struct */ 7774 ice_for_each_txq(vsi, i) 7775 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 7776 if (txqueue == vsi->tx_rings[i]->q_index) { 7777 tx_ring = vsi->tx_rings[i]; 7778 break; 7779 } 7780 7781 /* Reset recovery level if enough time has elapsed after last timeout. 7782 * Also ensure no new reset action happens before next timeout period. 7783 */ 7784 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 7785 pf->tx_timeout_recovery_level = 1; 7786 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 7787 netdev->watchdog_timeo))) 7788 return; 7789 7790 if (tx_ring) { 7791 struct ice_hw *hw = &pf->hw; 7792 u32 head, val = 0; 7793 7794 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) & 7795 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 7796 /* Read interrupt register */ 7797 val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 7798 7799 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 7800 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 7801 head, tx_ring->next_to_use, val); 7802 } 7803 7804 pf->tx_timeout_last_recovery = jiffies; 7805 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 7806 pf->tx_timeout_recovery_level, txqueue); 7807 7808 switch (pf->tx_timeout_recovery_level) { 7809 case 1: 7810 set_bit(ICE_PFR_REQ, pf->state); 7811 break; 7812 case 2: 7813 set_bit(ICE_CORER_REQ, pf->state); 7814 break; 7815 case 3: 7816 set_bit(ICE_GLOBR_REQ, pf->state); 7817 break; 7818 default: 7819 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 7820 set_bit(ICE_DOWN, pf->state); 7821 set_bit(ICE_VSI_NEEDS_RESTART, vsi->state); 7822 set_bit(ICE_SERVICE_DIS, pf->state); 7823 break; 7824 } 7825 7826 ice_service_task_schedule(pf); 7827 pf->tx_timeout_recovery_level++; 7828 } 7829 7830 /** 7831 * ice_setup_tc_cls_flower - flower classifier offloads 7832 * @np: net device to configure 7833 * @filter_dev: device on which filter is added 7834 * @cls_flower: offload data 7835 */ 7836 static int 7837 ice_setup_tc_cls_flower(struct ice_netdev_priv *np, 7838 struct net_device *filter_dev, 7839 struct flow_cls_offload *cls_flower) 7840 { 7841 struct ice_vsi *vsi = np->vsi; 7842 7843 if (cls_flower->common.chain_index) 7844 return -EOPNOTSUPP; 7845 7846 switch (cls_flower->command) { 7847 case FLOW_CLS_REPLACE: 7848 return ice_add_cls_flower(filter_dev, vsi, cls_flower); 7849 case FLOW_CLS_DESTROY: 7850 return ice_del_cls_flower(vsi, cls_flower); 7851 default: 7852 return -EINVAL; 7853 } 7854 } 7855 7856 /** 7857 * ice_setup_tc_block_cb - callback handler registered for TC block 7858 * @type: TC SETUP type 7859 * @type_data: TC flower offload data that contains user input 7860 * @cb_priv: netdev private data 7861 */ 7862 static int 7863 ice_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 7864 { 7865 struct ice_netdev_priv *np = cb_priv; 7866 7867 switch (type) { 7868 case TC_SETUP_CLSFLOWER: 7869 return ice_setup_tc_cls_flower(np, np->vsi->netdev, 7870 type_data); 7871 default: 7872 return -EOPNOTSUPP; 7873 } 7874 } 7875 7876 /** 7877 * ice_validate_mqprio_qopt - Validate TCF input parameters 7878 * @vsi: Pointer to VSI 7879 * @mqprio_qopt: input parameters for mqprio queue configuration 7880 * 7881 * This function validates MQPRIO params, such as qcount (power of 2 wherever 7882 * needed), and make sure user doesn't specify qcount and BW rate limit 7883 * for TCs, which are more than "num_tc" 7884 */ 7885 static int 7886 ice_validate_mqprio_qopt(struct ice_vsi *vsi, 7887 struct tc_mqprio_qopt_offload *mqprio_qopt) 7888 { 7889 int non_power_of_2_qcount = 0; 7890 struct ice_pf *pf = vsi->back; 7891 int max_rss_q_cnt = 0; 7892 u64 sum_min_rate = 0; 7893 struct device *dev; 7894 int i, speed; 7895 u8 num_tc; 7896 7897 if (vsi->type != ICE_VSI_PF) 7898 return -EINVAL; 7899 7900 if (mqprio_qopt->qopt.offset[0] != 0 || 7901 mqprio_qopt->qopt.num_tc < 1 || 7902 mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC) 7903 return -EINVAL; 7904 7905 dev = ice_pf_to_dev(pf); 7906 vsi->ch_rss_size = 0; 7907 num_tc = mqprio_qopt->qopt.num_tc; 7908 speed = ice_get_link_speed_kbps(vsi); 7909 7910 for (i = 0; num_tc; i++) { 7911 int qcount = mqprio_qopt->qopt.count[i]; 7912 u64 max_rate, min_rate, rem; 7913 7914 if (!qcount) 7915 return -EINVAL; 7916 7917 if (is_power_of_2(qcount)) { 7918 if (non_power_of_2_qcount && 7919 qcount > non_power_of_2_qcount) { 7920 dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n", 7921 qcount, non_power_of_2_qcount); 7922 return -EINVAL; 7923 } 7924 if (qcount > max_rss_q_cnt) 7925 max_rss_q_cnt = qcount; 7926 } else { 7927 if (non_power_of_2_qcount && 7928 qcount != non_power_of_2_qcount) { 7929 dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n", 7930 qcount, non_power_of_2_qcount); 7931 return -EINVAL; 7932 } 7933 if (qcount < max_rss_q_cnt) { 7934 dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n", 7935 qcount, max_rss_q_cnt); 7936 return -EINVAL; 7937 } 7938 max_rss_q_cnt = qcount; 7939 non_power_of_2_qcount = qcount; 7940 } 7941 7942 /* TC command takes input in K/N/Gbps or K/M/Gbit etc but 7943 * converts the bandwidth rate limit into Bytes/s when 7944 * passing it down to the driver. So convert input bandwidth 7945 * from Bytes/s to Kbps 7946 */ 7947 max_rate = mqprio_qopt->max_rate[i]; 7948 max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR); 7949 7950 /* min_rate is minimum guaranteed rate and it can't be zero */ 7951 min_rate = mqprio_qopt->min_rate[i]; 7952 min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR); 7953 sum_min_rate += min_rate; 7954 7955 if (min_rate && min_rate < ICE_MIN_BW_LIMIT) { 7956 dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i, 7957 min_rate, ICE_MIN_BW_LIMIT); 7958 return -EINVAL; 7959 } 7960 7961 if (max_rate && max_rate > speed) { 7962 dev_err(dev, "TC%d: max_rate(%llu Kbps) > link speed of %u Kbps\n", 7963 i, max_rate, speed); 7964 return -EINVAL; 7965 } 7966 7967 iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem); 7968 if (rem) { 7969 dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps", 7970 i, ICE_MIN_BW_LIMIT); 7971 return -EINVAL; 7972 } 7973 7974 iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem); 7975 if (rem) { 7976 dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps", 7977 i, ICE_MIN_BW_LIMIT); 7978 return -EINVAL; 7979 } 7980 7981 /* min_rate can't be more than max_rate, except when max_rate 7982 * is zero (implies max_rate sought is max line rate). In such 7983 * a case min_rate can be more than max. 7984 */ 7985 if (max_rate && min_rate > max_rate) { 7986 dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n", 7987 min_rate, max_rate); 7988 return -EINVAL; 7989 } 7990 7991 if (i >= mqprio_qopt->qopt.num_tc - 1) 7992 break; 7993 if (mqprio_qopt->qopt.offset[i + 1] != 7994 (mqprio_qopt->qopt.offset[i] + qcount)) 7995 return -EINVAL; 7996 } 7997 if (vsi->num_rxq < 7998 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7999 return -EINVAL; 8000 if (vsi->num_txq < 8001 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 8002 return -EINVAL; 8003 8004 if (sum_min_rate && sum_min_rate > (u64)speed) { 8005 dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n", 8006 sum_min_rate, speed); 8007 return -EINVAL; 8008 } 8009 8010 /* make sure vsi->ch_rss_size is set correctly based on TC's qcount */ 8011 vsi->ch_rss_size = max_rss_q_cnt; 8012 8013 return 0; 8014 } 8015 8016 /** 8017 * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF 8018 * @pf: ptr to PF device 8019 * @vsi: ptr to VSI 8020 */ 8021 static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi) 8022 { 8023 struct device *dev = ice_pf_to_dev(pf); 8024 bool added = false; 8025 struct ice_hw *hw; 8026 int flow; 8027 8028 if (!(vsi->num_gfltr || vsi->num_bfltr)) 8029 return -EINVAL; 8030 8031 hw = &pf->hw; 8032 for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) { 8033 struct ice_fd_hw_prof *prof; 8034 int tun, status; 8035 u64 entry_h; 8036 8037 if (!(hw->fdir_prof && hw->fdir_prof[flow] && 8038 hw->fdir_prof[flow]->cnt)) 8039 continue; 8040 8041 for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { 8042 enum ice_flow_priority prio; 8043 u64 prof_id; 8044 8045 /* add this VSI to FDir profile for this flow */ 8046 prio = ICE_FLOW_PRIO_NORMAL; 8047 prof = hw->fdir_prof[flow]; 8048 prof_id = flow + tun * ICE_FLTR_PTYPE_MAX; 8049 status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id, 8050 prof->vsi_h[0], vsi->idx, 8051 prio, prof->fdir_seg[tun], 8052 &entry_h); 8053 if (status) { 8054 dev_err(dev, "channel VSI idx %d, not able to add to group %d\n", 8055 vsi->idx, flow); 8056 continue; 8057 } 8058 8059 prof->entry_h[prof->cnt][tun] = entry_h; 8060 } 8061 8062 /* store VSI for filter replay and delete */ 8063 prof->vsi_h[prof->cnt] = vsi->idx; 8064 prof->cnt++; 8065 8066 added = true; 8067 dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx, 8068 flow); 8069 } 8070 8071 if (!added) 8072 dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx); 8073 8074 return 0; 8075 } 8076 8077 /** 8078 * ice_add_channel - add a channel by adding VSI 8079 * @pf: ptr to PF device 8080 * @sw_id: underlying HW switching element ID 8081 * @ch: ptr to channel structure 8082 * 8083 * Add a channel (VSI) using add_vsi and queue_map 8084 */ 8085 static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch) 8086 { 8087 struct device *dev = ice_pf_to_dev(pf); 8088 struct ice_vsi *vsi; 8089 8090 if (ch->type != ICE_VSI_CHNL) { 8091 dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type); 8092 return -EINVAL; 8093 } 8094 8095 vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch); 8096 if (!vsi || vsi->type != ICE_VSI_CHNL) { 8097 dev_err(dev, "create chnl VSI failure\n"); 8098 return -EINVAL; 8099 } 8100 8101 ice_add_vsi_to_fdir(pf, vsi); 8102 8103 ch->sw_id = sw_id; 8104 ch->vsi_num = vsi->vsi_num; 8105 ch->info.mapping_flags = vsi->info.mapping_flags; 8106 ch->ch_vsi = vsi; 8107 /* set the back pointer of channel for newly created VSI */ 8108 vsi->ch = ch; 8109 8110 memcpy(&ch->info.q_mapping, &vsi->info.q_mapping, 8111 sizeof(vsi->info.q_mapping)); 8112 memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping, 8113 sizeof(vsi->info.tc_mapping)); 8114 8115 return 0; 8116 } 8117 8118 /** 8119 * ice_chnl_cfg_res 8120 * @vsi: the VSI being setup 8121 * @ch: ptr to channel structure 8122 * 8123 * Configure channel specific resources such as rings, vector. 8124 */ 8125 static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch) 8126 { 8127 int i; 8128 8129 for (i = 0; i < ch->num_txq; i++) { 8130 struct ice_q_vector *tx_q_vector, *rx_q_vector; 8131 struct ice_ring_container *rc; 8132 struct ice_tx_ring *tx_ring; 8133 struct ice_rx_ring *rx_ring; 8134 8135 tx_ring = vsi->tx_rings[ch->base_q + i]; 8136 rx_ring = vsi->rx_rings[ch->base_q + i]; 8137 if (!tx_ring || !rx_ring) 8138 continue; 8139 8140 /* setup ring being channel enabled */ 8141 tx_ring->ch = ch; 8142 rx_ring->ch = ch; 8143 8144 /* following code block sets up vector specific attributes */ 8145 tx_q_vector = tx_ring->q_vector; 8146 rx_q_vector = rx_ring->q_vector; 8147 if (!tx_q_vector && !rx_q_vector) 8148 continue; 8149 8150 if (tx_q_vector) { 8151 tx_q_vector->ch = ch; 8152 /* setup Tx and Rx ITR setting if DIM is off */ 8153 rc = &tx_q_vector->tx; 8154 if (!ITR_IS_DYNAMIC(rc)) 8155 ice_write_itr(rc, rc->itr_setting); 8156 } 8157 if (rx_q_vector) { 8158 rx_q_vector->ch = ch; 8159 /* setup Tx and Rx ITR setting if DIM is off */ 8160 rc = &rx_q_vector->rx; 8161 if (!ITR_IS_DYNAMIC(rc)) 8162 ice_write_itr(rc, rc->itr_setting); 8163 } 8164 } 8165 8166 /* it is safe to assume that, if channel has non-zero num_t[r]xq, then 8167 * GLINT_ITR register would have written to perform in-context 8168 * update, hence perform flush 8169 */ 8170 if (ch->num_txq || ch->num_rxq) 8171 ice_flush(&vsi->back->hw); 8172 } 8173 8174 /** 8175 * ice_cfg_chnl_all_res - configure channel resources 8176 * @vsi: pte to main_vsi 8177 * @ch: ptr to channel structure 8178 * 8179 * This function configures channel specific resources such as flow-director 8180 * counter index, and other resources such as queues, vectors, ITR settings 8181 */ 8182 static void 8183 ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch) 8184 { 8185 /* configure channel (aka ADQ) resources such as queues, vectors, 8186 * ITR settings for channel specific vectors and anything else 8187 */ 8188 ice_chnl_cfg_res(vsi, ch); 8189 } 8190 8191 /** 8192 * ice_setup_hw_channel - setup new channel 8193 * @pf: ptr to PF device 8194 * @vsi: the VSI being setup 8195 * @ch: ptr to channel structure 8196 * @sw_id: underlying HW switching element ID 8197 * @type: type of channel to be created (VMDq2/VF) 8198 * 8199 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8200 * and configures Tx rings accordingly 8201 */ 8202 static int 8203 ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8204 struct ice_channel *ch, u16 sw_id, u8 type) 8205 { 8206 struct device *dev = ice_pf_to_dev(pf); 8207 int ret; 8208 8209 ch->base_q = vsi->next_base_q; 8210 ch->type = type; 8211 8212 ret = ice_add_channel(pf, sw_id, ch); 8213 if (ret) { 8214 dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id); 8215 return ret; 8216 } 8217 8218 /* configure/setup ADQ specific resources */ 8219 ice_cfg_chnl_all_res(vsi, ch); 8220 8221 /* make sure to update the next_base_q so that subsequent channel's 8222 * (aka ADQ) VSI queue map is correct 8223 */ 8224 vsi->next_base_q = vsi->next_base_q + ch->num_rxq; 8225 dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num, 8226 ch->num_rxq); 8227 8228 return 0; 8229 } 8230 8231 /** 8232 * ice_setup_channel - setup new channel using uplink element 8233 * @pf: ptr to PF device 8234 * @vsi: the VSI being setup 8235 * @ch: ptr to channel structure 8236 * 8237 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8238 * and uplink switching element 8239 */ 8240 static bool 8241 ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8242 struct ice_channel *ch) 8243 { 8244 struct device *dev = ice_pf_to_dev(pf); 8245 u16 sw_id; 8246 int ret; 8247 8248 if (vsi->type != ICE_VSI_PF) { 8249 dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type); 8250 return false; 8251 } 8252 8253 sw_id = pf->first_sw->sw_id; 8254 8255 /* create channel (VSI) */ 8256 ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL); 8257 if (ret) { 8258 dev_err(dev, "failed to setup hw_channel\n"); 8259 return false; 8260 } 8261 dev_dbg(dev, "successfully created channel()\n"); 8262 8263 return ch->ch_vsi ? true : false; 8264 } 8265 8266 /** 8267 * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 8268 * @vsi: VSI to be configured 8269 * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit 8270 * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit 8271 */ 8272 static int 8273 ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate) 8274 { 8275 int err; 8276 8277 err = ice_set_min_bw_limit(vsi, min_tx_rate); 8278 if (err) 8279 return err; 8280 8281 return ice_set_max_bw_limit(vsi, max_tx_rate); 8282 } 8283 8284 /** 8285 * ice_create_q_channel - function to create channel 8286 * @vsi: VSI to be configured 8287 * @ch: ptr to channel (it contains channel specific params) 8288 * 8289 * This function creates channel (VSI) using num_queues specified by user, 8290 * reconfigs RSS if needed. 8291 */ 8292 static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch) 8293 { 8294 struct ice_pf *pf = vsi->back; 8295 struct device *dev; 8296 8297 if (!ch) 8298 return -EINVAL; 8299 8300 dev = ice_pf_to_dev(pf); 8301 if (!ch->num_txq || !ch->num_rxq) { 8302 dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq); 8303 return -EINVAL; 8304 } 8305 8306 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) { 8307 dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n", 8308 vsi->cnt_q_avail, ch->num_txq); 8309 return -EINVAL; 8310 } 8311 8312 if (!ice_setup_channel(pf, vsi, ch)) { 8313 dev_info(dev, "Failed to setup channel\n"); 8314 return -EINVAL; 8315 } 8316 /* configure BW rate limit */ 8317 if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) { 8318 int ret; 8319 8320 ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate, 8321 ch->min_tx_rate); 8322 if (ret) 8323 dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n", 8324 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8325 else 8326 dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n", 8327 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8328 } 8329 8330 vsi->cnt_q_avail -= ch->num_txq; 8331 8332 return 0; 8333 } 8334 8335 /** 8336 * ice_rem_all_chnl_fltrs - removes all channel filters 8337 * @pf: ptr to PF, TC-flower based filter are tracked at PF level 8338 * 8339 * Remove all advanced switch filters only if they are channel specific 8340 * tc-flower based filter 8341 */ 8342 static void ice_rem_all_chnl_fltrs(struct ice_pf *pf) 8343 { 8344 struct ice_tc_flower_fltr *fltr; 8345 struct hlist_node *node; 8346 8347 /* to remove all channel filters, iterate an ordered list of filters */ 8348 hlist_for_each_entry_safe(fltr, node, 8349 &pf->tc_flower_fltr_list, 8350 tc_flower_node) { 8351 struct ice_rule_query_data rule; 8352 int status; 8353 8354 /* for now process only channel specific filters */ 8355 if (!ice_is_chnl_fltr(fltr)) 8356 continue; 8357 8358 rule.rid = fltr->rid; 8359 rule.rule_id = fltr->rule_id; 8360 rule.vsi_handle = fltr->dest_vsi_handle; 8361 status = ice_rem_adv_rule_by_id(&pf->hw, &rule); 8362 if (status) { 8363 if (status == -ENOENT) 8364 dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n", 8365 rule.rule_id); 8366 else 8367 dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n", 8368 status); 8369 } else if (fltr->dest_vsi) { 8370 /* update advanced switch filter count */ 8371 if (fltr->dest_vsi->type == ICE_VSI_CHNL) { 8372 u32 flags = fltr->flags; 8373 8374 fltr->dest_vsi->num_chnl_fltr--; 8375 if (flags & (ICE_TC_FLWR_FIELD_DST_MAC | 8376 ICE_TC_FLWR_FIELD_ENC_DST_MAC)) 8377 pf->num_dmac_chnl_fltrs--; 8378 } 8379 } 8380 8381 hlist_del(&fltr->tc_flower_node); 8382 kfree(fltr); 8383 } 8384 } 8385 8386 /** 8387 * ice_remove_q_channels - Remove queue channels for the TCs 8388 * @vsi: VSI to be configured 8389 * @rem_fltr: delete advanced switch filter or not 8390 * 8391 * Remove queue channels for the TCs 8392 */ 8393 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr) 8394 { 8395 struct ice_channel *ch, *ch_tmp; 8396 struct ice_pf *pf = vsi->back; 8397 int i; 8398 8399 /* remove all tc-flower based filter if they are channel filters only */ 8400 if (rem_fltr) 8401 ice_rem_all_chnl_fltrs(pf); 8402 8403 /* remove ntuple filters since queue configuration is being changed */ 8404 if (vsi->netdev->features & NETIF_F_NTUPLE) { 8405 struct ice_hw *hw = &pf->hw; 8406 8407 mutex_lock(&hw->fdir_fltr_lock); 8408 ice_fdir_del_all_fltrs(vsi); 8409 mutex_unlock(&hw->fdir_fltr_lock); 8410 } 8411 8412 /* perform cleanup for channels if they exist */ 8413 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 8414 struct ice_vsi *ch_vsi; 8415 8416 list_del(&ch->list); 8417 ch_vsi = ch->ch_vsi; 8418 if (!ch_vsi) { 8419 kfree(ch); 8420 continue; 8421 } 8422 8423 /* Reset queue contexts */ 8424 for (i = 0; i < ch->num_rxq; i++) { 8425 struct ice_tx_ring *tx_ring; 8426 struct ice_rx_ring *rx_ring; 8427 8428 tx_ring = vsi->tx_rings[ch->base_q + i]; 8429 rx_ring = vsi->rx_rings[ch->base_q + i]; 8430 if (tx_ring) { 8431 tx_ring->ch = NULL; 8432 if (tx_ring->q_vector) 8433 tx_ring->q_vector->ch = NULL; 8434 } 8435 if (rx_ring) { 8436 rx_ring->ch = NULL; 8437 if (rx_ring->q_vector) 8438 rx_ring->q_vector->ch = NULL; 8439 } 8440 } 8441 8442 /* Release FD resources for the channel VSI */ 8443 ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx); 8444 8445 /* clear the VSI from scheduler tree */ 8446 ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx); 8447 8448 /* Delete VSI from FW, PF and HW VSI arrays */ 8449 ice_vsi_delete(ch->ch_vsi); 8450 8451 /* free the channel */ 8452 kfree(ch); 8453 } 8454 8455 /* clear the channel VSI map which is stored in main VSI */ 8456 ice_for_each_chnl_tc(i) 8457 vsi->tc_map_vsi[i] = NULL; 8458 8459 /* reset main VSI's all TC information */ 8460 vsi->all_enatc = 0; 8461 vsi->all_numtc = 0; 8462 } 8463 8464 /** 8465 * ice_rebuild_channels - rebuild channel 8466 * @pf: ptr to PF 8467 * 8468 * Recreate channel VSIs and replay filters 8469 */ 8470 static int ice_rebuild_channels(struct ice_pf *pf) 8471 { 8472 struct device *dev = ice_pf_to_dev(pf); 8473 struct ice_vsi *main_vsi; 8474 bool rem_adv_fltr = true; 8475 struct ice_channel *ch; 8476 struct ice_vsi *vsi; 8477 int tc_idx = 1; 8478 int i, err; 8479 8480 main_vsi = ice_get_main_vsi(pf); 8481 if (!main_vsi) 8482 return 0; 8483 8484 if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) || 8485 main_vsi->old_numtc == 1) 8486 return 0; /* nothing to be done */ 8487 8488 /* reconfigure main VSI based on old value of TC and cached values 8489 * for MQPRIO opts 8490 */ 8491 err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc); 8492 if (err) { 8493 dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n", 8494 main_vsi->old_ena_tc, main_vsi->vsi_num); 8495 return err; 8496 } 8497 8498 /* rebuild ADQ VSIs */ 8499 ice_for_each_vsi(pf, i) { 8500 enum ice_vsi_type type; 8501 8502 vsi = pf->vsi[i]; 8503 if (!vsi || vsi->type != ICE_VSI_CHNL) 8504 continue; 8505 8506 type = vsi->type; 8507 8508 /* rebuild ADQ VSI */ 8509 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT); 8510 if (err) { 8511 dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n", 8512 ice_vsi_type_str(type), vsi->idx, err); 8513 goto cleanup; 8514 } 8515 8516 /* Re-map HW VSI number, using VSI handle that has been 8517 * previously validated in ice_replay_vsi() call above 8518 */ 8519 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 8520 8521 /* replay filters for the VSI */ 8522 err = ice_replay_vsi(&pf->hw, vsi->idx); 8523 if (err) { 8524 dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n", 8525 ice_vsi_type_str(type), err, vsi->idx); 8526 rem_adv_fltr = false; 8527 goto cleanup; 8528 } 8529 dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n", 8530 ice_vsi_type_str(type), vsi->idx); 8531 8532 /* store ADQ VSI at correct TC index in main VSI's 8533 * map of TC to VSI 8534 */ 8535 main_vsi->tc_map_vsi[tc_idx++] = vsi; 8536 } 8537 8538 /* ADQ VSI(s) has been rebuilt successfully, so setup 8539 * channel for main VSI's Tx and Rx rings 8540 */ 8541 list_for_each_entry(ch, &main_vsi->ch_list, list) { 8542 struct ice_vsi *ch_vsi; 8543 8544 ch_vsi = ch->ch_vsi; 8545 if (!ch_vsi) 8546 continue; 8547 8548 /* reconfig channel resources */ 8549 ice_cfg_chnl_all_res(main_vsi, ch); 8550 8551 /* replay BW rate limit if it is non-zero */ 8552 if (!ch->max_tx_rate && !ch->min_tx_rate) 8553 continue; 8554 8555 err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate, 8556 ch->min_tx_rate); 8557 if (err) 8558 dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 8559 err, ch->max_tx_rate, ch->min_tx_rate, 8560 ch_vsi->vsi_num); 8561 else 8562 dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 8563 ch->max_tx_rate, ch->min_tx_rate, 8564 ch_vsi->vsi_num); 8565 } 8566 8567 /* reconfig RSS for main VSI */ 8568 if (main_vsi->ch_rss_size) 8569 ice_vsi_cfg_rss_lut_key(main_vsi); 8570 8571 return 0; 8572 8573 cleanup: 8574 ice_remove_q_channels(main_vsi, rem_adv_fltr); 8575 return err; 8576 } 8577 8578 /** 8579 * ice_create_q_channels - Add queue channel for the given TCs 8580 * @vsi: VSI to be configured 8581 * 8582 * Configures queue channel mapping to the given TCs 8583 */ 8584 static int ice_create_q_channels(struct ice_vsi *vsi) 8585 { 8586 struct ice_pf *pf = vsi->back; 8587 struct ice_channel *ch; 8588 int ret = 0, i; 8589 8590 ice_for_each_chnl_tc(i) { 8591 if (!(vsi->all_enatc & BIT(i))) 8592 continue; 8593 8594 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 8595 if (!ch) { 8596 ret = -ENOMEM; 8597 goto err_free; 8598 } 8599 INIT_LIST_HEAD(&ch->list); 8600 ch->num_rxq = vsi->mqprio_qopt.qopt.count[i]; 8601 ch->num_txq = vsi->mqprio_qopt.qopt.count[i]; 8602 ch->base_q = vsi->mqprio_qopt.qopt.offset[i]; 8603 ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i]; 8604 ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i]; 8605 8606 /* convert to Kbits/s */ 8607 if (ch->max_tx_rate) 8608 ch->max_tx_rate = div_u64(ch->max_tx_rate, 8609 ICE_BW_KBPS_DIVISOR); 8610 if (ch->min_tx_rate) 8611 ch->min_tx_rate = div_u64(ch->min_tx_rate, 8612 ICE_BW_KBPS_DIVISOR); 8613 8614 ret = ice_create_q_channel(vsi, ch); 8615 if (ret) { 8616 dev_err(ice_pf_to_dev(pf), 8617 "failed creating channel TC:%d\n", i); 8618 kfree(ch); 8619 goto err_free; 8620 } 8621 list_add_tail(&ch->list, &vsi->ch_list); 8622 vsi->tc_map_vsi[i] = ch->ch_vsi; 8623 dev_dbg(ice_pf_to_dev(pf), 8624 "successfully created channel: VSI %pK\n", ch->ch_vsi); 8625 } 8626 return 0; 8627 8628 err_free: 8629 ice_remove_q_channels(vsi, false); 8630 8631 return ret; 8632 } 8633 8634 /** 8635 * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes 8636 * @netdev: net device to configure 8637 * @type_data: TC offload data 8638 */ 8639 static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data) 8640 { 8641 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8642 struct ice_netdev_priv *np = netdev_priv(netdev); 8643 struct ice_vsi *vsi = np->vsi; 8644 struct ice_pf *pf = vsi->back; 8645 u16 mode, ena_tc_qdisc = 0; 8646 int cur_txq, cur_rxq; 8647 u8 hw = 0, num_tcf; 8648 struct device *dev; 8649 int ret, i; 8650 8651 dev = ice_pf_to_dev(pf); 8652 num_tcf = mqprio_qopt->qopt.num_tc; 8653 hw = mqprio_qopt->qopt.hw; 8654 mode = mqprio_qopt->mode; 8655 if (!hw) { 8656 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 8657 vsi->ch_rss_size = 0; 8658 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8659 goto config_tcf; 8660 } 8661 8662 /* Generate queue region map for number of TCF requested */ 8663 for (i = 0; i < num_tcf; i++) 8664 ena_tc_qdisc |= BIT(i); 8665 8666 switch (mode) { 8667 case TC_MQPRIO_MODE_CHANNEL: 8668 8669 if (pf->hw.port_info->is_custom_tx_enabled) { 8670 dev_err(dev, "Custom Tx scheduler feature enabled, can't configure ADQ\n"); 8671 return -EBUSY; 8672 } 8673 ice_tear_down_devlink_rate_tree(pf); 8674 8675 ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt); 8676 if (ret) { 8677 netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n", 8678 ret); 8679 return ret; 8680 } 8681 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8682 set_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 8683 /* don't assume state of hw_tc_offload during driver load 8684 * and set the flag for TC flower filter if hw_tc_offload 8685 * already ON 8686 */ 8687 if (vsi->netdev->features & NETIF_F_HW_TC) 8688 set_bit(ICE_FLAG_CLS_FLOWER, pf->flags); 8689 break; 8690 default: 8691 return -EINVAL; 8692 } 8693 8694 config_tcf: 8695 8696 /* Requesting same TCF configuration as already enabled */ 8697 if (ena_tc_qdisc == vsi->tc_cfg.ena_tc && 8698 mode != TC_MQPRIO_MODE_CHANNEL) 8699 return 0; 8700 8701 /* Pause VSI queues */ 8702 ice_dis_vsi(vsi, true); 8703 8704 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) 8705 ice_remove_q_channels(vsi, true); 8706 8707 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 8708 vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf), 8709 num_online_cpus()); 8710 vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf), 8711 num_online_cpus()); 8712 } else { 8713 /* logic to rebuild VSI, same like ethtool -L */ 8714 u16 offset = 0, qcount_tx = 0, qcount_rx = 0; 8715 8716 for (i = 0; i < num_tcf; i++) { 8717 if (!(ena_tc_qdisc & BIT(i))) 8718 continue; 8719 8720 offset = vsi->mqprio_qopt.qopt.offset[i]; 8721 qcount_rx = vsi->mqprio_qopt.qopt.count[i]; 8722 qcount_tx = vsi->mqprio_qopt.qopt.count[i]; 8723 } 8724 vsi->req_txq = offset + qcount_tx; 8725 vsi->req_rxq = offset + qcount_rx; 8726 8727 /* store away original rss_size info, so that it gets reused 8728 * form ice_vsi_rebuild during tc-qdisc delete stage - to 8729 * determine, what should be the rss_sizefor main VSI 8730 */ 8731 vsi->orig_rss_size = vsi->rss_size; 8732 } 8733 8734 /* save current values of Tx and Rx queues before calling VSI rebuild 8735 * for fallback option 8736 */ 8737 cur_txq = vsi->num_txq; 8738 cur_rxq = vsi->num_rxq; 8739 8740 /* proceed with rebuild main VSI using correct number of queues */ 8741 ret = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 8742 if (ret) { 8743 /* fallback to current number of queues */ 8744 dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n"); 8745 vsi->req_txq = cur_txq; 8746 vsi->req_rxq = cur_rxq; 8747 clear_bit(ICE_RESET_FAILED, pf->state); 8748 if (ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT)) { 8749 dev_err(dev, "Rebuild of main VSI failed again\n"); 8750 return ret; 8751 } 8752 } 8753 8754 vsi->all_numtc = num_tcf; 8755 vsi->all_enatc = ena_tc_qdisc; 8756 ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc); 8757 if (ret) { 8758 netdev_err(netdev, "failed configuring TC for VSI id=%d\n", 8759 vsi->vsi_num); 8760 goto exit; 8761 } 8762 8763 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 8764 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 8765 u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0]; 8766 8767 /* set TC0 rate limit if specified */ 8768 if (max_tx_rate || min_tx_rate) { 8769 /* convert to Kbits/s */ 8770 if (max_tx_rate) 8771 max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR); 8772 if (min_tx_rate) 8773 min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR); 8774 8775 ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate); 8776 if (!ret) { 8777 dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n", 8778 max_tx_rate, min_tx_rate, vsi->vsi_num); 8779 } else { 8780 dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n", 8781 max_tx_rate, min_tx_rate, vsi->vsi_num); 8782 goto exit; 8783 } 8784 } 8785 ret = ice_create_q_channels(vsi); 8786 if (ret) { 8787 netdev_err(netdev, "failed configuring queue channels\n"); 8788 goto exit; 8789 } else { 8790 netdev_dbg(netdev, "successfully configured channels\n"); 8791 } 8792 } 8793 8794 if (vsi->ch_rss_size) 8795 ice_vsi_cfg_rss_lut_key(vsi); 8796 8797 exit: 8798 /* if error, reset the all_numtc and all_enatc */ 8799 if (ret) { 8800 vsi->all_numtc = 0; 8801 vsi->all_enatc = 0; 8802 } 8803 /* resume VSI */ 8804 ice_ena_vsi(vsi, true); 8805 8806 return ret; 8807 } 8808 8809 static LIST_HEAD(ice_block_cb_list); 8810 8811 static int 8812 ice_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8813 void *type_data) 8814 { 8815 struct ice_netdev_priv *np = netdev_priv(netdev); 8816 struct ice_pf *pf = np->vsi->back; 8817 int err; 8818 8819 switch (type) { 8820 case TC_SETUP_BLOCK: 8821 return flow_block_cb_setup_simple(type_data, 8822 &ice_block_cb_list, 8823 ice_setup_tc_block_cb, 8824 np, np, true); 8825 case TC_SETUP_QDISC_MQPRIO: 8826 /* setup traffic classifier for receive side */ 8827 mutex_lock(&pf->tc_mutex); 8828 err = ice_setup_tc_mqprio_qdisc(netdev, type_data); 8829 mutex_unlock(&pf->tc_mutex); 8830 return err; 8831 default: 8832 return -EOPNOTSUPP; 8833 } 8834 return -EOPNOTSUPP; 8835 } 8836 8837 static struct ice_indr_block_priv * 8838 ice_indr_block_priv_lookup(struct ice_netdev_priv *np, 8839 struct net_device *netdev) 8840 { 8841 struct ice_indr_block_priv *cb_priv; 8842 8843 list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) { 8844 if (!cb_priv->netdev) 8845 return NULL; 8846 if (cb_priv->netdev == netdev) 8847 return cb_priv; 8848 } 8849 return NULL; 8850 } 8851 8852 static int 8853 ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data, 8854 void *indr_priv) 8855 { 8856 struct ice_indr_block_priv *priv = indr_priv; 8857 struct ice_netdev_priv *np = priv->np; 8858 8859 switch (type) { 8860 case TC_SETUP_CLSFLOWER: 8861 return ice_setup_tc_cls_flower(np, priv->netdev, 8862 (struct flow_cls_offload *) 8863 type_data); 8864 default: 8865 return -EOPNOTSUPP; 8866 } 8867 } 8868 8869 static int 8870 ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch, 8871 struct ice_netdev_priv *np, 8872 struct flow_block_offload *f, void *data, 8873 void (*cleanup)(struct flow_block_cb *block_cb)) 8874 { 8875 struct ice_indr_block_priv *indr_priv; 8876 struct flow_block_cb *block_cb; 8877 8878 if (!ice_is_tunnel_supported(netdev) && 8879 !(is_vlan_dev(netdev) && 8880 vlan_dev_real_dev(netdev) == np->vsi->netdev)) 8881 return -EOPNOTSUPP; 8882 8883 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 8884 return -EOPNOTSUPP; 8885 8886 switch (f->command) { 8887 case FLOW_BLOCK_BIND: 8888 indr_priv = ice_indr_block_priv_lookup(np, netdev); 8889 if (indr_priv) 8890 return -EEXIST; 8891 8892 indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL); 8893 if (!indr_priv) 8894 return -ENOMEM; 8895 8896 indr_priv->netdev = netdev; 8897 indr_priv->np = np; 8898 list_add(&indr_priv->list, &np->tc_indr_block_priv_list); 8899 8900 block_cb = 8901 flow_indr_block_cb_alloc(ice_indr_setup_block_cb, 8902 indr_priv, indr_priv, 8903 ice_rep_indr_tc_block_unbind, 8904 f, netdev, sch, data, np, 8905 cleanup); 8906 8907 if (IS_ERR(block_cb)) { 8908 list_del(&indr_priv->list); 8909 kfree(indr_priv); 8910 return PTR_ERR(block_cb); 8911 } 8912 flow_block_cb_add(block_cb, f); 8913 list_add_tail(&block_cb->driver_list, &ice_block_cb_list); 8914 break; 8915 case FLOW_BLOCK_UNBIND: 8916 indr_priv = ice_indr_block_priv_lookup(np, netdev); 8917 if (!indr_priv) 8918 return -ENOENT; 8919 8920 block_cb = flow_block_cb_lookup(f->block, 8921 ice_indr_setup_block_cb, 8922 indr_priv); 8923 if (!block_cb) 8924 return -ENOENT; 8925 8926 flow_indr_block_cb_remove(block_cb, f); 8927 8928 list_del(&block_cb->driver_list); 8929 break; 8930 default: 8931 return -EOPNOTSUPP; 8932 } 8933 return 0; 8934 } 8935 8936 static int 8937 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 8938 void *cb_priv, enum tc_setup_type type, void *type_data, 8939 void *data, 8940 void (*cleanup)(struct flow_block_cb *block_cb)) 8941 { 8942 switch (type) { 8943 case TC_SETUP_BLOCK: 8944 return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data, 8945 data, cleanup); 8946 8947 default: 8948 return -EOPNOTSUPP; 8949 } 8950 } 8951 8952 /** 8953 * ice_open - Called when a network interface becomes active 8954 * @netdev: network interface device structure 8955 * 8956 * The open entry point is called when a network interface is made 8957 * active by the system (IFF_UP). At this point all resources needed 8958 * for transmit and receive operations are allocated, the interrupt 8959 * handler is registered with the OS, the netdev watchdog is enabled, 8960 * and the stack is notified that the interface is ready. 8961 * 8962 * Returns 0 on success, negative value on failure 8963 */ 8964 int ice_open(struct net_device *netdev) 8965 { 8966 struct ice_netdev_priv *np = netdev_priv(netdev); 8967 struct ice_pf *pf = np->vsi->back; 8968 8969 if (ice_is_reset_in_progress(pf->state)) { 8970 netdev_err(netdev, "can't open net device while reset is in progress"); 8971 return -EBUSY; 8972 } 8973 8974 return ice_open_internal(netdev); 8975 } 8976 8977 /** 8978 * ice_open_internal - Called when a network interface becomes active 8979 * @netdev: network interface device structure 8980 * 8981 * Internal ice_open implementation. Should not be used directly except for ice_open and reset 8982 * handling routine 8983 * 8984 * Returns 0 on success, negative value on failure 8985 */ 8986 int ice_open_internal(struct net_device *netdev) 8987 { 8988 struct ice_netdev_priv *np = netdev_priv(netdev); 8989 struct ice_vsi *vsi = np->vsi; 8990 struct ice_pf *pf = vsi->back; 8991 struct ice_port_info *pi; 8992 int err; 8993 8994 if (test_bit(ICE_NEEDS_RESTART, pf->state)) { 8995 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 8996 return -EIO; 8997 } 8998 8999 netif_carrier_off(netdev); 9000 9001 pi = vsi->port_info; 9002 err = ice_update_link_info(pi); 9003 if (err) { 9004 netdev_err(netdev, "Failed to get link info, error %d\n", err); 9005 return err; 9006 } 9007 9008 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 9009 9010 /* Set PHY if there is media, otherwise, turn off PHY */ 9011 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 9012 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 9013 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) { 9014 err = ice_init_phy_user_cfg(pi); 9015 if (err) { 9016 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 9017 err); 9018 return err; 9019 } 9020 } 9021 9022 err = ice_configure_phy(vsi); 9023 if (err) { 9024 netdev_err(netdev, "Failed to set physical link up, error %d\n", 9025 err); 9026 return err; 9027 } 9028 } else { 9029 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 9030 ice_set_link(vsi, false); 9031 } 9032 9033 err = ice_vsi_open(vsi); 9034 if (err) 9035 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 9036 vsi->vsi_num, vsi->vsw->sw_id); 9037 9038 /* Update existing tunnels information */ 9039 udp_tunnel_get_rx_info(netdev); 9040 9041 return err; 9042 } 9043 9044 /** 9045 * ice_stop - Disables a network interface 9046 * @netdev: network interface device structure 9047 * 9048 * The stop entry point is called when an interface is de-activated by the OS, 9049 * and the netdevice enters the DOWN state. The hardware is still under the 9050 * driver's control, but the netdev interface is disabled. 9051 * 9052 * Returns success only - not allowed to fail 9053 */ 9054 int ice_stop(struct net_device *netdev) 9055 { 9056 struct ice_netdev_priv *np = netdev_priv(netdev); 9057 struct ice_vsi *vsi = np->vsi; 9058 struct ice_pf *pf = vsi->back; 9059 9060 if (ice_is_reset_in_progress(pf->state)) { 9061 netdev_err(netdev, "can't stop net device while reset is in progress"); 9062 return -EBUSY; 9063 } 9064 9065 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 9066 int link_err = ice_force_phys_link_state(vsi, false); 9067 9068 if (link_err) { 9069 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 9070 vsi->vsi_num, link_err); 9071 return -EIO; 9072 } 9073 } 9074 9075 ice_vsi_close(vsi); 9076 9077 return 0; 9078 } 9079 9080 /** 9081 * ice_features_check - Validate encapsulated packet conforms to limits 9082 * @skb: skb buffer 9083 * @netdev: This port's netdev 9084 * @features: Offload features that the stack believes apply 9085 */ 9086 static netdev_features_t 9087 ice_features_check(struct sk_buff *skb, 9088 struct net_device __always_unused *netdev, 9089 netdev_features_t features) 9090 { 9091 bool gso = skb_is_gso(skb); 9092 size_t len; 9093 9094 /* No point in doing any of this if neither checksum nor GSO are 9095 * being requested for this frame. We can rule out both by just 9096 * checking for CHECKSUM_PARTIAL 9097 */ 9098 if (skb->ip_summed != CHECKSUM_PARTIAL) 9099 return features; 9100 9101 /* We cannot support GSO if the MSS is going to be less than 9102 * 64 bytes. If it is then we need to drop support for GSO. 9103 */ 9104 if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS)) 9105 features &= ~NETIF_F_GSO_MASK; 9106 9107 len = skb_network_offset(skb); 9108 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 9109 goto out_rm_features; 9110 9111 len = skb_network_header_len(skb); 9112 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 9113 goto out_rm_features; 9114 9115 if (skb->encapsulation) { 9116 /* this must work for VXLAN frames AND IPIP/SIT frames, and in 9117 * the case of IPIP frames, the transport header pointer is 9118 * after the inner header! So check to make sure that this 9119 * is a GRE or UDP_TUNNEL frame before doing that math. 9120 */ 9121 if (gso && (skb_shinfo(skb)->gso_type & 9122 (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) { 9123 len = skb_inner_network_header(skb) - 9124 skb_transport_header(skb); 9125 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 9126 goto out_rm_features; 9127 } 9128 9129 len = skb_inner_network_header_len(skb); 9130 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 9131 goto out_rm_features; 9132 } 9133 9134 return features; 9135 out_rm_features: 9136 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9137 } 9138 9139 static const struct net_device_ops ice_netdev_safe_mode_ops = { 9140 .ndo_open = ice_open, 9141 .ndo_stop = ice_stop, 9142 .ndo_start_xmit = ice_start_xmit, 9143 .ndo_set_mac_address = ice_set_mac_address, 9144 .ndo_validate_addr = eth_validate_addr, 9145 .ndo_change_mtu = ice_change_mtu, 9146 .ndo_get_stats64 = ice_get_stats64, 9147 .ndo_tx_timeout = ice_tx_timeout, 9148 .ndo_bpf = ice_xdp_safe_mode, 9149 }; 9150 9151 static const struct net_device_ops ice_netdev_ops = { 9152 .ndo_open = ice_open, 9153 .ndo_stop = ice_stop, 9154 .ndo_start_xmit = ice_start_xmit, 9155 .ndo_select_queue = ice_select_queue, 9156 .ndo_features_check = ice_features_check, 9157 .ndo_fix_features = ice_fix_features, 9158 .ndo_set_rx_mode = ice_set_rx_mode, 9159 .ndo_set_mac_address = ice_set_mac_address, 9160 .ndo_validate_addr = eth_validate_addr, 9161 .ndo_change_mtu = ice_change_mtu, 9162 .ndo_get_stats64 = ice_get_stats64, 9163 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 9164 .ndo_eth_ioctl = ice_eth_ioctl, 9165 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 9166 .ndo_set_vf_mac = ice_set_vf_mac, 9167 .ndo_get_vf_config = ice_get_vf_cfg, 9168 .ndo_set_vf_trust = ice_set_vf_trust, 9169 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 9170 .ndo_set_vf_link_state = ice_set_vf_link_state, 9171 .ndo_get_vf_stats = ice_get_vf_stats, 9172 .ndo_set_vf_rate = ice_set_vf_bw, 9173 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 9174 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 9175 .ndo_setup_tc = ice_setup_tc, 9176 .ndo_set_features = ice_set_features, 9177 .ndo_bridge_getlink = ice_bridge_getlink, 9178 .ndo_bridge_setlink = ice_bridge_setlink, 9179 .ndo_fdb_add = ice_fdb_add, 9180 .ndo_fdb_del = ice_fdb_del, 9181 #ifdef CONFIG_RFS_ACCEL 9182 .ndo_rx_flow_steer = ice_rx_flow_steer, 9183 #endif 9184 .ndo_tx_timeout = ice_tx_timeout, 9185 .ndo_bpf = ice_xdp, 9186 .ndo_xdp_xmit = ice_xdp_xmit, 9187 .ndo_xsk_wakeup = ice_xsk_wakeup, 9188 }; 9189