1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/ipv6.h> 6 #include <linux/types.h> 7 #include <net/netdev_queues.h> 8 9 #include "fbnic.h" 10 #include "fbnic_netdev.h" 11 #include "fbnic_txrx.h" 12 13 int __fbnic_open(struct fbnic_net *fbn) 14 { 15 struct fbnic_dev *fbd = fbn->fbd; 16 int err; 17 18 err = fbnic_alloc_napi_vectors(fbn); 19 if (err) 20 return err; 21 22 err = fbnic_alloc_resources(fbn); 23 if (err) 24 goto free_napi_vectors; 25 26 err = fbnic_set_netif_queues(fbn); 27 if (err) 28 goto free_resources; 29 30 /* Send ownership message and flush to verify FW has seen it */ 31 err = fbnic_fw_xmit_ownership_msg(fbd, true); 32 if (err) { 33 dev_warn(fbd->dev, 34 "Error %d sending host ownership message to the firmware\n", 35 err); 36 goto err_reset_queues; 37 } 38 39 err = fbnic_time_start(fbn); 40 if (err) 41 goto release_ownership; 42 43 err = fbnic_fw_init_heartbeat(fbd, false); 44 if (err) 45 goto time_stop; 46 47 err = fbnic_mac_request_irq(fbd); 48 if (err) 49 goto time_stop; 50 51 /* Pull the BMC config and initialize the RPC */ 52 fbnic_bmc_rpc_init(fbd); 53 fbnic_rss_reinit(fbd, fbn); 54 55 phylink_resume(fbn->phylink); 56 57 return 0; 58 time_stop: 59 fbnic_time_stop(fbn); 60 release_ownership: 61 fbnic_fw_xmit_ownership_msg(fbn->fbd, false); 62 err_reset_queues: 63 fbnic_reset_netif_queues(fbn); 64 free_resources: 65 fbnic_free_resources(fbn); 66 free_napi_vectors: 67 fbnic_free_napi_vectors(fbn); 68 return err; 69 } 70 71 static int fbnic_open(struct net_device *netdev) 72 { 73 struct fbnic_net *fbn = netdev_priv(netdev); 74 int err; 75 76 fbnic_napi_name_irqs(fbn->fbd); 77 78 err = __fbnic_open(fbn); 79 if (!err) 80 fbnic_up(fbn); 81 82 return err; 83 } 84 85 static int fbnic_stop(struct net_device *netdev) 86 { 87 struct fbnic_net *fbn = netdev_priv(netdev); 88 89 fbnic_mac_free_irq(fbn->fbd); 90 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd)); 91 92 fbnic_down(fbn); 93 94 fbnic_time_stop(fbn); 95 fbnic_fw_xmit_ownership_msg(fbn->fbd, false); 96 97 fbnic_reset_netif_queues(fbn); 98 fbnic_free_resources(fbn); 99 fbnic_free_napi_vectors(fbn); 100 101 return 0; 102 } 103 104 static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr) 105 { 106 struct fbnic_net *fbn = netdev_priv(netdev); 107 struct fbnic_mac_addr *avail_addr; 108 109 if (WARN_ON(!is_valid_ether_addr(addr))) 110 return -EADDRNOTAVAIL; 111 112 avail_addr = __fbnic_uc_sync(fbn->fbd, addr); 113 if (!avail_addr) 114 return -ENOSPC; 115 116 /* Add type flag indicating this address is in use by the host */ 117 set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam); 118 119 return 0; 120 } 121 122 static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr) 123 { 124 struct fbnic_net *fbn = netdev_priv(netdev); 125 struct fbnic_dev *fbd = fbn->fbd; 126 int i, ret; 127 128 /* Scan from middle of list to bottom, filling bottom up. 129 * Skip the first entry which is reserved for dev_addr and 130 * leave the last entry to use for promiscuous filtering. 131 */ 132 for (i = fbd->mac_addr_boundary, ret = -ENOENT; 133 i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) { 134 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; 135 136 if (!ether_addr_equal(mac_addr->value.addr8, addr)) 137 continue; 138 139 ret = __fbnic_uc_unsync(mac_addr); 140 } 141 142 return ret; 143 } 144 145 static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr) 146 { 147 struct fbnic_net *fbn = netdev_priv(netdev); 148 struct fbnic_mac_addr *avail_addr; 149 150 if (WARN_ON(!is_multicast_ether_addr(addr))) 151 return -EADDRNOTAVAIL; 152 153 avail_addr = __fbnic_mc_sync(fbn->fbd, addr); 154 if (!avail_addr) 155 return -ENOSPC; 156 157 /* Add type flag indicating this address is in use by the host */ 158 set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam); 159 160 return 0; 161 } 162 163 static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr) 164 { 165 struct fbnic_net *fbn = netdev_priv(netdev); 166 struct fbnic_dev *fbd = fbn->fbd; 167 int i, ret; 168 169 /* Scan from middle of list to top, filling top down. 170 * Skip over the address reserved for the BMC MAC and 171 * exclude index 0 as that belongs to the broadcast address 172 */ 173 for (i = fbd->mac_addr_boundary, ret = -ENOENT; 174 --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) { 175 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; 176 177 if (!ether_addr_equal(mac_addr->value.addr8, addr)) 178 continue; 179 180 ret = __fbnic_mc_unsync(mac_addr); 181 } 182 183 return ret; 184 } 185 186 void __fbnic_set_rx_mode(struct fbnic_dev *fbd, 187 struct netdev_hw_addr_list *uc, 188 struct netdev_hw_addr_list *mc) 189 { 190 bool uc_promisc = false, mc_promisc = false; 191 struct net_device *netdev = fbd->netdev; 192 struct fbnic_mac_addr *mac_addr; 193 int err; 194 195 /* Populate host address from dev_addr */ 196 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX]; 197 if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) || 198 mac_addr->state != FBNIC_TCAM_S_VALID) { 199 ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr); 200 mac_addr->state = FBNIC_TCAM_S_UPDATE; 201 set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam); 202 } 203 204 /* Populate broadcast address if broadcast is enabled */ 205 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX]; 206 if (netdev->flags & IFF_BROADCAST) { 207 if (!is_broadcast_ether_addr(mac_addr->value.addr8) || 208 mac_addr->state != FBNIC_TCAM_S_VALID) { 209 eth_broadcast_addr(mac_addr->value.addr8); 210 mac_addr->state = FBNIC_TCAM_S_ADD; 211 } 212 set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam); 213 } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { 214 __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST); 215 } 216 217 /* Synchronize unicast and multicast address lists */ 218 err = __hw_addr_sync_dev(uc, netdev, fbnic_uc_sync, fbnic_uc_unsync); 219 if (err == -ENOSPC) 220 uc_promisc = true; 221 err = __hw_addr_sync_dev(mc, netdev, fbnic_mc_sync, fbnic_mc_unsync); 222 if (err == -ENOSPC) 223 mc_promisc = true; 224 225 uc_promisc |= !!(netdev->flags & IFF_PROMISC); 226 mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc; 227 228 /* Update the promiscuous rules */ 229 fbnic_promisc_sync(fbd, uc_promisc, mc_promisc); 230 231 /* Add rules for BMC all multicast if it is enabled */ 232 fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc); 233 234 /* Sift out any unshared BMC rules and place them in BMC only section */ 235 fbnic_sift_macda(fbd); 236 237 /* Write updates to hardware */ 238 fbnic_write_rules(fbd); 239 fbnic_write_macda(fbd); 240 fbnic_write_tce_tcam(fbd); 241 } 242 243 static int fbnic_set_rx_mode(struct net_device *netdev, 244 struct netdev_hw_addr_list *uc, 245 struct netdev_hw_addr_list *mc) 246 { 247 struct fbnic_net *fbn = netdev_priv(netdev); 248 struct fbnic_dev *fbd = fbn->fbd; 249 250 /* No need to update the hardware if we are not running */ 251 if (netif_running(netdev)) 252 __fbnic_set_rx_mode(fbd, uc, mc); 253 254 return 0; 255 } 256 257 static int fbnic_set_mac(struct net_device *netdev, void *p) 258 { 259 struct fbnic_net *fbn = netdev_priv(netdev); 260 struct sockaddr *addr = p; 261 262 if (!is_valid_ether_addr(addr->sa_data)) 263 return -EADDRNOTAVAIL; 264 265 eth_hw_addr_set(netdev, addr->sa_data); 266 267 if (netif_running(netdev)) 268 __fbnic_set_rx_mode(fbn->fbd, &netdev->uc, &netdev->mc); 269 270 return 0; 271 } 272 273 static int fbnic_change_mtu(struct net_device *dev, int new_mtu) 274 { 275 struct fbnic_net *fbn = netdev_priv(dev); 276 277 if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { 278 dev_err(&dev->dev, 279 "MTU %d is larger than HDS threshold %d in XDP mode\n", 280 new_mtu, fbn->hds_thresh); 281 282 return -EINVAL; 283 } 284 285 WRITE_ONCE(dev->mtu, new_mtu); 286 287 return 0; 288 } 289 290 void fbnic_clear_rx_mode(struct fbnic_dev *fbd) 291 { 292 struct net_device *netdev = fbd->netdev; 293 int idx; 294 295 for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { 296 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; 297 298 if (mac_addr->state != FBNIC_TCAM_S_VALID) 299 continue; 300 301 bitmap_clear(mac_addr->act_tcam, 302 FBNIC_MAC_ADDR_T_HOST_START, 303 FBNIC_MAC_ADDR_T_HOST_LEN); 304 305 if (bitmap_empty(mac_addr->act_tcam, 306 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES)) 307 mac_addr->state = FBNIC_TCAM_S_DELETE; 308 } 309 310 /* Write updates to hardware */ 311 fbnic_write_macda(fbd); 312 313 __dev_uc_unsync(netdev, NULL); 314 __dev_mc_unsync(netdev, NULL); 315 } 316 317 static int fbnic_hwtstamp_get(struct net_device *netdev, 318 struct kernel_hwtstamp_config *config) 319 { 320 struct fbnic_net *fbn = netdev_priv(netdev); 321 322 *config = fbn->hwtstamp_config; 323 324 return 0; 325 } 326 327 static int fbnic_hwtstamp_set(struct net_device *netdev, 328 struct kernel_hwtstamp_config *config, 329 struct netlink_ext_ack *extack) 330 { 331 struct fbnic_net *fbn = netdev_priv(netdev); 332 int old_rx_filter; 333 334 if (config->source != HWTSTAMP_SOURCE_NETDEV) 335 return -EOPNOTSUPP; 336 337 if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config)) 338 return 0; 339 340 /* Upscale the filters */ 341 switch (config->rx_filter) { 342 case HWTSTAMP_FILTER_NONE: 343 case HWTSTAMP_FILTER_ALL: 344 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 345 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 346 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 347 case HWTSTAMP_FILTER_PTP_V2_EVENT: 348 break; 349 case HWTSTAMP_FILTER_NTP_ALL: 350 config->rx_filter = HWTSTAMP_FILTER_ALL; 351 break; 352 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 353 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 354 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 355 break; 356 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 357 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 358 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 359 break; 360 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 361 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 362 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; 363 break; 364 case HWTSTAMP_FILTER_PTP_V2_SYNC: 365 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 366 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 367 break; 368 default: 369 return -ERANGE; 370 } 371 372 /* Configure */ 373 old_rx_filter = fbn->hwtstamp_config.rx_filter; 374 memcpy(&fbn->hwtstamp_config, config, sizeof(*config)); 375 376 if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) { 377 fbnic_rss_reinit(fbn->fbd, fbn); 378 fbnic_write_rules(fbn->fbd); 379 } 380 381 /* Save / report back filter configuration 382 * Note that our filter configuration is inexact. Instead of 383 * filtering for a specific UDP port or L2 Ethertype we are 384 * filtering in all UDP or all non-IP packets for timestamping. So 385 * if anything other than FILTER_ALL is requested we report 386 * FILTER_SOME indicating that we will be timestamping a few 387 * additional packets. 388 */ 389 if (config->rx_filter > HWTSTAMP_FILTER_ALL) 390 config->rx_filter = HWTSTAMP_FILTER_SOME; 391 392 return 0; 393 } 394 395 static void fbnic_get_stats64(struct net_device *dev, 396 struct rtnl_link_stats64 *stats64) 397 { 398 u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0; 399 u64 rx_over = 0, rx_missed = 0, rx_length = 0; 400 u64 tx_bytes, tx_packets, tx_dropped = 0; 401 struct fbnic_net *fbn = netdev_priv(dev); 402 struct fbnic_dev *fbd = fbn->fbd; 403 struct fbnic_queue_stats *stats; 404 405 unsigned int start, i; 406 407 fbnic_get_hw_stats(fbd); 408 409 stats = &fbn->tx_stats; 410 411 tx_bytes = stats->bytes; 412 tx_packets = stats->packets; 413 tx_dropped = stats->dropped; 414 415 /* Record drops from Tx HW Datapath */ 416 spin_lock(&fbd->hw_stats.lock); 417 tx_dropped += fbd->hw_stats.tmi.drop.frames.value + 418 fbd->hw_stats.tti.cm_drop.frames.value + 419 fbd->hw_stats.tti.frame_drop.frames.value + 420 fbd->hw_stats.tti.tbi_drop.frames.value; 421 spin_unlock(&fbd->hw_stats.lock); 422 423 stats64->tx_bytes = tx_bytes; 424 stats64->tx_packets = tx_packets; 425 stats64->tx_dropped = tx_dropped; 426 427 for (i = 0; i < fbn->num_tx_queues; i++) { 428 struct fbnic_ring *txr = fbn->tx[i]; 429 430 if (!txr) 431 continue; 432 433 stats = &txr->stats; 434 do { 435 start = u64_stats_fetch_begin(&stats->syncp); 436 tx_bytes = stats->bytes; 437 tx_packets = stats->packets; 438 tx_dropped = stats->dropped; 439 } while (u64_stats_fetch_retry(&stats->syncp, start)); 440 441 stats64->tx_bytes += tx_bytes; 442 stats64->tx_packets += tx_packets; 443 stats64->tx_dropped += tx_dropped; 444 } 445 446 stats = &fbn->rx_stats; 447 448 rx_bytes = stats->bytes; 449 rx_packets = stats->packets; 450 rx_dropped = stats->dropped; 451 452 spin_lock(&fbd->hw_stats.lock); 453 /* Record drops for the host FIFOs. 454 * 4: network to Host, 6: BMC to Host 455 * Exclude the BMC and MC FIFOs as those stats may contain drops 456 * due to unrelated items such as TCAM misses. They are still 457 * accessible through the ethtool stats. 458 */ 459 i = FBNIC_RXB_FIFO_HOST; 460 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; 461 i = FBNIC_RXB_FIFO_BMC_TO_HOST; 462 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; 463 464 for (i = 0; i < fbd->max_num_queues; i++) { 465 /* Report packets dropped due to CQ/BDQ being full/empty */ 466 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value; 467 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value; 468 469 /* Report packets with errors */ 470 rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value; 471 } 472 spin_unlock(&fbd->hw_stats.lock); 473 474 stats64->rx_bytes = rx_bytes; 475 stats64->rx_packets = rx_packets; 476 stats64->rx_dropped = rx_dropped; 477 stats64->rx_over_errors = rx_over; 478 stats64->rx_errors = rx_errors; 479 stats64->rx_missed_errors = rx_missed; 480 481 for (i = 0; i < fbn->num_rx_queues; i++) { 482 struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i]; 483 struct fbnic_ring *rxr = fbn->rx[i]; 484 485 if (!rxr) 486 continue; 487 488 stats = &rxr->stats; 489 do { 490 start = u64_stats_fetch_begin(&stats->syncp); 491 rx_bytes = stats->bytes; 492 rx_packets = stats->packets; 493 rx_dropped = stats->dropped; 494 rx_length = stats->rx.length_errors; 495 } while (u64_stats_fetch_retry(&stats->syncp, start)); 496 497 stats64->rx_bytes += rx_bytes; 498 stats64->rx_packets += rx_packets; 499 stats64->rx_dropped += rx_dropped; 500 stats64->rx_errors += rx_length; 501 stats64->rx_length_errors += rx_length; 502 503 if (!xdpr) 504 continue; 505 506 stats = &xdpr->stats; 507 do { 508 start = u64_stats_fetch_begin(&stats->syncp); 509 tx_bytes = stats->bytes; 510 tx_packets = stats->packets; 511 tx_dropped = stats->dropped; 512 } while (u64_stats_fetch_retry(&stats->syncp, start)); 513 514 stats64->tx_bytes += tx_bytes; 515 stats64->tx_packets += tx_packets; 516 stats64->tx_dropped += tx_dropped; 517 } 518 } 519 520 bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu, 521 u32 hds_thresh) 522 { 523 if (!prog) 524 return false; 525 526 if (prog->aux->xdp_has_frags) 527 return false; 528 529 return mtu + ETH_HLEN > hds_thresh; 530 } 531 532 static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf) 533 { 534 struct bpf_prog *prog = bpf->prog, *prev_prog; 535 struct fbnic_net *fbn = netdev_priv(netdev); 536 537 if (bpf->command != XDP_SETUP_PROG) 538 return -EINVAL; 539 540 if (fbnic_check_split_frames(prog, netdev->mtu, 541 fbn->hds_thresh)) { 542 NL_SET_ERR_MSG_MOD(bpf->extack, 543 "MTU too high, or HDS threshold is too low for single buffer XDP"); 544 return -EOPNOTSUPP; 545 } 546 547 prev_prog = xchg(&fbn->xdp_prog, prog); 548 if (prev_prog) 549 bpf_prog_put(prev_prog); 550 551 return 0; 552 } 553 554 static const struct net_device_ops fbnic_netdev_ops = { 555 .ndo_open = fbnic_open, 556 .ndo_stop = fbnic_stop, 557 .ndo_validate_addr = eth_validate_addr, 558 .ndo_start_xmit = fbnic_xmit_frame, 559 .ndo_features_check = fbnic_features_check, 560 .ndo_set_mac_address = fbnic_set_mac, 561 .ndo_change_mtu = fbnic_change_mtu, 562 .ndo_set_rx_mode_async = fbnic_set_rx_mode, 563 .ndo_get_stats64 = fbnic_get_stats64, 564 .ndo_bpf = fbnic_bpf, 565 .ndo_hwtstamp_get = fbnic_hwtstamp_get, 566 .ndo_hwtstamp_set = fbnic_hwtstamp_set, 567 }; 568 569 static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx, 570 struct netdev_queue_stats_rx *rx) 571 { 572 u64 bytes, packets, alloc_fail, alloc_fail_bdq; 573 struct fbnic_net *fbn = netdev_priv(dev); 574 struct fbnic_ring *rxr = fbn->rx[idx]; 575 struct fbnic_dev *fbd = fbn->fbd; 576 struct fbnic_queue_stats *stats; 577 u64 csum_complete, csum_none; 578 struct fbnic_q_triad *qt; 579 unsigned int start; 580 581 if (!rxr) 582 return; 583 584 /* fbn->rx points to completion queues */ 585 qt = container_of(rxr, struct fbnic_q_triad, cmpl); 586 587 stats = &rxr->stats; 588 do { 589 start = u64_stats_fetch_begin(&stats->syncp); 590 bytes = stats->bytes; 591 packets = stats->packets; 592 alloc_fail = stats->rx.alloc_failed; 593 csum_complete = stats->rx.csum_complete; 594 csum_none = stats->rx.csum_none; 595 } while (u64_stats_fetch_retry(&stats->syncp, start)); 596 597 stats = &qt->sub0.stats; 598 do { 599 start = u64_stats_fetch_begin(&stats->syncp); 600 alloc_fail_bdq = stats->bdq.alloc_failed; 601 } while (u64_stats_fetch_retry(&stats->syncp, start)); 602 alloc_fail += alloc_fail_bdq; 603 604 stats = &qt->sub1.stats; 605 do { 606 start = u64_stats_fetch_begin(&stats->syncp); 607 alloc_fail_bdq = stats->bdq.alloc_failed; 608 } while (u64_stats_fetch_retry(&stats->syncp, start)); 609 alloc_fail += alloc_fail_bdq; 610 611 rx->bytes = bytes; 612 rx->packets = packets; 613 rx->alloc_fail = alloc_fail; 614 rx->csum_complete = csum_complete; 615 rx->csum_none = csum_none; 616 617 fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q); 618 619 spin_lock(&fbd->hw_stats.lock); 620 rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value + 621 fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value; 622 rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value + 623 rx->hw_drop_overruns; 624 spin_unlock(&fbd->hw_stats.lock); 625 } 626 627 static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, 628 struct netdev_queue_stats_tx *tx) 629 { 630 struct fbnic_net *fbn = netdev_priv(dev); 631 struct fbnic_ring *txr = fbn->tx[idx]; 632 struct fbnic_queue_stats *stats; 633 u64 stop, wake, csum, lso; 634 struct fbnic_ring *xdpr; 635 unsigned int start; 636 u64 bytes, packets; 637 638 if (!txr) 639 return; 640 641 stats = &txr->stats; 642 do { 643 start = u64_stats_fetch_begin(&stats->syncp); 644 bytes = stats->bytes; 645 packets = stats->packets; 646 csum = stats->twq.csum_partial; 647 lso = stats->twq.lso; 648 stop = stats->twq.stop; 649 wake = stats->twq.wake; 650 } while (u64_stats_fetch_retry(&stats->syncp, start)); 651 652 tx->bytes = bytes; 653 tx->packets = packets; 654 tx->needs_csum = csum + lso; 655 tx->hw_gso_wire_packets = lso; 656 tx->stop = stop; 657 tx->wake = wake; 658 659 xdpr = fbn->tx[FBNIC_MAX_TXQS + idx]; 660 if (xdpr) { 661 stats = &xdpr->stats; 662 do { 663 start = u64_stats_fetch_begin(&stats->syncp); 664 bytes = stats->bytes; 665 packets = stats->packets; 666 } while (u64_stats_fetch_retry(&stats->syncp, start)); 667 668 tx->bytes += bytes; 669 tx->packets += packets; 670 } 671 } 672 673 static void fbnic_get_base_stats(struct net_device *dev, 674 struct netdev_queue_stats_rx *rx, 675 struct netdev_queue_stats_tx *tx) 676 { 677 struct fbnic_net *fbn = netdev_priv(dev); 678 679 tx->bytes = fbn->tx_stats.bytes; 680 tx->packets = fbn->tx_stats.packets; 681 tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso; 682 tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso; 683 tx->stop = fbn->tx_stats.twq.stop; 684 tx->wake = fbn->tx_stats.twq.wake; 685 686 rx->bytes = fbn->rx_stats.bytes; 687 rx->packets = fbn->rx_stats.packets; 688 rx->alloc_fail = fbn->rx_stats.rx.alloc_failed + 689 fbn->bdq_stats.bdq.alloc_failed; 690 rx->csum_complete = fbn->rx_stats.rx.csum_complete; 691 rx->csum_none = fbn->rx_stats.rx.csum_none; 692 } 693 694 static const struct netdev_stat_ops fbnic_stat_ops = { 695 .get_queue_stats_rx = fbnic_get_queue_stats_rx, 696 .get_queue_stats_tx = fbnic_get_queue_stats_tx, 697 .get_base_stats = fbnic_get_base_stats, 698 }; 699 700 void fbnic_reset_queues(struct fbnic_net *fbn, 701 unsigned int tx, unsigned int rx) 702 { 703 struct fbnic_dev *fbd = fbn->fbd; 704 unsigned int max_napis; 705 706 max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS; 707 708 tx = min(tx, max_napis); 709 fbn->num_tx_queues = tx; 710 711 rx = min(rx, max_napis); 712 fbn->num_rx_queues = rx; 713 714 fbn->num_napi = max(tx, rx); 715 } 716 717 /** 718 * fbnic_netdev_free - Free the netdev associate with fbnic 719 * @fbd: Driver specific structure to free netdev from 720 * 721 * Allocate and initialize the netdev and netdev private structure. Bind 722 * together the hardware, netdev, and pci data structures. 723 **/ 724 void fbnic_netdev_free(struct fbnic_dev *fbd) 725 { 726 fbnic_phylink_destroy(fbd->netdev); 727 728 free_netdev(fbd->netdev); 729 fbd->netdev = NULL; 730 } 731 732 /** 733 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic 734 * @fbd: Driver specific structure to associate netdev with 735 * 736 * Allocate and initialize the netdev and netdev private structure. Bind 737 * together the hardware, netdev, and pci data structures. 738 * 739 * Return: Pointer to net_device on success, NULL on failure 740 **/ 741 struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) 742 { 743 struct net_device *netdev; 744 struct fbnic_net *fbn; 745 int default_queues; 746 747 netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS); 748 if (!netdev) 749 return NULL; 750 751 SET_NETDEV_DEV(netdev, fbd->dev); 752 fbd->netdev = netdev; 753 754 netdev->netdev_ops = &fbnic_netdev_ops; 755 netdev->stat_ops = &fbnic_stat_ops; 756 netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; 757 netdev->netmem_tx = NETMEM_TX_DMA; 758 759 fbnic_set_ethtool_ops(netdev); 760 761 fbn = netdev_priv(netdev); 762 763 fbn->netdev = netdev; 764 fbn->fbd = fbd; 765 766 fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT; 767 fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT; 768 fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT; 769 fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT; 770 771 fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT; 772 fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; 773 fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; 774 775 /* Initialize the hds_thresh */ 776 netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; 777 fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; 778 779 default_queues = netif_get_num_default_rss_queues(); 780 if (default_queues > fbd->max_num_queues) 781 default_queues = fbd->max_num_queues; 782 783 fbnic_reset_queues(fbn, default_queues, default_queues); 784 785 fbnic_reset_indir_tbl(fbn); 786 fbnic_rss_key_fill(fbn->rss_key); 787 fbnic_rss_init_en_mask(fbn); 788 789 netdev->priv_flags |= IFF_UNICAST_FLT; 790 791 netdev->gso_partial_features = 792 NETIF_F_GSO_GRE | 793 NETIF_F_GSO_GRE_CSUM | 794 NETIF_F_GSO_IPXIP4 | 795 NETIF_F_GSO_UDP_TUNNEL | 796 NETIF_F_GSO_UDP_TUNNEL_CSUM; 797 798 netdev->features |= 799 netdev->gso_partial_features | 800 FBNIC_TUN_GSO_FEATURES | 801 NETIF_F_RXHASH | 802 NETIF_F_SG | 803 NETIF_F_HW_CSUM | 804 NETIF_F_RXCSUM | 805 NETIF_F_TSO | 806 NETIF_F_TSO_ECN | 807 NETIF_F_TSO6 | 808 NETIF_F_GSO_PARTIAL | 809 NETIF_F_GSO_UDP_L4; 810 811 netdev->hw_features |= netdev->features; 812 netdev->vlan_features |= netdev->features; 813 netdev->hw_enc_features |= netdev->features; 814 netdev->features |= NETIF_F_NTUPLE; 815 816 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; 817 818 netdev->min_mtu = IPV6_MIN_MTU; 819 netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; 820 821 /* TBD: This is workaround for BMC as phylink doesn't have support 822 * for leavling the link enabled if a BMC is present. 823 */ 824 netdev->ethtool->wol_enabled = true; 825 826 netif_carrier_off(netdev); 827 828 netif_tx_stop_all_queues(netdev); 829 830 if (fbnic_phylink_create(netdev)) { 831 free_netdev(netdev); 832 fbd->netdev = NULL; 833 return NULL; 834 } 835 836 return netdev; 837 } 838 839 static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr) 840 { 841 addr[0] = (dsn >> 56) & 0xFF; 842 addr[1] = (dsn >> 48) & 0xFF; 843 addr[2] = (dsn >> 40) & 0xFF; 844 addr[3] = (dsn >> 16) & 0xFF; 845 addr[4] = (dsn >> 8) & 0xFF; 846 addr[5] = dsn & 0xFF; 847 848 return is_valid_ether_addr(addr) ? 0 : -EINVAL; 849 } 850 851 /** 852 * fbnic_netdev_register - Initialize general software structures 853 * @netdev: Netdev containing structure to initialize and register 854 * 855 * Initialize the MAC address for the netdev and register it. 856 * 857 * Return: 0 on success, negative on failure 858 **/ 859 int fbnic_netdev_register(struct net_device *netdev) 860 { 861 struct fbnic_net *fbn = netdev_priv(netdev); 862 struct fbnic_dev *fbd = fbn->fbd; 863 u64 dsn = fbd->dsn; 864 u8 addr[ETH_ALEN]; 865 int err; 866 867 err = fbnic_dsn_to_mac_addr(dsn, addr); 868 if (!err) { 869 ether_addr_copy(netdev->perm_addr, addr); 870 eth_hw_addr_set(netdev, addr); 871 } else { 872 /* A randomly assigned MAC address will cause provisioning 873 * issues so instead just fail to spawn the netdev and 874 * avoid any confusion. 875 */ 876 dev_err(fbd->dev, "MAC addr %pM invalid\n", addr); 877 return err; 878 } 879 880 return register_netdev(netdev); 881 } 882 883 void fbnic_netdev_unregister(struct net_device *netdev) 884 { 885 unregister_netdev(netdev); 886 } 887