1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/ipv6.h> 6 #include <linux/types.h> 7 #include <net/netdev_queues.h> 8 9 #include "fbnic.h" 10 #include "fbnic_netdev.h" 11 #include "fbnic_txrx.h" 12 13 int __fbnic_open(struct fbnic_net *fbn) 14 { 15 struct fbnic_dev *fbd = fbn->fbd; 16 int err; 17 18 err = fbnic_alloc_napi_vectors(fbn); 19 if (err) 20 return err; 21 22 err = fbnic_alloc_resources(fbn); 23 if (err) 24 goto free_napi_vectors; 25 26 err = fbnic_set_netif_queues(fbn); 27 if (err) 28 goto free_resources; 29 30 /* Send ownership message and flush to verify FW has seen it */ 31 err = fbnic_fw_xmit_ownership_msg(fbd, true); 32 if (err) { 33 dev_warn(fbd->dev, 34 "Error %d sending host ownership message to the firmware\n", 35 err); 36 goto err_reset_queues; 37 } 38 39 err = fbnic_time_start(fbn); 40 if (err) 41 goto release_ownership; 42 43 err = fbnic_fw_init_heartbeat(fbd, false); 44 if (err) 45 goto time_stop; 46 47 err = fbnic_mac_request_irq(fbd); 48 if (err) 49 goto time_stop; 50 51 /* Pull the BMC config and initialize the RPC */ 52 fbnic_bmc_rpc_init(fbd); 53 fbnic_rss_reinit(fbd, fbn); 54 55 phylink_resume(fbn->phylink); 56 57 return 0; 58 time_stop: 59 fbnic_time_stop(fbn); 60 release_ownership: 61 fbnic_fw_xmit_ownership_msg(fbn->fbd, false); 62 err_reset_queues: 63 fbnic_reset_netif_queues(fbn); 64 free_resources: 65 fbnic_free_resources(fbn); 66 free_napi_vectors: 67 fbnic_free_napi_vectors(fbn); 68 return err; 69 } 70 71 static int fbnic_open(struct net_device *netdev) 72 { 73 struct fbnic_net *fbn = netdev_priv(netdev); 74 int err; 75 76 fbnic_napi_name_irqs(fbn->fbd); 77 78 err = __fbnic_open(fbn); 79 if (!err) 80 fbnic_up(fbn); 81 82 return err; 83 } 84 85 static int fbnic_stop(struct net_device *netdev) 86 { 87 struct fbnic_net *fbn = netdev_priv(netdev); 88 89 fbnic_mac_free_irq(fbn->fbd); 90 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd)); 91 92 fbnic_down(fbn); 93 94 fbnic_time_stop(fbn); 95 fbnic_fw_xmit_ownership_msg(fbn->fbd, false); 96 97 fbnic_reset_netif_queues(fbn); 98 fbnic_free_resources(fbn); 99 fbnic_free_napi_vectors(fbn); 100 101 return 0; 102 } 103 104 static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr) 105 { 106 struct fbnic_net *fbn = netdev_priv(netdev); 107 struct fbnic_mac_addr *avail_addr; 108 109 if (WARN_ON(!is_valid_ether_addr(addr))) 110 return -EADDRNOTAVAIL; 111 112 avail_addr = __fbnic_uc_sync(fbn->fbd, addr); 113 if (!avail_addr) 114 return -ENOSPC; 115 116 /* Add type flag indicating this address is in use by the host */ 117 set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam); 118 119 return 0; 120 } 121 122 static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr) 123 { 124 struct fbnic_net *fbn = netdev_priv(netdev); 125 struct fbnic_dev *fbd = fbn->fbd; 126 int i, ret; 127 128 /* Scan from middle of list to bottom, filling bottom up. 129 * Skip the first entry which is reserved for dev_addr and 130 * leave the last entry to use for promiscuous filtering. 131 */ 132 for (i = fbd->mac_addr_boundary, ret = -ENOENT; 133 i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) { 134 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; 135 136 if (!ether_addr_equal(mac_addr->value.addr8, addr)) 137 continue; 138 139 ret = __fbnic_uc_unsync(mac_addr); 140 } 141 142 return ret; 143 } 144 145 static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr) 146 { 147 struct fbnic_net *fbn = netdev_priv(netdev); 148 struct fbnic_mac_addr *avail_addr; 149 150 if (WARN_ON(!is_multicast_ether_addr(addr))) 151 return -EADDRNOTAVAIL; 152 153 avail_addr = __fbnic_mc_sync(fbn->fbd, addr); 154 if (!avail_addr) 155 return -ENOSPC; 156 157 /* Add type flag indicating this address is in use by the host */ 158 set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam); 159 160 return 0; 161 } 162 163 static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr) 164 { 165 struct fbnic_net *fbn = netdev_priv(netdev); 166 struct fbnic_dev *fbd = fbn->fbd; 167 int i, ret; 168 169 /* Scan from middle of list to top, filling top down. 170 * Skip over the address reserved for the BMC MAC and 171 * exclude index 0 as that belongs to the broadcast address 172 */ 173 for (i = fbd->mac_addr_boundary, ret = -ENOENT; 174 --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) { 175 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; 176 177 if (!ether_addr_equal(mac_addr->value.addr8, addr)) 178 continue; 179 180 ret = __fbnic_mc_unsync(mac_addr); 181 } 182 183 return ret; 184 } 185 186 void __fbnic_set_rx_mode(struct fbnic_dev *fbd, 187 struct netdev_hw_addr_list *uc, 188 struct netdev_hw_addr_list *mc) 189 { 190 bool uc_promisc = false, mc_promisc = false; 191 struct net_device *netdev = fbd->netdev; 192 struct fbnic_mac_addr *mac_addr; 193 int err; 194 195 /* Populate host address from dev_addr */ 196 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX]; 197 if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) || 198 mac_addr->state != FBNIC_TCAM_S_VALID) { 199 ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr); 200 mac_addr->state = FBNIC_TCAM_S_UPDATE; 201 set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam); 202 } 203 204 /* Populate broadcast address if broadcast is enabled */ 205 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX]; 206 if (netdev->flags & IFF_BROADCAST) { 207 if (!is_broadcast_ether_addr(mac_addr->value.addr8) || 208 mac_addr->state != FBNIC_TCAM_S_VALID) { 209 eth_broadcast_addr(mac_addr->value.addr8); 210 mac_addr->state = FBNIC_TCAM_S_ADD; 211 } 212 set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam); 213 } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { 214 __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST); 215 } 216 217 /* Synchronize unicast and multicast address lists */ 218 err = __hw_addr_sync_dev(uc, netdev, fbnic_uc_sync, fbnic_uc_unsync); 219 if (err == -ENOSPC) 220 uc_promisc = true; 221 err = __hw_addr_sync_dev(mc, netdev, fbnic_mc_sync, fbnic_mc_unsync); 222 if (err == -ENOSPC) 223 mc_promisc = true; 224 225 uc_promisc |= !!(netdev->flags & IFF_PROMISC); 226 mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc; 227 228 /* Update the promiscuous rules */ 229 fbnic_promisc_sync(fbd, uc_promisc, mc_promisc); 230 231 /* Add rules for BMC all multicast if it is enabled */ 232 fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc); 233 234 /* Sift out any unshared BMC rules and place them in BMC only section */ 235 fbnic_sift_macda(fbd); 236 237 /* Write updates to hardware */ 238 fbnic_write_rules(fbd); 239 fbnic_write_macda(fbd); 240 fbnic_write_tce_tcam(fbd); 241 } 242 243 static int fbnic_set_rx_mode(struct net_device *netdev, 244 struct netdev_hw_addr_list *uc, 245 struct netdev_hw_addr_list *mc) 246 { 247 struct fbnic_net *fbn = netdev_priv(netdev); 248 struct fbnic_dev *fbd = fbn->fbd; 249 250 /* No need to update the hardware if we are not running */ 251 if (netif_running(netdev)) 252 __fbnic_set_rx_mode(fbd, uc, mc); 253 254 return 0; 255 } 256 257 static int fbnic_set_mac(struct net_device *netdev, void *p) 258 { 259 struct fbnic_net *fbn = netdev_priv(netdev); 260 struct sockaddr *addr = p; 261 262 if (!is_valid_ether_addr(addr->sa_data)) 263 return -EADDRNOTAVAIL; 264 265 eth_hw_addr_set(netdev, addr->sa_data); 266 267 if (netif_running(netdev)) { 268 netif_addr_lock_bh(netdev); 269 __fbnic_set_rx_mode(fbn->fbd, &netdev->uc, &netdev->mc); 270 netif_addr_unlock_bh(netdev); 271 } 272 273 return 0; 274 } 275 276 static int fbnic_change_mtu(struct net_device *dev, int new_mtu) 277 { 278 struct fbnic_net *fbn = netdev_priv(dev); 279 280 if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { 281 dev_err(&dev->dev, 282 "MTU %d is larger than HDS threshold %d in XDP mode\n", 283 new_mtu, fbn->hds_thresh); 284 285 return -EINVAL; 286 } 287 288 WRITE_ONCE(dev->mtu, new_mtu); 289 290 return 0; 291 } 292 293 void fbnic_clear_rx_mode(struct fbnic_dev *fbd) 294 { 295 struct net_device *netdev = fbd->netdev; 296 int idx; 297 298 for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { 299 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; 300 301 if (mac_addr->state != FBNIC_TCAM_S_VALID) 302 continue; 303 304 bitmap_clear(mac_addr->act_tcam, 305 FBNIC_MAC_ADDR_T_HOST_START, 306 FBNIC_MAC_ADDR_T_HOST_LEN); 307 308 if (bitmap_empty(mac_addr->act_tcam, 309 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES)) 310 mac_addr->state = FBNIC_TCAM_S_DELETE; 311 } 312 313 /* Write updates to hardware */ 314 fbnic_write_macda(fbd); 315 316 netif_addr_lock_bh(netdev); 317 __dev_uc_unsync(netdev, NULL); 318 __dev_mc_unsync(netdev, NULL); 319 netif_addr_unlock_bh(netdev); 320 } 321 322 static int fbnic_hwtstamp_get(struct net_device *netdev, 323 struct kernel_hwtstamp_config *config) 324 { 325 struct fbnic_net *fbn = netdev_priv(netdev); 326 327 *config = fbn->hwtstamp_config; 328 329 return 0; 330 } 331 332 static int fbnic_hwtstamp_set(struct net_device *netdev, 333 struct kernel_hwtstamp_config *config, 334 struct netlink_ext_ack *extack) 335 { 336 struct fbnic_net *fbn = netdev_priv(netdev); 337 int old_rx_filter; 338 339 if (config->source != HWTSTAMP_SOURCE_NETDEV) 340 return -EOPNOTSUPP; 341 342 if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config)) 343 return 0; 344 345 /* Upscale the filters */ 346 switch (config->rx_filter) { 347 case HWTSTAMP_FILTER_NONE: 348 case HWTSTAMP_FILTER_ALL: 349 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 350 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 351 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 352 case HWTSTAMP_FILTER_PTP_V2_EVENT: 353 break; 354 case HWTSTAMP_FILTER_NTP_ALL: 355 config->rx_filter = HWTSTAMP_FILTER_ALL; 356 break; 357 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 358 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 359 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 360 break; 361 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 362 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 363 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 364 break; 365 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 366 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 367 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; 368 break; 369 case HWTSTAMP_FILTER_PTP_V2_SYNC: 370 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 371 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 372 break; 373 default: 374 return -ERANGE; 375 } 376 377 /* Configure */ 378 old_rx_filter = fbn->hwtstamp_config.rx_filter; 379 memcpy(&fbn->hwtstamp_config, config, sizeof(*config)); 380 381 if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) { 382 fbnic_rss_reinit(fbn->fbd, fbn); 383 fbnic_write_rules(fbn->fbd); 384 } 385 386 /* Save / report back filter configuration 387 * Note that our filter configuration is inexact. Instead of 388 * filtering for a specific UDP port or L2 Ethertype we are 389 * filtering in all UDP or all non-IP packets for timestamping. So 390 * if anything other than FILTER_ALL is requested we report 391 * FILTER_SOME indicating that we will be timestamping a few 392 * additional packets. 393 */ 394 if (config->rx_filter > HWTSTAMP_FILTER_ALL) 395 config->rx_filter = HWTSTAMP_FILTER_SOME; 396 397 return 0; 398 } 399 400 static void fbnic_get_stats64(struct net_device *dev, 401 struct rtnl_link_stats64 *stats64) 402 { 403 u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0; 404 u64 rx_over = 0, rx_missed = 0, rx_length = 0; 405 u64 tx_bytes, tx_packets, tx_dropped = 0; 406 struct fbnic_net *fbn = netdev_priv(dev); 407 struct fbnic_dev *fbd = fbn->fbd; 408 struct fbnic_queue_stats *stats; 409 410 unsigned int start, i; 411 412 fbnic_get_hw_stats(fbd); 413 414 stats = &fbn->tx_stats; 415 416 tx_bytes = stats->bytes; 417 tx_packets = stats->packets; 418 tx_dropped = stats->dropped; 419 420 /* Record drops from Tx HW Datapath */ 421 spin_lock(&fbd->hw_stats.lock); 422 tx_dropped += fbd->hw_stats.tmi.drop.frames.value + 423 fbd->hw_stats.tti.cm_drop.frames.value + 424 fbd->hw_stats.tti.frame_drop.frames.value + 425 fbd->hw_stats.tti.tbi_drop.frames.value; 426 spin_unlock(&fbd->hw_stats.lock); 427 428 stats64->tx_bytes = tx_bytes; 429 stats64->tx_packets = tx_packets; 430 stats64->tx_dropped = tx_dropped; 431 432 for (i = 0; i < fbn->num_tx_queues; i++) { 433 struct fbnic_ring *txr = fbn->tx[i]; 434 435 if (!txr) 436 continue; 437 438 stats = &txr->stats; 439 do { 440 start = u64_stats_fetch_begin(&stats->syncp); 441 tx_bytes = stats->bytes; 442 tx_packets = stats->packets; 443 tx_dropped = stats->dropped; 444 } while (u64_stats_fetch_retry(&stats->syncp, start)); 445 446 stats64->tx_bytes += tx_bytes; 447 stats64->tx_packets += tx_packets; 448 stats64->tx_dropped += tx_dropped; 449 } 450 451 stats = &fbn->rx_stats; 452 453 rx_bytes = stats->bytes; 454 rx_packets = stats->packets; 455 rx_dropped = stats->dropped; 456 457 spin_lock(&fbd->hw_stats.lock); 458 /* Record drops for the host FIFOs. 459 * 4: network to Host, 6: BMC to Host 460 * Exclude the BMC and MC FIFOs as those stats may contain drops 461 * due to unrelated items such as TCAM misses. They are still 462 * accessible through the ethtool stats. 463 */ 464 i = FBNIC_RXB_FIFO_HOST; 465 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; 466 i = FBNIC_RXB_FIFO_BMC_TO_HOST; 467 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; 468 469 for (i = 0; i < fbd->max_num_queues; i++) { 470 /* Report packets dropped due to CQ/BDQ being full/empty */ 471 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value; 472 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value; 473 474 /* Report packets with errors */ 475 rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value; 476 } 477 spin_unlock(&fbd->hw_stats.lock); 478 479 stats64->rx_bytes = rx_bytes; 480 stats64->rx_packets = rx_packets; 481 stats64->rx_dropped = rx_dropped; 482 stats64->rx_over_errors = rx_over; 483 stats64->rx_errors = rx_errors; 484 stats64->rx_missed_errors = rx_missed; 485 486 for (i = 0; i < fbn->num_rx_queues; i++) { 487 struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i]; 488 struct fbnic_ring *rxr = fbn->rx[i]; 489 490 if (!rxr) 491 continue; 492 493 stats = &rxr->stats; 494 do { 495 start = u64_stats_fetch_begin(&stats->syncp); 496 rx_bytes = stats->bytes; 497 rx_packets = stats->packets; 498 rx_dropped = stats->dropped; 499 rx_length = stats->rx.length_errors; 500 } while (u64_stats_fetch_retry(&stats->syncp, start)); 501 502 stats64->rx_bytes += rx_bytes; 503 stats64->rx_packets += rx_packets; 504 stats64->rx_dropped += rx_dropped; 505 stats64->rx_errors += rx_length; 506 stats64->rx_length_errors += rx_length; 507 508 if (!xdpr) 509 continue; 510 511 stats = &xdpr->stats; 512 do { 513 start = u64_stats_fetch_begin(&stats->syncp); 514 tx_bytes = stats->bytes; 515 tx_packets = stats->packets; 516 tx_dropped = stats->dropped; 517 } while (u64_stats_fetch_retry(&stats->syncp, start)); 518 519 stats64->tx_bytes += tx_bytes; 520 stats64->tx_packets += tx_packets; 521 stats64->tx_dropped += tx_dropped; 522 } 523 } 524 525 bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu, 526 u32 hds_thresh) 527 { 528 if (!prog) 529 return false; 530 531 if (prog->aux->xdp_has_frags) 532 return false; 533 534 return mtu + ETH_HLEN > hds_thresh; 535 } 536 537 static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf) 538 { 539 struct bpf_prog *prog = bpf->prog, *prev_prog; 540 struct fbnic_net *fbn = netdev_priv(netdev); 541 542 if (bpf->command != XDP_SETUP_PROG) 543 return -EINVAL; 544 545 if (fbnic_check_split_frames(prog, netdev->mtu, 546 fbn->hds_thresh)) { 547 NL_SET_ERR_MSG_MOD(bpf->extack, 548 "MTU too high, or HDS threshold is too low for single buffer XDP"); 549 return -EOPNOTSUPP; 550 } 551 552 prev_prog = xchg(&fbn->xdp_prog, prog); 553 if (prev_prog) 554 bpf_prog_put(prev_prog); 555 556 return 0; 557 } 558 559 static const struct net_device_ops fbnic_netdev_ops = { 560 .ndo_open = fbnic_open, 561 .ndo_stop = fbnic_stop, 562 .ndo_validate_addr = eth_validate_addr, 563 .ndo_start_xmit = fbnic_xmit_frame, 564 .ndo_features_check = fbnic_features_check, 565 .ndo_set_mac_address = fbnic_set_mac, 566 .ndo_change_mtu = fbnic_change_mtu, 567 .ndo_set_rx_mode_async = fbnic_set_rx_mode, 568 .ndo_get_stats64 = fbnic_get_stats64, 569 .ndo_bpf = fbnic_bpf, 570 .ndo_hwtstamp_get = fbnic_hwtstamp_get, 571 .ndo_hwtstamp_set = fbnic_hwtstamp_set, 572 }; 573 574 static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx, 575 struct netdev_queue_stats_rx *rx) 576 { 577 u64 bytes, packets, alloc_fail, alloc_fail_bdq; 578 struct fbnic_net *fbn = netdev_priv(dev); 579 struct fbnic_ring *rxr = fbn->rx[idx]; 580 struct fbnic_dev *fbd = fbn->fbd; 581 struct fbnic_queue_stats *stats; 582 u64 csum_complete, csum_none; 583 struct fbnic_q_triad *qt; 584 unsigned int start; 585 586 if (!rxr) 587 return; 588 589 /* fbn->rx points to completion queues */ 590 qt = container_of(rxr, struct fbnic_q_triad, cmpl); 591 592 stats = &rxr->stats; 593 do { 594 start = u64_stats_fetch_begin(&stats->syncp); 595 bytes = stats->bytes; 596 packets = stats->packets; 597 alloc_fail = stats->rx.alloc_failed; 598 csum_complete = stats->rx.csum_complete; 599 csum_none = stats->rx.csum_none; 600 } while (u64_stats_fetch_retry(&stats->syncp, start)); 601 602 stats = &qt->sub0.stats; 603 do { 604 start = u64_stats_fetch_begin(&stats->syncp); 605 alloc_fail_bdq = stats->bdq.alloc_failed; 606 } while (u64_stats_fetch_retry(&stats->syncp, start)); 607 alloc_fail += alloc_fail_bdq; 608 609 stats = &qt->sub1.stats; 610 do { 611 start = u64_stats_fetch_begin(&stats->syncp); 612 alloc_fail_bdq = stats->bdq.alloc_failed; 613 } while (u64_stats_fetch_retry(&stats->syncp, start)); 614 alloc_fail += alloc_fail_bdq; 615 616 rx->bytes = bytes; 617 rx->packets = packets; 618 rx->alloc_fail = alloc_fail; 619 rx->csum_complete = csum_complete; 620 rx->csum_none = csum_none; 621 622 fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q); 623 624 spin_lock(&fbd->hw_stats.lock); 625 rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value + 626 fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value; 627 rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value + 628 rx->hw_drop_overruns; 629 spin_unlock(&fbd->hw_stats.lock); 630 } 631 632 static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, 633 struct netdev_queue_stats_tx *tx) 634 { 635 struct fbnic_net *fbn = netdev_priv(dev); 636 struct fbnic_ring *txr = fbn->tx[idx]; 637 struct fbnic_queue_stats *stats; 638 u64 stop, wake, csum, lso; 639 struct fbnic_ring *xdpr; 640 unsigned int start; 641 u64 bytes, packets; 642 643 if (!txr) 644 return; 645 646 stats = &txr->stats; 647 do { 648 start = u64_stats_fetch_begin(&stats->syncp); 649 bytes = stats->bytes; 650 packets = stats->packets; 651 csum = stats->twq.csum_partial; 652 lso = stats->twq.lso; 653 stop = stats->twq.stop; 654 wake = stats->twq.wake; 655 } while (u64_stats_fetch_retry(&stats->syncp, start)); 656 657 tx->bytes = bytes; 658 tx->packets = packets; 659 tx->needs_csum = csum + lso; 660 tx->hw_gso_wire_packets = lso; 661 tx->stop = stop; 662 tx->wake = wake; 663 664 xdpr = fbn->tx[FBNIC_MAX_TXQS + idx]; 665 if (xdpr) { 666 stats = &xdpr->stats; 667 do { 668 start = u64_stats_fetch_begin(&stats->syncp); 669 bytes = stats->bytes; 670 packets = stats->packets; 671 } while (u64_stats_fetch_retry(&stats->syncp, start)); 672 673 tx->bytes += bytes; 674 tx->packets += packets; 675 } 676 } 677 678 static void fbnic_get_base_stats(struct net_device *dev, 679 struct netdev_queue_stats_rx *rx, 680 struct netdev_queue_stats_tx *tx) 681 { 682 struct fbnic_net *fbn = netdev_priv(dev); 683 684 tx->bytes = fbn->tx_stats.bytes; 685 tx->packets = fbn->tx_stats.packets; 686 tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso; 687 tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso; 688 tx->stop = fbn->tx_stats.twq.stop; 689 tx->wake = fbn->tx_stats.twq.wake; 690 691 rx->bytes = fbn->rx_stats.bytes; 692 rx->packets = fbn->rx_stats.packets; 693 rx->alloc_fail = fbn->rx_stats.rx.alloc_failed + 694 fbn->bdq_stats.bdq.alloc_failed; 695 rx->csum_complete = fbn->rx_stats.rx.csum_complete; 696 rx->csum_none = fbn->rx_stats.rx.csum_none; 697 } 698 699 static const struct netdev_stat_ops fbnic_stat_ops = { 700 .get_queue_stats_rx = fbnic_get_queue_stats_rx, 701 .get_queue_stats_tx = fbnic_get_queue_stats_tx, 702 .get_base_stats = fbnic_get_base_stats, 703 }; 704 705 void fbnic_reset_queues(struct fbnic_net *fbn, 706 unsigned int tx, unsigned int rx) 707 { 708 struct fbnic_dev *fbd = fbn->fbd; 709 unsigned int max_napis; 710 711 max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS; 712 713 tx = min(tx, max_napis); 714 fbn->num_tx_queues = tx; 715 716 rx = min(rx, max_napis); 717 fbn->num_rx_queues = rx; 718 719 fbn->num_napi = max(tx, rx); 720 } 721 722 /** 723 * fbnic_netdev_free - Free the netdev associate with fbnic 724 * @fbd: Driver specific structure to free netdev from 725 * 726 * Allocate and initialize the netdev and netdev private structure. Bind 727 * together the hardware, netdev, and pci data structures. 728 **/ 729 void fbnic_netdev_free(struct fbnic_dev *fbd) 730 { 731 fbnic_phylink_destroy(fbd->netdev); 732 733 free_netdev(fbd->netdev); 734 fbd->netdev = NULL; 735 } 736 737 /** 738 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic 739 * @fbd: Driver specific structure to associate netdev with 740 * 741 * Allocate and initialize the netdev and netdev private structure. Bind 742 * together the hardware, netdev, and pci data structures. 743 * 744 * Return: Pointer to net_device on success, NULL on failure 745 **/ 746 struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) 747 { 748 struct net_device *netdev; 749 struct fbnic_net *fbn; 750 int default_queues; 751 752 netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS); 753 if (!netdev) 754 return NULL; 755 756 SET_NETDEV_DEV(netdev, fbd->dev); 757 fbd->netdev = netdev; 758 759 netdev->netdev_ops = &fbnic_netdev_ops; 760 netdev->stat_ops = &fbnic_stat_ops; 761 netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; 762 netdev->netmem_tx = NETMEM_TX_DMA; 763 764 fbnic_set_ethtool_ops(netdev); 765 766 fbn = netdev_priv(netdev); 767 768 fbn->netdev = netdev; 769 fbn->fbd = fbd; 770 771 fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT; 772 fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT; 773 fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT; 774 fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT; 775 776 fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT; 777 fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; 778 fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; 779 780 /* Initialize the hds_thresh */ 781 netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; 782 fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; 783 784 default_queues = netif_get_num_default_rss_queues(); 785 if (default_queues > fbd->max_num_queues) 786 default_queues = fbd->max_num_queues; 787 788 fbnic_reset_queues(fbn, default_queues, default_queues); 789 790 fbnic_reset_indir_tbl(fbn); 791 fbnic_rss_key_fill(fbn->rss_key); 792 fbnic_rss_init_en_mask(fbn); 793 794 netdev->priv_flags |= IFF_UNICAST_FLT; 795 796 netdev->gso_partial_features = 797 NETIF_F_GSO_GRE | 798 NETIF_F_GSO_GRE_CSUM | 799 NETIF_F_GSO_IPXIP4 | 800 NETIF_F_GSO_UDP_TUNNEL | 801 NETIF_F_GSO_UDP_TUNNEL_CSUM; 802 803 netdev->features |= 804 netdev->gso_partial_features | 805 FBNIC_TUN_GSO_FEATURES | 806 NETIF_F_RXHASH | 807 NETIF_F_SG | 808 NETIF_F_HW_CSUM | 809 NETIF_F_RXCSUM | 810 NETIF_F_TSO | 811 NETIF_F_TSO_ECN | 812 NETIF_F_TSO6 | 813 NETIF_F_GSO_PARTIAL | 814 NETIF_F_GSO_UDP_L4; 815 816 netdev->hw_features |= netdev->features; 817 netdev->vlan_features |= netdev->features; 818 netdev->hw_enc_features |= netdev->features; 819 netdev->features |= NETIF_F_NTUPLE; 820 821 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; 822 823 netdev->min_mtu = IPV6_MIN_MTU; 824 netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; 825 826 /* TBD: This is workaround for BMC as phylink doesn't have support 827 * for leavling the link enabled if a BMC is present. 828 */ 829 netdev->ethtool->wol_enabled = true; 830 831 netif_carrier_off(netdev); 832 833 netif_tx_stop_all_queues(netdev); 834 835 if (fbnic_phylink_create(netdev)) { 836 free_netdev(netdev); 837 fbd->netdev = NULL; 838 return NULL; 839 } 840 841 return netdev; 842 } 843 844 static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr) 845 { 846 addr[0] = (dsn >> 56) & 0xFF; 847 addr[1] = (dsn >> 48) & 0xFF; 848 addr[2] = (dsn >> 40) & 0xFF; 849 addr[3] = (dsn >> 16) & 0xFF; 850 addr[4] = (dsn >> 8) & 0xFF; 851 addr[5] = dsn & 0xFF; 852 853 return is_valid_ether_addr(addr) ? 0 : -EINVAL; 854 } 855 856 /** 857 * fbnic_netdev_register - Initialize general software structures 858 * @netdev: Netdev containing structure to initialize and register 859 * 860 * Initialize the MAC address for the netdev and register it. 861 * 862 * Return: 0 on success, negative on failure 863 **/ 864 int fbnic_netdev_register(struct net_device *netdev) 865 { 866 struct fbnic_net *fbn = netdev_priv(netdev); 867 struct fbnic_dev *fbd = fbn->fbd; 868 u64 dsn = fbd->dsn; 869 u8 addr[ETH_ALEN]; 870 int err; 871 872 err = fbnic_dsn_to_mac_addr(dsn, addr); 873 if (!err) { 874 ether_addr_copy(netdev->perm_addr, addr); 875 eth_hw_addr_set(netdev, addr); 876 } else { 877 /* A randomly assigned MAC address will cause provisioning 878 * issues so instead just fail to spawn the netdev and 879 * avoid any confusion. 880 */ 881 dev_err(fbd->dev, "MAC addr %pM invalid\n", addr); 882 return err; 883 } 884 885 return register_netdev(netdev); 886 } 887 888 void fbnic_netdev_unregister(struct net_device *netdev) 889 { 890 unregister_netdev(netdev); 891 } 892