1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <asm/byteorder.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/dmapool.h> 7 #include <linux/delay.h> 8 #include <linux/errno.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/netdevice.h> 13 #include <net/netdev_lock.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if.h> 18 #include <net/ip.h> 19 #include <linux/skbuff.h> 20 #include <net/page_pool/helpers.h> 21 22 #include "bnge.h" 23 #include "bnge_hwrm.h" 24 #include "bnge_hwrm_lib.h" 25 #include "bnge_ethtool.h" 26 #include "bnge_rmem.h" 27 #include "bnge_txrx.h" 28 29 #define BNGE_RING_TO_TC_OFF(bd, tx) \ 30 ((tx) % (bd)->tx_nr_rings_per_tc) 31 32 #define BNGE_RING_TO_TC(bd, tx) \ 33 ((tx) / (bd)->tx_nr_rings_per_tc) 34 35 #define BNGE_TC_TO_RING_BASE(bd, tc) \ 36 ((tc) * (bd)->tx_nr_rings_per_tc) 37 38 static void bnge_free_stats_mem(struct bnge_net *bn, 39 struct bnge_stats_mem *stats) 40 { 41 struct bnge_dev *bd = bn->bd; 42 43 kfree(stats->hw_masks); 44 stats->hw_masks = NULL; 45 kfree(stats->sw_stats); 46 stats->sw_stats = NULL; 47 if (stats->hw_stats) { 48 dma_free_coherent(bd->dev, stats->len, stats->hw_stats, 49 stats->hw_stats_map); 50 stats->hw_stats = NULL; 51 } 52 } 53 54 static int bnge_alloc_stats_mem(struct bnge_net *bn, 55 struct bnge_stats_mem *stats, bool alloc_masks) 56 { 57 struct bnge_dev *bd = bn->bd; 58 59 stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len, 60 &stats->hw_stats_map, GFP_KERNEL); 61 if (!stats->hw_stats) 62 return -ENOMEM; 63 64 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 65 if (!stats->sw_stats) 66 goto stats_mem_err; 67 68 if (alloc_masks) { 69 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 70 if (!stats->hw_masks) 71 goto stats_mem_err; 72 } 73 74 u64_stats_init(&stats->syncp); 75 76 return 0; 77 78 stats_mem_err: 79 bnge_free_stats_mem(bn, stats); 80 return -ENOMEM; 81 } 82 83 static void bnge_free_ring_stats(struct bnge_net *bn) 84 { 85 struct bnge_dev *bd = bn->bd; 86 int i; 87 88 if (!bn->bnapi) 89 return; 90 91 for (i = 0; i < bd->nq_nr_rings; i++) { 92 struct bnge_napi *bnapi = bn->bnapi[i]; 93 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 94 95 bnge_free_stats_mem(bn, &nqr->stats); 96 } 97 } 98 99 static void bnge_fill_masks(u64 *mask_arr, u64 mask, int count) 100 { 101 int i; 102 103 for (i = 0; i < count; i++) 104 mask_arr[i] = mask; 105 } 106 107 void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 108 { 109 int i; 110 111 for (i = 0; i < count; i++) 112 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 113 } 114 115 static void bnge_init_stats(struct bnge_net *bn) 116 { 117 struct bnge_napi *bnapi = bn->bnapi[0]; 118 struct bnge_nq_ring_info *nqr; 119 struct bnge_stats_mem *stats; 120 struct bnge_dev *bd = bn->bd; 121 __le64 *rx_stats, *tx_stats; 122 int rc, rx_count, tx_count; 123 u64 *rx_masks, *tx_masks; 124 u8 flags; 125 126 nqr = &bnapi->nq_ring; 127 stats = &nqr->stats; 128 rc = bnge_hwrm_func_qstat_ext(bd, stats); 129 if (rc) { 130 u64 mask = (1ULL << 48) - 1; 131 132 bnge_fill_masks(stats->hw_masks, mask, stats->len / 8); 133 } 134 135 if (bn->flags & BNGE_FLAG_PORT_STATS) { 136 stats = &bn->port_stats; 137 rx_stats = stats->hw_stats; 138 rx_masks = stats->hw_masks; 139 rx_count = sizeof(struct rx_port_stats) / 8; 140 tx_stats = rx_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 141 tx_masks = rx_masks + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 142 tx_count = sizeof(struct tx_port_stats) / 8; 143 144 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 145 rc = bnge_hwrm_port_qstats(bd, flags); 146 if (rc) { 147 u64 mask = (1ULL << 40) - 1; 148 149 bnge_fill_masks(rx_masks, mask, rx_count); 150 bnge_fill_masks(tx_masks, mask, tx_count); 151 } else { 152 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 153 bnge_copy_hw_masks(tx_masks, tx_stats, tx_count); 154 } 155 bnge_hwrm_port_qstats(bd, 0); 156 } 157 158 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 159 stats = &bn->rx_port_stats_ext; 160 rx_stats = stats->hw_stats; 161 rx_masks = stats->hw_masks; 162 rx_count = sizeof(struct rx_port_stats_ext) / 8; 163 stats = &bn->tx_port_stats_ext; 164 tx_stats = stats->hw_stats; 165 tx_masks = stats->hw_masks; 166 tx_count = sizeof(struct tx_port_stats_ext) / 8; 167 168 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 169 rc = bnge_hwrm_port_qstats_ext(bd, flags); 170 if (rc) { 171 u64 mask = (1ULL << 40) - 1; 172 173 bnge_fill_masks(rx_masks, mask, rx_count); 174 if (tx_stats) 175 bnge_fill_masks(tx_masks, mask, tx_count); 176 } else { 177 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 178 if (tx_stats) 179 bnge_copy_hw_masks(tx_masks, tx_stats, 180 tx_count); 181 } 182 bnge_hwrm_port_qstats_ext(bd, 0); 183 } 184 } 185 186 static void bnge_free_port_ext_stats(struct bnge_net *bn) 187 { 188 bn->flags &= ~BNGE_FLAG_PORT_STATS_EXT; 189 bnge_free_stats_mem(bn, &bn->rx_port_stats_ext); 190 bnge_free_stats_mem(bn, &bn->tx_port_stats_ext); 191 } 192 193 static void bnge_free_port_stats(struct bnge_net *bn) 194 { 195 bn->flags &= ~BNGE_FLAG_PORT_STATS; 196 bnge_free_stats_mem(bn, &bn->port_stats); 197 bnge_free_port_ext_stats(bn); 198 } 199 200 static int bnge_alloc_ring_stats(struct bnge_net *bn) 201 { 202 struct bnge_dev *bd = bn->bd; 203 u32 size, i; 204 int rc; 205 206 size = bd->hw_ring_stats_size; 207 208 for (i = 0; i < bd->nq_nr_rings; i++) { 209 struct bnge_napi *bnapi = bn->bnapi[i]; 210 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 211 212 nqr->stats.len = size; 213 rc = bnge_alloc_stats_mem(bn, &nqr->stats, !i); 214 if (rc) 215 goto err_free_ring_stats; 216 217 nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 218 } 219 220 return 0; 221 222 err_free_ring_stats: 223 bnge_free_ring_stats(bn); 224 return rc; 225 } 226 227 static void bnge_alloc_port_ext_stats(struct bnge_net *bn) 228 { 229 struct bnge_dev *bd = bn->bd; 230 int rc; 231 232 if (!(bd->fw_cap & BNGE_FW_CAP_EXT_STATS_SUPPORTED)) 233 return; 234 235 if (!bn->rx_port_stats_ext.hw_stats) { 236 bn->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 237 rc = bnge_alloc_stats_mem(bn, &bn->rx_port_stats_ext, true); 238 /* Extended stats are optional */ 239 if (rc) 240 return; 241 } 242 243 if (!bn->tx_port_stats_ext.hw_stats) { 244 bn->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 245 rc = bnge_alloc_stats_mem(bn, &bn->tx_port_stats_ext, true); 246 /* Extended stats are optional */ 247 if (rc) { 248 bnge_free_port_ext_stats(bn); 249 return; 250 } 251 } 252 bn->flags |= BNGE_FLAG_PORT_STATS_EXT; 253 } 254 255 static int bnge_alloc_port_stats(struct bnge_net *bn) 256 { 257 int rc; 258 259 if (!bn->port_stats.hw_stats) { 260 bn->port_stats.len = BNGE_PORT_STATS_SIZE; 261 rc = bnge_alloc_stats_mem(bn, &bn->port_stats, true); 262 if (rc) 263 return rc; 264 265 bn->flags |= BNGE_FLAG_PORT_STATS; 266 } 267 268 bnge_alloc_port_ext_stats(bn); 269 return 0; 270 } 271 272 void __bnge_queue_sp_work(struct bnge_net *bn) 273 { 274 queue_work(bn->bnge_pf_wq, &bn->sp_task); 275 } 276 277 static void bnge_queue_sp_work(struct bnge_net *bn, unsigned int event) 278 { 279 set_bit(event, &bn->sp_event); 280 __bnge_queue_sp_work(bn); 281 } 282 283 static void bnge_timer(struct timer_list *t) 284 { 285 struct bnge_net *bn = timer_container_of(bn, t, timer); 286 struct bnge_dev *bd = bn->bd; 287 288 if (!netif_running(bn->netdev) || 289 !test_bit(BNGE_STATE_OPEN, &bd->state)) 290 return; 291 292 if (READ_ONCE(bd->link_info.phy_retry)) { 293 if (time_after(jiffies, READ_ONCE(bd->link_info.phy_retry_expires))) { 294 WRITE_ONCE(bd->link_info.phy_retry, false); 295 netdev_warn(bn->netdev, "failed to update PHY settings after maximum retries.\n"); 296 } else { 297 bnge_queue_sp_work(bn, BNGE_UPDATE_PHY_SP_EVENT); 298 } 299 } 300 301 if (BNGE_LINK_IS_UP(bd) && bn->stats_coal_ticks) 302 bnge_queue_sp_work(bn, BNGE_PERIODIC_STATS_SP_EVENT); 303 304 mod_timer(&bn->timer, jiffies + bn->current_interval); 305 } 306 307 static void bnge_add_one_ctr(u64 hw, u64 *sw, u64 mask) 308 { 309 u64 sw_tmp, sw_val; 310 311 hw &= mask; 312 sw_val = READ_ONCE(*sw); 313 sw_tmp = (sw_val & ~mask) | hw; 314 if (hw < (sw_val & mask)) 315 sw_tmp += mask + 1; 316 WRITE_ONCE(*sw, sw_tmp); 317 } 318 319 static void __bnge_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 320 int count, struct u64_stats_sync *syncp) 321 { 322 unsigned long flags; 323 int i; 324 325 flags = u64_stats_update_begin_irqsave(syncp); 326 for (i = 0; i < count; i++) { 327 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 328 329 if (masks[i] == -1ULL) 330 WRITE_ONCE(sw_stats[i], hw); 331 else 332 bnge_add_one_ctr(hw, &sw_stats[i], masks[i]); 333 } 334 u64_stats_update_end_irqrestore(syncp, flags); 335 } 336 337 static void bnge_accumulate_stats(struct bnge_stats_mem *stats) 338 { 339 if (!stats->hw_stats) 340 return; 341 342 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 343 stats->hw_masks, stats->len / 8, &stats->syncp); 344 } 345 346 static void bnge_accumulate_all_stats(struct bnge_dev *bd) 347 { 348 struct bnge_net *bn = netdev_priv(bd->netdev); 349 struct bnge_stats_mem *ring0_stats = NULL; 350 int i; 351 352 for (i = 0; i < bd->nq_nr_rings; i++) { 353 struct bnge_napi *bnapi = bn->bnapi[i]; 354 struct bnge_nq_ring_info *nqr; 355 struct bnge_stats_mem *stats; 356 357 nqr = &bnapi->nq_ring; 358 stats = &nqr->stats; 359 360 if (!ring0_stats) 361 ring0_stats = &bn->bnapi[0]->nq_ring.stats; 362 363 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 364 ring0_stats->hw_masks, 365 ring0_stats->len / 8, &stats->syncp); 366 } 367 368 if (bn->flags & BNGE_FLAG_PORT_STATS) { 369 struct bnge_stats_mem *stats = &bn->port_stats; 370 __le64 *hw_stats = stats->hw_stats; 371 u64 *sw_stats = stats->sw_stats; 372 u64 *masks = stats->hw_masks; 373 u16 cnt; 374 375 cnt = sizeof(struct rx_port_stats) / 8; 376 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 377 &stats->syncp); 378 379 hw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 380 sw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 381 masks += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 382 cnt = sizeof(struct tx_port_stats) / 8; 383 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 384 &stats->syncp); 385 } 386 387 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 388 bnge_accumulate_stats(&bn->rx_port_stats_ext); 389 bnge_accumulate_stats(&bn->tx_port_stats_ext); 390 } 391 } 392 393 static void bnge_sp_task(struct work_struct *work) 394 { 395 struct bnge_net *bn = container_of(work, struct bnge_net, sp_task); 396 bool speed_chng, cfg_chng, link_chng; 397 struct bnge_dev *bd = bn->bd; 398 399 netdev_lock(bn->netdev); 400 if (!test_bit(BNGE_STATE_OPEN, &bd->state)) { 401 netdev_unlock(bn->netdev); 402 return; 403 } 404 405 if (test_and_clear_bit(BNGE_PERIODIC_STATS_SP_EVENT, &bn->sp_event)) { 406 bnge_hwrm_port_qstats(bd, 0); 407 bnge_hwrm_port_qstats_ext(bd, 0); 408 bnge_accumulate_all_stats(bd); 409 } 410 411 if (test_and_clear_bit(BNGE_UPDATE_PHY_SP_EVENT, &bn->sp_event)) { 412 int rc; 413 414 rc = bnge_update_phy_setting(bn); 415 if (rc) { 416 netdev_warn(bn->netdev, "update PHY settings retry failed\n"); 417 } else { 418 WRITE_ONCE(bd->link_info.phy_retry, false); 419 netdev_info(bn->netdev, "update PHY settings retry succeeded\n"); 420 } 421 } 422 423 speed_chng = test_and_clear_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, 424 &bn->sp_event); 425 cfg_chng = test_and_clear_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, 426 &bn->sp_event); 427 link_chng = test_and_clear_bit(BNGE_LINK_CHNG_SP_EVENT, 428 &bn->sp_event); 429 430 if (speed_chng || cfg_chng || link_chng) { 431 int rc; 432 433 if (speed_chng) 434 bnge_hwrm_phy_qcaps(bd); 435 436 rc = bnge_update_link(bn, true); 437 if (rc) 438 netdev_err(bn->netdev, "SP task cannot update link (rc: %d)\n", 439 rc); 440 441 if (speed_chng || cfg_chng) 442 bnge_init_ethtool_link_settings(bn); 443 } 444 445 netdev_unlock(bn->netdev); 446 } 447 448 static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr) 449 { 450 struct bnge_ring_struct *ring = &nqr->ring_struct; 451 452 kfree(nqr->desc_ring); 453 nqr->desc_ring = NULL; 454 ring->ring_mem.pg_arr = NULL; 455 kfree(nqr->desc_mapping); 456 nqr->desc_mapping = NULL; 457 ring->ring_mem.dma_arr = NULL; 458 } 459 460 static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr) 461 { 462 struct bnge_ring_struct *ring = &cpr->ring_struct; 463 464 kfree(cpr->desc_ring); 465 cpr->desc_ring = NULL; 466 ring->ring_mem.pg_arr = NULL; 467 kfree(cpr->desc_mapping); 468 cpr->desc_mapping = NULL; 469 ring->ring_mem.dma_arr = NULL; 470 } 471 472 static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n) 473 { 474 nqr->desc_ring = kzalloc_objs(*nqr->desc_ring, n); 475 if (!nqr->desc_ring) 476 return -ENOMEM; 477 478 nqr->desc_mapping = kzalloc_objs(*nqr->desc_mapping, n); 479 if (!nqr->desc_mapping) 480 goto err_free_desc_ring; 481 return 0; 482 483 err_free_desc_ring: 484 kfree(nqr->desc_ring); 485 nqr->desc_ring = NULL; 486 return -ENOMEM; 487 } 488 489 static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n) 490 { 491 cpr->desc_ring = kzalloc_objs(*cpr->desc_ring, n); 492 if (!cpr->desc_ring) 493 return -ENOMEM; 494 495 cpr->desc_mapping = kzalloc_objs(*cpr->desc_mapping, n); 496 if (!cpr->desc_mapping) 497 goto err_free_desc_ring; 498 return 0; 499 500 err_free_desc_ring: 501 kfree(cpr->desc_ring); 502 cpr->desc_ring = NULL; 503 return -ENOMEM; 504 } 505 506 static void bnge_free_nq_arrays(struct bnge_net *bn) 507 { 508 struct bnge_dev *bd = bn->bd; 509 int i; 510 511 for (i = 0; i < bd->nq_nr_rings; i++) { 512 struct bnge_napi *bnapi = bn->bnapi[i]; 513 514 bnge_free_nq_desc_arr(&bnapi->nq_ring); 515 } 516 } 517 518 static int bnge_alloc_nq_arrays(struct bnge_net *bn) 519 { 520 struct bnge_dev *bd = bn->bd; 521 int i, rc; 522 523 for (i = 0; i < bd->nq_nr_rings; i++) { 524 struct bnge_napi *bnapi = bn->bnapi[i]; 525 526 rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages); 527 if (rc) 528 goto err_free_nq_arrays; 529 } 530 return 0; 531 532 err_free_nq_arrays: 533 bnge_free_nq_arrays(bn); 534 return rc; 535 } 536 537 static void bnge_free_nq_tree(struct bnge_net *bn) 538 { 539 struct bnge_dev *bd = bn->bd; 540 int i; 541 542 for (i = 0; i < bd->nq_nr_rings; i++) { 543 struct bnge_napi *bnapi = bn->bnapi[i]; 544 struct bnge_nq_ring_info *nqr; 545 struct bnge_ring_struct *ring; 546 int j; 547 548 nqr = &bnapi->nq_ring; 549 ring = &nqr->ring_struct; 550 551 bnge_free_ring(bd, &ring->ring_mem); 552 553 if (!nqr->cp_ring_arr) 554 continue; 555 556 for (j = 0; j < nqr->cp_ring_count; j++) { 557 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 558 559 ring = &cpr->ring_struct; 560 bnge_free_ring(bd, &ring->ring_mem); 561 bnge_free_cp_desc_arr(cpr); 562 } 563 kfree(nqr->cp_ring_arr); 564 nqr->cp_ring_arr = NULL; 565 nqr->cp_ring_count = 0; 566 } 567 } 568 569 static int alloc_one_cp_ring(struct bnge_net *bn, 570 struct bnge_cp_ring_info *cpr) 571 { 572 struct bnge_ring_mem_info *rmem; 573 struct bnge_ring_struct *ring; 574 struct bnge_dev *bd = bn->bd; 575 int rc; 576 577 rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages); 578 if (rc) 579 return -ENOMEM; 580 ring = &cpr->ring_struct; 581 rmem = &ring->ring_mem; 582 rmem->nr_pages = bn->cp_nr_pages; 583 rmem->page_size = HW_CMPD_RING_SIZE; 584 rmem->pg_arr = (void **)cpr->desc_ring; 585 rmem->dma_arr = cpr->desc_mapping; 586 rmem->flags = BNGE_RMEM_RING_PTE_FLAG; 587 rc = bnge_alloc_ring(bd, rmem); 588 if (rc) 589 goto err_free_cp_desc_arr; 590 return rc; 591 592 err_free_cp_desc_arr: 593 bnge_free_cp_desc_arr(cpr); 594 return rc; 595 } 596 597 static int bnge_alloc_nq_tree(struct bnge_net *bn) 598 { 599 struct bnge_dev *bd = bn->bd; 600 int i, j, ulp_msix, rc; 601 int tcs = 1; 602 603 ulp_msix = bnge_aux_get_msix(bd); 604 for (i = 0, j = 0; i < bd->nq_nr_rings; i++) { 605 bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL); 606 struct bnge_napi *bnapi = bn->bnapi[i]; 607 struct bnge_nq_ring_info *nqr; 608 struct bnge_cp_ring_info *cpr; 609 struct bnge_ring_struct *ring; 610 int cp_count = 0, k; 611 int rx = 0, tx = 0; 612 613 nqr = &bnapi->nq_ring; 614 nqr->bnapi = bnapi; 615 ring = &nqr->ring_struct; 616 617 rc = bnge_alloc_ring(bd, &ring->ring_mem); 618 if (rc) 619 goto err_free_nq_tree; 620 621 ring->map_idx = ulp_msix + i; 622 623 if (i < bd->rx_nr_rings) { 624 cp_count++; 625 rx = 1; 626 } 627 628 if ((sh && i < bd->tx_nr_rings) || 629 (!sh && i >= bd->rx_nr_rings)) { 630 cp_count += tcs; 631 tx = 1; 632 } 633 634 nqr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); 635 if (!nqr->cp_ring_arr) { 636 rc = -ENOMEM; 637 goto err_free_nq_tree; 638 } 639 640 nqr->cp_ring_count = cp_count; 641 642 for (k = 0; k < cp_count; k++) { 643 cpr = &nqr->cp_ring_arr[k]; 644 rc = alloc_one_cp_ring(bn, cpr); 645 if (rc) 646 goto err_free_nq_tree; 647 648 cpr->bnapi = bnapi; 649 cpr->cp_idx = k; 650 if (!k && rx) { 651 bn->rx_ring[i].rx_cpr = cpr; 652 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX; 653 } else { 654 int n, tc = k - rx; 655 656 n = BNGE_TC_TO_RING_BASE(bd, tc) + j; 657 bn->tx_ring[n].tx_cpr = cpr; 658 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX; 659 } 660 } 661 if (tx) 662 j++; 663 } 664 return 0; 665 666 err_free_nq_tree: 667 bnge_free_nq_tree(bn); 668 return rc; 669 } 670 671 static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr) 672 { 673 return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE; 674 } 675 676 static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn, 677 struct bnge_rx_ring_info *rxr) 678 { 679 int i, max_idx; 680 681 if (!rxr->rx_buf_ring) 682 return; 683 684 max_idx = bn->rx_nr_pages * RX_DESC_CNT; 685 686 for (i = 0; i < max_idx; i++) { 687 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 688 void *data = rx_buf->data; 689 690 if (!data) 691 continue; 692 693 rx_buf->data = NULL; 694 page_pool_free_va(rxr->head_pool, data, true); 695 } 696 } 697 698 static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn, 699 struct bnge_rx_ring_info *rxr) 700 { 701 int i, max_idx; 702 703 if (!rxr->rx_agg_buf_ring) 704 return; 705 706 max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT; 707 708 for (i = 0; i < max_idx; i++) { 709 struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i]; 710 netmem_ref netmem = rx_agg_buf->netmem; 711 712 if (!netmem) 713 continue; 714 715 rx_agg_buf->netmem = 0; 716 __clear_bit(i, rxr->rx_agg_bmap); 717 718 page_pool_recycle_direct_netmem(rxr->page_pool, netmem); 719 } 720 } 721 722 static void bnge_free_one_tpa_info_data(struct bnge_net *bn, 723 struct bnge_rx_ring_info *rxr) 724 { 725 int i; 726 727 for (i = 0; i < bn->max_tpa; i++) { 728 struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[i]; 729 u8 *data = tpa_info->data; 730 731 if (!data) 732 continue; 733 734 tpa_info->data = NULL; 735 page_pool_free_va(rxr->head_pool, data, false); 736 } 737 } 738 739 static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn, 740 struct bnge_rx_ring_info *rxr) 741 { 742 struct bnge_tpa_idx_map *map; 743 744 if (rxr->rx_tpa) 745 bnge_free_one_tpa_info_data(bn, rxr); 746 747 bnge_free_one_rx_ring_bufs(bn, rxr); 748 bnge_free_one_agg_ring_bufs(bn, rxr); 749 750 map = rxr->rx_tpa_idx_map; 751 if (map) 752 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 753 } 754 755 static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn) 756 { 757 struct bnge_dev *bd = bn->bd; 758 int i; 759 760 if (!bn->rx_ring) 761 return; 762 763 for (i = 0; i < bd->rx_nr_rings; i++) 764 bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]); 765 } 766 767 static void bnge_free_tx_skbs(struct bnge_net *bn) 768 { 769 struct bnge_dev *bd = bn->bd; 770 u16 max_idx; 771 int i; 772 773 max_idx = bn->tx_nr_pages * TX_DESC_CNT; 774 for (i = 0; i < bd->tx_nr_rings; i++) { 775 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 776 int j; 777 778 if (!txr->tx_buf_ring) 779 continue; 780 781 for (j = 0; j < max_idx;) { 782 struct bnge_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 783 struct sk_buff *skb; 784 int k, last; 785 786 skb = tx_buf->skb; 787 if (!skb) { 788 j++; 789 continue; 790 } 791 792 tx_buf->skb = NULL; 793 794 dma_unmap_single(bd->dev, 795 dma_unmap_addr(tx_buf, mapping), 796 skb_headlen(skb), 797 DMA_TO_DEVICE); 798 799 last = tx_buf->nr_frags; 800 j += 2; 801 for (k = 0; k < last; k++, j++) { 802 int ring_idx = j & bn->tx_ring_mask; 803 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 804 805 tx_buf = &txr->tx_buf_ring[ring_idx]; 806 dma_unmap_page(bd->dev, 807 dma_unmap_addr(tx_buf, mapping), 808 skb_frag_size(frag), 809 DMA_TO_DEVICE); 810 } 811 dev_kfree_skb(skb); 812 } 813 netdev_tx_reset_queue(netdev_get_tx_queue(bd->netdev, i)); 814 } 815 } 816 817 static void bnge_free_all_rings_bufs(struct bnge_net *bn) 818 { 819 bnge_free_rx_ring_pair_bufs(bn); 820 bnge_free_tx_skbs(bn); 821 } 822 823 static void bnge_free_tpa_info(struct bnge_net *bn) 824 { 825 struct bnge_dev *bd = bn->bd; 826 int i, j; 827 828 for (i = 0; i < bd->rx_nr_rings; i++) { 829 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 830 831 kfree(rxr->rx_tpa_idx_map); 832 rxr->rx_tpa_idx_map = NULL; 833 if (rxr->rx_tpa) { 834 for (j = 0; j < bn->max_tpa; j++) { 835 kfree(rxr->rx_tpa[j].agg_arr); 836 rxr->rx_tpa[j].agg_arr = NULL; 837 } 838 } 839 kfree(rxr->rx_tpa); 840 rxr->rx_tpa = NULL; 841 } 842 } 843 844 static int bnge_alloc_tpa_info(struct bnge_net *bn) 845 { 846 struct bnge_dev *bd = bn->bd; 847 int i, j; 848 849 if (!bd->max_tpa_v2) 850 return 0; 851 852 bn->max_tpa = max_t(u16, bd->max_tpa_v2, MAX_TPA); 853 for (i = 0; i < bd->rx_nr_rings; i++) { 854 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 855 856 rxr->rx_tpa = kzalloc_objs(struct bnge_tpa_info, bn->max_tpa); 857 if (!rxr->rx_tpa) 858 goto err_free_tpa_info; 859 860 for (j = 0; j < bn->max_tpa; j++) { 861 struct rx_agg_cmp *agg; 862 863 agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); 864 if (!agg) 865 goto err_free_tpa_info; 866 rxr->rx_tpa[j].agg_arr = agg; 867 } 868 rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); 869 if (!rxr->rx_tpa_idx_map) 870 goto err_free_tpa_info; 871 } 872 return 0; 873 874 err_free_tpa_info: 875 bnge_free_tpa_info(bn); 876 return -ENOMEM; 877 } 878 879 static void bnge_free_rx_rings(struct bnge_net *bn) 880 { 881 struct bnge_dev *bd = bn->bd; 882 int i; 883 884 bnge_free_tpa_info(bn); 885 for (i = 0; i < bd->rx_nr_rings; i++) { 886 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 887 struct bnge_ring_struct *ring; 888 889 page_pool_destroy(rxr->page_pool); 890 page_pool_destroy(rxr->head_pool); 891 rxr->page_pool = rxr->head_pool = NULL; 892 893 kfree(rxr->rx_agg_bmap); 894 rxr->rx_agg_bmap = NULL; 895 896 ring = &rxr->rx_ring_struct; 897 bnge_free_ring(bd, &ring->ring_mem); 898 899 ring = &rxr->rx_agg_ring_struct; 900 bnge_free_ring(bd, &ring->ring_mem); 901 } 902 } 903 904 static int bnge_alloc_rx_page_pool(struct bnge_net *bn, 905 struct bnge_rx_ring_info *rxr, 906 int numa_node) 907 { 908 const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE; 909 const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; 910 struct page_pool_params pp = { 0 }; 911 struct bnge_dev *bd = bn->bd; 912 struct page_pool *pool; 913 914 pp.pool_size = bn->rx_agg_ring_size / agg_size_fac; 915 pp.nid = numa_node; 916 pp.netdev = bn->netdev; 917 pp.dev = bd->dev; 918 pp.dma_dir = bn->rx_dir; 919 pp.max_len = PAGE_SIZE; 920 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | 921 PP_FLAG_ALLOW_UNREADABLE_NETMEM; 922 pp.queue_idx = rxr->bnapi->index; 923 924 pool = page_pool_create(&pp); 925 if (IS_ERR(pool)) 926 return PTR_ERR(pool); 927 rxr->page_pool = pool; 928 929 rxr->need_head_pool = page_pool_is_unreadable(pool); 930 if (bnge_separate_head_pool(rxr)) { 931 pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024); 932 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 933 pool = page_pool_create(&pp); 934 if (IS_ERR(pool)) 935 goto err_destroy_pp; 936 } else { 937 page_pool_get(pool); 938 } 939 rxr->head_pool = pool; 940 return 0; 941 942 err_destroy_pp: 943 page_pool_destroy(rxr->page_pool); 944 rxr->page_pool = NULL; 945 return PTR_ERR(pool); 946 } 947 948 static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr) 949 { 950 page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi); 951 page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); 952 } 953 954 static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn, 955 struct bnge_rx_ring_info *rxr) 956 { 957 u16 mem_size; 958 959 rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1; 960 mem_size = rxr->rx_agg_bmap_size / 8; 961 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 962 if (!rxr->rx_agg_bmap) 963 return -ENOMEM; 964 965 return 0; 966 } 967 968 static int bnge_alloc_rx_rings(struct bnge_net *bn) 969 { 970 int i, rc = 0, agg_rings = 0, cpu; 971 struct bnge_dev *bd = bn->bd; 972 973 if (bnge_is_agg_reqd(bd)) 974 agg_rings = 1; 975 976 for (i = 0; i < bd->rx_nr_rings; i++) { 977 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 978 struct bnge_ring_struct *ring; 979 int cpu_node; 980 981 ring = &rxr->rx_ring_struct; 982 983 cpu = cpumask_local_spread(i, dev_to_node(bd->dev)); 984 cpu_node = cpu_to_node(cpu); 985 netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", 986 i, cpu_node); 987 rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node); 988 if (rc) 989 goto err_free_rx_rings; 990 bnge_enable_rx_page_pool(rxr); 991 992 rc = bnge_alloc_ring(bd, &ring->ring_mem); 993 if (rc) 994 goto err_free_rx_rings; 995 996 ring->grp_idx = i; 997 if (agg_rings) { 998 ring = &rxr->rx_agg_ring_struct; 999 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1000 if (rc) 1001 goto err_free_rx_rings; 1002 1003 ring->grp_idx = i; 1004 rc = bnge_alloc_rx_agg_bmap(bn, rxr); 1005 if (rc) 1006 goto err_free_rx_rings; 1007 } 1008 } 1009 1010 if (bn->priv_flags & BNGE_NET_EN_TPA) { 1011 rc = bnge_alloc_tpa_info(bn); 1012 if (rc) 1013 goto err_free_rx_rings; 1014 } 1015 return rc; 1016 1017 err_free_rx_rings: 1018 bnge_free_rx_rings(bn); 1019 return rc; 1020 } 1021 1022 static void bnge_free_tx_rings(struct bnge_net *bn) 1023 { 1024 struct bnge_dev *bd = bn->bd; 1025 int i; 1026 1027 for (i = 0; i < bd->tx_nr_rings; i++) { 1028 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1029 struct bnge_ring_struct *ring; 1030 1031 ring = &txr->tx_ring_struct; 1032 1033 bnge_free_ring(bd, &ring->ring_mem); 1034 } 1035 } 1036 1037 static int bnge_alloc_tx_rings(struct bnge_net *bn) 1038 { 1039 struct bnge_dev *bd = bn->bd; 1040 int i, j, rc; 1041 1042 for (i = 0, j = 0; i < bd->tx_nr_rings; i++) { 1043 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1044 struct bnge_ring_struct *ring; 1045 u8 qidx; 1046 1047 ring = &txr->tx_ring_struct; 1048 1049 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1050 if (rc) 1051 goto err_free_tx_rings; 1052 1053 ring->grp_idx = txr->bnapi->index; 1054 qidx = bd->tc_to_qidx[j]; 1055 ring->queue_id = bd->q_info[qidx].queue_id; 1056 if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1)) 1057 j++; 1058 } 1059 return 0; 1060 1061 err_free_tx_rings: 1062 bnge_free_tx_rings(bn); 1063 return rc; 1064 } 1065 1066 static void bnge_free_vnic_attributes(struct bnge_net *bn) 1067 { 1068 struct pci_dev *pdev = bn->bd->pdev; 1069 struct bnge_vnic_info *vnic; 1070 int i; 1071 1072 if (!bn->vnic_info) 1073 return; 1074 1075 for (i = 0; i < bn->nr_vnics; i++) { 1076 vnic = &bn->vnic_info[i]; 1077 1078 kfree(vnic->uc_list); 1079 vnic->uc_list = NULL; 1080 1081 if (vnic->mc_list) { 1082 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 1083 vnic->mc_list, vnic->mc_list_mapping); 1084 vnic->mc_list = NULL; 1085 } 1086 1087 if (vnic->rss_table) { 1088 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 1089 vnic->rss_table, 1090 vnic->rss_table_dma_addr); 1091 vnic->rss_table = NULL; 1092 } 1093 1094 vnic->rss_hash_key = NULL; 1095 vnic->flags = 0; 1096 } 1097 } 1098 1099 static int bnge_alloc_vnic_attributes(struct bnge_net *bn) 1100 { 1101 struct bnge_dev *bd = bn->bd; 1102 struct bnge_vnic_info *vnic; 1103 int i, size; 1104 1105 for (i = 0; i < bn->nr_vnics; i++) { 1106 vnic = &bn->vnic_info[i]; 1107 1108 if (vnic->flags & BNGE_VNIC_UCAST_FLAG) { 1109 int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN; 1110 1111 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 1112 if (!vnic->uc_list) 1113 goto err_free_vnic_attributes; 1114 } 1115 1116 if (vnic->flags & BNGE_VNIC_MCAST_FLAG) { 1117 vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN; 1118 vnic->mc_list = 1119 dma_alloc_coherent(bd->dev, 1120 vnic->mc_list_size, 1121 &vnic->mc_list_mapping, 1122 GFP_KERNEL); 1123 if (!vnic->mc_list) 1124 goto err_free_vnic_attributes; 1125 } 1126 1127 /* Allocate rss table and hash key */ 1128 size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE); 1129 1130 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 1131 vnic->rss_table = dma_alloc_coherent(bd->dev, 1132 vnic->rss_table_size, 1133 &vnic->rss_table_dma_addr, 1134 GFP_KERNEL); 1135 if (!vnic->rss_table) 1136 goto err_free_vnic_attributes; 1137 1138 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 1139 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 1140 } 1141 return 0; 1142 1143 err_free_vnic_attributes: 1144 bnge_free_vnic_attributes(bn); 1145 return -ENOMEM; 1146 } 1147 1148 static int bnge_alloc_vnics(struct bnge_net *bn) 1149 { 1150 int num_vnics; 1151 1152 /* Allocate only 1 VNIC for now 1153 * Additional VNICs will be added based on RFS/NTUPLE in future patches 1154 */ 1155 num_vnics = 1; 1156 1157 bn->vnic_info = kzalloc_objs(struct bnge_vnic_info, num_vnics); 1158 if (!bn->vnic_info) 1159 return -ENOMEM; 1160 1161 bn->nr_vnics = num_vnics; 1162 1163 return 0; 1164 } 1165 1166 static void bnge_free_vnics(struct bnge_net *bn) 1167 { 1168 kfree(bn->vnic_info); 1169 bn->vnic_info = NULL; 1170 bn->nr_vnics = 0; 1171 } 1172 1173 static void bnge_free_ring_grps(struct bnge_net *bn) 1174 { 1175 kfree(bn->grp_info); 1176 bn->grp_info = NULL; 1177 } 1178 1179 static int bnge_init_ring_grps(struct bnge_net *bn) 1180 { 1181 struct bnge_dev *bd = bn->bd; 1182 int i; 1183 1184 bn->grp_info = kzalloc_objs(struct bnge_ring_grp_info, bd->nq_nr_rings); 1185 if (!bn->grp_info) 1186 return -ENOMEM; 1187 for (i = 0; i < bd->nq_nr_rings; i++) { 1188 bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 1189 bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 1190 bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 1191 bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 1192 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 1193 } 1194 1195 return 0; 1196 } 1197 1198 static void bnge_free_core(struct bnge_net *bn) 1199 { 1200 bnge_free_vnic_attributes(bn); 1201 bnge_free_tx_rings(bn); 1202 bnge_free_rx_rings(bn); 1203 bnge_free_nq_tree(bn); 1204 bnge_free_nq_arrays(bn); 1205 bnge_free_ring_stats(bn); 1206 bnge_free_ring_grps(bn); 1207 bnge_free_vnics(bn); 1208 kfree(bn->tx_ring_map); 1209 bn->tx_ring_map = NULL; 1210 kfree(bn->tx_ring); 1211 bn->tx_ring = NULL; 1212 kfree(bn->rx_ring); 1213 bn->rx_ring = NULL; 1214 kfree(bn->bnapi); 1215 bn->bnapi = NULL; 1216 } 1217 1218 static int bnge_alloc_core(struct bnge_net *bn) 1219 { 1220 struct bnge_dev *bd = bn->bd; 1221 int i, j, size, arr_size; 1222 int rc = -ENOMEM; 1223 void *bnapi; 1224 1225 arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) * 1226 bd->nq_nr_rings); 1227 size = L1_CACHE_ALIGN(sizeof(struct bnge_napi)); 1228 bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL); 1229 if (!bnapi) 1230 return rc; 1231 1232 bn->bnapi = bnapi; 1233 bnapi += arr_size; 1234 for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) { 1235 struct bnge_nq_ring_info *nqr; 1236 1237 bn->bnapi[i] = bnapi; 1238 bn->bnapi[i]->index = i; 1239 bn->bnapi[i]->bn = bn; 1240 nqr = &bn->bnapi[i]->nq_ring; 1241 nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1242 } 1243 1244 bn->rx_ring = kzalloc_objs(struct bnge_rx_ring_info, bd->rx_nr_rings); 1245 if (!bn->rx_ring) 1246 goto err_free_bnapi; 1247 1248 for (i = 0; i < bd->rx_nr_rings; i++) { 1249 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1250 1251 rxr->rx_ring_struct.ring_mem.flags = 1252 BNGE_RMEM_RING_PTE_FLAG; 1253 rxr->rx_agg_ring_struct.ring_mem.flags = 1254 BNGE_RMEM_RING_PTE_FLAG; 1255 rxr->bnapi = bn->bnapi[i]; 1256 bn->bnapi[i]->rx_ring = &bn->rx_ring[i]; 1257 } 1258 1259 bn->tx_ring = kzalloc_objs(struct bnge_tx_ring_info, bd->tx_nr_rings); 1260 if (!bn->tx_ring) 1261 goto err_free_rx_ring; 1262 1263 bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16), 1264 GFP_KERNEL); 1265 if (!bn->tx_ring_map) 1266 goto err_free_tx_ring; 1267 1268 if (bd->flags & BNGE_EN_SHARED_CHNL) 1269 j = 0; 1270 else 1271 j = bd->rx_nr_rings; 1272 1273 for (i = 0; i < bd->tx_nr_rings; i++) { 1274 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1275 struct bnge_napi *bnapi2; 1276 int k; 1277 1278 txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1279 bn->tx_ring_map[i] = i; 1280 k = j + BNGE_RING_TO_TC_OFF(bd, i); 1281 1282 bnapi2 = bn->bnapi[k]; 1283 txr->txq_index = i; 1284 txr->tx_napi_idx = 1285 BNGE_RING_TO_TC(bd, txr->txq_index); 1286 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 1287 txr->bnapi = bnapi2; 1288 } 1289 1290 rc = bnge_alloc_ring_stats(bn); 1291 if (rc) 1292 goto err_free_tx_ring_map; 1293 1294 bnge_init_stats(bn); 1295 1296 rc = bnge_alloc_vnics(bn); 1297 if (rc) 1298 goto err_free_ring_stats; 1299 1300 rc = bnge_alloc_nq_arrays(bn); 1301 if (rc) 1302 goto err_free_vnics; 1303 1304 bnge_init_ring_struct(bn); 1305 1306 rc = bnge_alloc_rx_rings(bn); 1307 if (rc) 1308 goto err_free_nq_arrays; 1309 1310 rc = bnge_alloc_tx_rings(bn); 1311 if (rc) 1312 goto err_free_rx_rings; 1313 1314 rc = bnge_alloc_nq_tree(bn); 1315 if (rc) 1316 goto err_free_tx_rings; 1317 1318 bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG | 1319 BNGE_VNIC_MCAST_FLAG | 1320 BNGE_VNIC_UCAST_FLAG; 1321 rc = bnge_alloc_vnic_attributes(bn); 1322 if (rc) 1323 goto err_free_nq_tree; 1324 return 0; 1325 1326 err_free_nq_tree: 1327 bnge_free_nq_tree(bn); 1328 err_free_tx_rings: 1329 bnge_free_tx_rings(bn); 1330 err_free_rx_rings: 1331 bnge_free_rx_rings(bn); 1332 err_free_nq_arrays: 1333 bnge_free_nq_arrays(bn); 1334 err_free_vnics: 1335 bnge_free_vnics(bn); 1336 err_free_ring_stats: 1337 bnge_free_ring_stats(bn); 1338 err_free_tx_ring_map: 1339 kfree(bn->tx_ring_map); 1340 bn->tx_ring_map = NULL; 1341 err_free_tx_ring: 1342 kfree(bn->tx_ring); 1343 bn->tx_ring = NULL; 1344 err_free_rx_ring: 1345 kfree(bn->rx_ring); 1346 bn->rx_ring = NULL; 1347 err_free_bnapi: 1348 kfree(bn->bnapi); 1349 bn->bnapi = NULL; 1350 return rc; 1351 } 1352 1353 u32 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr) 1354 { 1355 return rxr->rx_cpr->ring_struct.fw_ring_id; 1356 } 1357 1358 u32 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr) 1359 { 1360 return txr->tx_cpr->ring_struct.fw_ring_id; 1361 } 1362 1363 static void bnge_db_nq_arm(struct bnge_net *bn, 1364 struct bnge_db_info *db, u32 idx) 1365 { 1366 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_ARM | 1367 DB_RING_IDX(db, idx), db->doorbell); 1368 } 1369 1370 static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1371 { 1372 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK | 1373 DB_RING_IDX(db, idx), db->doorbell); 1374 } 1375 1376 static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1377 { 1378 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL | 1379 DB_RING_IDX(db, idx), db->doorbell); 1380 } 1381 1382 static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n) 1383 { 1384 struct bnge_napi *bnapi = bn->bnapi[n]; 1385 struct bnge_nq_ring_info *nqr; 1386 1387 nqr = &bnapi->nq_ring; 1388 1389 return nqr->ring_struct.map_idx; 1390 } 1391 1392 static void bnge_init_nq_tree(struct bnge_net *bn) 1393 { 1394 struct bnge_dev *bd = bn->bd; 1395 int i, j; 1396 1397 for (i = 0; i < bd->nq_nr_rings; i++) { 1398 struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring; 1399 struct bnge_ring_struct *ring = &nqr->ring_struct; 1400 1401 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1402 for (j = 0; j < nqr->cp_ring_count; j++) { 1403 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 1404 1405 ring = &cpr->ring_struct; 1406 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1407 } 1408 } 1409 } 1410 1411 static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn, 1412 dma_addr_t *mapping, 1413 struct bnge_rx_ring_info *rxr, 1414 unsigned int *offset, 1415 gfp_t gfp) 1416 { 1417 netmem_ref netmem; 1418 1419 if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) { 1420 netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, 1421 BNGE_RX_PAGE_SIZE, gfp); 1422 } else { 1423 netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); 1424 *offset = 0; 1425 } 1426 if (!netmem) 1427 return 0; 1428 1429 *mapping = page_pool_get_dma_addr_netmem(netmem) + *offset; 1430 return netmem; 1431 } 1432 1433 u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping, 1434 struct bnge_rx_ring_info *rxr, 1435 gfp_t gfp) 1436 { 1437 unsigned int offset; 1438 struct page *page; 1439 1440 page = page_pool_alloc_frag(rxr->head_pool, &offset, 1441 bn->rx_buf_size, gfp); 1442 if (!page) 1443 return NULL; 1444 1445 *mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset; 1446 return page_address(page) + offset; 1447 } 1448 1449 int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr, 1450 u16 prod, gfp_t gfp) 1451 { 1452 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)]; 1453 struct rx_bd *rxbd; 1454 dma_addr_t mapping; 1455 u8 *data; 1456 1457 rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)]; 1458 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp); 1459 if (!data) 1460 return -ENOMEM; 1461 1462 rx_buf->data = data; 1463 rx_buf->data_ptr = data + bn->rx_offset; 1464 rx_buf->mapping = mapping; 1465 1466 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1467 1468 return 0; 1469 } 1470 1471 static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn, 1472 struct bnge_rx_ring_info *rxr, 1473 int ring_nr) 1474 { 1475 u32 prod = rxr->rx_prod; 1476 int i, rc = 0; 1477 1478 for (i = 0; i < bn->rx_ring_size; i++) { 1479 rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL); 1480 if (rc) 1481 break; 1482 prod = NEXT_RX(prod); 1483 } 1484 1485 /* Abort if not a single buffer can be allocated */ 1486 if (rc && !i) { 1487 netdev_err(bn->netdev, 1488 "RX ring %d: allocated %d/%d buffers, abort\n", 1489 ring_nr, i, bn->rx_ring_size); 1490 return rc; 1491 } 1492 1493 rxr->rx_prod = prod; 1494 1495 if (i < bn->rx_ring_size) 1496 netdev_warn(bn->netdev, 1497 "RX ring %d: allocated %d/%d buffers, continuing\n", 1498 ring_nr, i, bn->rx_ring_size); 1499 return 0; 1500 } 1501 1502 u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx) 1503 { 1504 u16 next, max = rxr->rx_agg_bmap_size; 1505 1506 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 1507 if (next >= max) 1508 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 1509 return next; 1510 } 1511 1512 int bnge_alloc_rx_netmem(struct bnge_net *bn, 1513 struct bnge_rx_ring_info *rxr, 1514 u16 prod, gfp_t gfp) 1515 { 1516 struct bnge_sw_rx_agg_bd *rx_agg_buf; 1517 u16 sw_prod = rxr->rx_sw_agg_prod; 1518 unsigned int offset = 0; 1519 struct rx_bd *rxbd; 1520 dma_addr_t mapping; 1521 netmem_ref netmem; 1522 1523 rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)]; 1524 netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp); 1525 if (!netmem) 1526 return -ENOMEM; 1527 1528 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1529 sw_prod = bnge_find_next_agg_idx(rxr, sw_prod); 1530 1531 __set_bit(sw_prod, rxr->rx_agg_bmap); 1532 rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod]; 1533 rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod)); 1534 1535 rx_agg_buf->netmem = netmem; 1536 rx_agg_buf->offset = offset; 1537 rx_agg_buf->mapping = mapping; 1538 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1539 rxbd->rx_bd_opaque = sw_prod; 1540 return 0; 1541 } 1542 1543 static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn, 1544 struct bnge_rx_ring_info *rxr, 1545 int ring_nr) 1546 { 1547 u32 prod = rxr->rx_agg_prod; 1548 int i, rc = 0; 1549 1550 for (i = 0; i < bn->rx_agg_ring_size; i++) { 1551 rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL); 1552 if (rc) 1553 break; 1554 prod = NEXT_RX_AGG(prod); 1555 } 1556 1557 if (rc && i < MAX_SKB_FRAGS) { 1558 netdev_err(bn->netdev, 1559 "Agg ring %d: allocated %d/%d buffers (min %d), abort\n", 1560 ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS); 1561 goto err_free_one_agg_ring_bufs; 1562 } 1563 1564 rxr->rx_agg_prod = prod; 1565 1566 if (i < bn->rx_agg_ring_size) 1567 netdev_warn(bn->netdev, 1568 "Agg ring %d: allocated %d/%d buffers, continuing\n", 1569 ring_nr, i, bn->rx_agg_ring_size); 1570 return 0; 1571 1572 err_free_one_agg_ring_bufs: 1573 bnge_free_one_agg_ring_bufs(bn, rxr); 1574 return -ENOMEM; 1575 } 1576 1577 static int bnge_alloc_one_tpa_info_data(struct bnge_net *bn, 1578 struct bnge_rx_ring_info *rxr) 1579 { 1580 dma_addr_t mapping; 1581 u8 *data; 1582 int i; 1583 1584 for (i = 0; i < bn->max_tpa; i++) { 1585 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, 1586 GFP_KERNEL); 1587 if (!data) 1588 goto err_free_tpa_info_data; 1589 1590 rxr->rx_tpa[i].data = data; 1591 rxr->rx_tpa[i].data_ptr = data + bn->rx_offset; 1592 rxr->rx_tpa[i].mapping = mapping; 1593 } 1594 return 0; 1595 1596 err_free_tpa_info_data: 1597 bnge_free_one_tpa_info_data(bn, rxr); 1598 return -ENOMEM; 1599 } 1600 1601 static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr) 1602 { 1603 struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr]; 1604 int rc; 1605 1606 rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr); 1607 if (rc) 1608 return rc; 1609 1610 if (bnge_is_agg_reqd(bn->bd)) { 1611 rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr); 1612 if (rc) 1613 goto err_free_one_rx_ring_bufs; 1614 } 1615 1616 if (rxr->rx_tpa) { 1617 rc = bnge_alloc_one_tpa_info_data(bn, rxr); 1618 if (rc) 1619 goto err_free_one_agg_ring_bufs; 1620 } 1621 1622 return 0; 1623 1624 err_free_one_agg_ring_bufs: 1625 bnge_free_one_agg_ring_bufs(bn, rxr); 1626 err_free_one_rx_ring_bufs: 1627 bnge_free_one_rx_ring_bufs(bn, rxr); 1628 return rc; 1629 } 1630 1631 static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type) 1632 { 1633 struct rx_bd **rx_desc_ring; 1634 u32 prod; 1635 int i; 1636 1637 rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 1638 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 1639 struct rx_bd *rxbd = rx_desc_ring[i]; 1640 int j; 1641 1642 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 1643 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 1644 rxbd->rx_bd_opaque = prod; 1645 } 1646 } 1647 } 1648 1649 static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn, 1650 struct bnge_rx_ring_info *rxr) 1651 { 1652 struct bnge_ring_struct *ring; 1653 u32 type; 1654 1655 type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) | 1656 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 1657 1658 if (NET_IP_ALIGN == 2) 1659 type |= RX_BD_FLAGS_SOP; 1660 1661 ring = &rxr->rx_ring_struct; 1662 bnge_init_rxbd_pages(ring, type); 1663 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1664 } 1665 1666 static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn, 1667 struct bnge_rx_ring_info *rxr) 1668 { 1669 struct bnge_ring_struct *ring; 1670 u32 type; 1671 1672 ring = &rxr->rx_agg_ring_struct; 1673 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1674 if (bnge_is_agg_reqd(bn->bd)) { 1675 type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 1676 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 1677 1678 bnge_init_rxbd_pages(ring, type); 1679 } 1680 } 1681 1682 static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr) 1683 { 1684 struct bnge_rx_ring_info *rxr; 1685 1686 rxr = &bn->rx_ring[ring_nr]; 1687 bnge_init_one_rx_ring_rxbd(bn, rxr); 1688 1689 netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX, 1690 &rxr->bnapi->napi); 1691 1692 bnge_init_one_agg_ring_rxbd(bn, rxr); 1693 } 1694 1695 static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn) 1696 { 1697 int i, rc; 1698 1699 for (i = 0; i < bn->bd->rx_nr_rings; i++) { 1700 rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i); 1701 if (rc) 1702 goto err_free_rx_ring_pair_bufs; 1703 } 1704 return 0; 1705 1706 err_free_rx_ring_pair_bufs: 1707 bnge_free_rx_ring_pair_bufs(bn); 1708 return rc; 1709 } 1710 1711 static void bnge_init_rx_rings(struct bnge_net *bn) 1712 { 1713 int i; 1714 1715 #define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 1716 #define BNGE_RX_DMA_OFFSET NET_SKB_PAD 1717 bn->rx_offset = BNGE_RX_OFFSET; 1718 bn->rx_dma_offset = BNGE_RX_DMA_OFFSET; 1719 1720 for (i = 0; i < bn->bd->rx_nr_rings; i++) 1721 bnge_init_one_rx_ring_pair(bn, i); 1722 } 1723 1724 static void bnge_init_tx_rings(struct bnge_net *bn) 1725 { 1726 int i; 1727 1728 bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT); 1729 1730 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 1731 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1732 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1733 1734 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1735 1736 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, 1737 &txr->bnapi->napi); 1738 } 1739 } 1740 1741 static void bnge_init_vnics(struct bnge_net *bn) 1742 { 1743 struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 1744 int i; 1745 1746 for (i = 0; i < bn->nr_vnics; i++) { 1747 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 1748 int j; 1749 1750 vnic->fw_vnic_id = INVALID_HW_RING_ID; 1751 vnic->vnic_id = i; 1752 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) 1753 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 1754 1755 if (bn->vnic_info[i].rss_hash_key) { 1756 if (i == BNGE_VNIC_DEFAULT) { 1757 u8 *key = (void *)vnic->rss_hash_key; 1758 int k; 1759 1760 if (!bn->rss_hash_key_valid && 1761 !bn->rss_hash_key_updated) { 1762 get_random_bytes(bn->rss_hash_key, 1763 HW_HASH_KEY_SIZE); 1764 bn->rss_hash_key_updated = true; 1765 } 1766 1767 memcpy(vnic->rss_hash_key, bn->rss_hash_key, 1768 HW_HASH_KEY_SIZE); 1769 1770 if (!bn->rss_hash_key_updated) 1771 continue; 1772 1773 bn->rss_hash_key_updated = false; 1774 bn->rss_hash_key_valid = true; 1775 1776 bn->toeplitz_prefix = 0; 1777 for (k = 0; k < 8; k++) { 1778 bn->toeplitz_prefix <<= 8; 1779 bn->toeplitz_prefix |= key[k]; 1780 } 1781 } else { 1782 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 1783 HW_HASH_KEY_SIZE); 1784 } 1785 } 1786 } 1787 } 1788 1789 static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db, 1790 u32 ring_type) 1791 { 1792 switch (ring_type) { 1793 case HWRM_RING_ALLOC_TX: 1794 db->db_ring_mask = bn->tx_ring_mask; 1795 break; 1796 case HWRM_RING_ALLOC_RX: 1797 db->db_ring_mask = bn->rx_ring_mask; 1798 break; 1799 case HWRM_RING_ALLOC_AGG: 1800 db->db_ring_mask = bn->rx_agg_ring_mask; 1801 break; 1802 case HWRM_RING_ALLOC_CMPL: 1803 case HWRM_RING_ALLOC_NQ: 1804 db->db_ring_mask = bn->cp_ring_mask; 1805 break; 1806 } 1807 db->db_epoch_mask = db->db_ring_mask + 1; 1808 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 1809 } 1810 1811 static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db, 1812 u32 ring_type, u32 map_idx, u32 xid) 1813 { 1814 struct bnge_dev *bd = bn->bd; 1815 1816 switch (ring_type) { 1817 case HWRM_RING_ALLOC_TX: 1818 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 1819 break; 1820 case HWRM_RING_ALLOC_RX: 1821 case HWRM_RING_ALLOC_AGG: 1822 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 1823 break; 1824 case HWRM_RING_ALLOC_CMPL: 1825 db->db_key64 = DBR_PATH_L2; 1826 break; 1827 case HWRM_RING_ALLOC_NQ: 1828 db->db_key64 = DBR_PATH_L2; 1829 break; 1830 } 1831 db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID; 1832 1833 db->doorbell = bd->bar1 + bd->db_offset; 1834 bnge_set_db_mask(bn, db, ring_type); 1835 } 1836 1837 static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn, 1838 struct bnge_cp_ring_info *cpr) 1839 { 1840 const u32 type = HWRM_RING_ALLOC_CMPL; 1841 struct bnge_napi *bnapi = cpr->bnapi; 1842 struct bnge_ring_struct *ring; 1843 u32 map_idx = bnapi->index; 1844 int rc; 1845 1846 ring = &cpr->ring_struct; 1847 ring->handle = BNGE_SET_NQ_HDL(cpr); 1848 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1849 if (rc) 1850 return rc; 1851 1852 bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 1853 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1854 1855 return 0; 1856 } 1857 1858 static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn, 1859 struct bnge_tx_ring_info *txr, u32 tx_idx) 1860 { 1861 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1862 const u32 type = HWRM_RING_ALLOC_TX; 1863 int rc; 1864 1865 rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx); 1866 if (rc) 1867 return rc; 1868 1869 bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 1870 1871 return 0; 1872 } 1873 1874 static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn, 1875 struct bnge_rx_ring_info *rxr) 1876 { 1877 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 1878 u32 type = HWRM_RING_ALLOC_AGG; 1879 struct bnge_dev *bd = bn->bd; 1880 u32 grp_idx = ring->grp_idx; 1881 u32 map_idx; 1882 int rc; 1883 1884 map_idx = grp_idx + bd->rx_nr_rings; 1885 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1886 if (rc) 1887 return rc; 1888 1889 bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx, 1890 ring->fw_ring_id); 1891 bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod); 1892 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1893 bn->grp_info[grp_idx].agg_fw_ring_id = (u16)ring->fw_ring_id; 1894 1895 return 0; 1896 } 1897 1898 static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn, 1899 struct bnge_rx_ring_info *rxr) 1900 { 1901 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 1902 struct bnge_napi *bnapi = rxr->bnapi; 1903 u32 type = HWRM_RING_ALLOC_RX; 1904 u32 map_idx = bnapi->index; 1905 int rc; 1906 1907 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1908 if (rc) 1909 return rc; 1910 1911 bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 1912 bn->grp_info[map_idx].rx_fw_ring_id = (u16)ring->fw_ring_id; 1913 1914 return 0; 1915 } 1916 1917 static int bnge_hwrm_ring_alloc(struct bnge_net *bn) 1918 { 1919 struct bnge_dev *bd = bn->bd; 1920 bool agg_rings; 1921 int i, rc = 0; 1922 1923 agg_rings = !!(bnge_is_agg_reqd(bd)); 1924 for (i = 0; i < bd->nq_nr_rings; i++) { 1925 struct bnge_napi *bnapi = bn->bnapi[i]; 1926 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 1927 struct bnge_ring_struct *ring = &nqr->ring_struct; 1928 u32 type = HWRM_RING_ALLOC_NQ; 1929 u32 map_idx = ring->map_idx; 1930 unsigned int vector; 1931 1932 vector = bd->irq_tbl[map_idx].vector; 1933 disable_irq_nosync(vector); 1934 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1935 if (rc) { 1936 enable_irq(vector); 1937 goto err_out; 1938 } 1939 bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id); 1940 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 1941 enable_irq(vector); 1942 bn->grp_info[i].nq_fw_ring_id = (u16)ring->fw_ring_id; 1943 1944 if (!i) { 1945 rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id); 1946 if (rc) 1947 netdev_warn(bn->netdev, "Failed to set async event completion ring.\n"); 1948 } 1949 } 1950 1951 for (i = 0; i < bd->tx_nr_rings; i++) { 1952 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1953 1954 rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr); 1955 if (rc) 1956 goto err_out; 1957 rc = bnge_hwrm_tx_ring_alloc(bn, txr, i); 1958 if (rc) 1959 goto err_out; 1960 } 1961 1962 for (i = 0; i < bd->rx_nr_rings; i++) { 1963 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1964 struct bnge_cp_ring_info *cpr; 1965 struct bnge_ring_struct *ring; 1966 struct bnge_napi *bnapi; 1967 u32 map_idx, type; 1968 1969 rc = bnge_hwrm_rx_ring_alloc(bn, rxr); 1970 if (rc) 1971 goto err_out; 1972 /* If we have agg rings, post agg buffers first. */ 1973 if (!agg_rings) 1974 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1975 1976 cpr = rxr->rx_cpr; 1977 bnapi = rxr->bnapi; 1978 type = HWRM_RING_ALLOC_CMPL; 1979 map_idx = bnapi->index; 1980 1981 ring = &cpr->ring_struct; 1982 ring->handle = BNGE_SET_NQ_HDL(cpr); 1983 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1984 if (rc) 1985 goto err_out; 1986 bnge_set_db(bn, &cpr->cp_db, type, map_idx, 1987 ring->fw_ring_id); 1988 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1989 } 1990 1991 if (agg_rings) { 1992 for (i = 0; i < bd->rx_nr_rings; i++) { 1993 rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]); 1994 if (rc) 1995 goto err_out; 1996 } 1997 } 1998 err_out: 1999 return rc; 2000 } 2001 2002 void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic) 2003 { 2004 __le16 *ring_tbl = vnic->rss_table; 2005 struct bnge_rx_ring_info *rxr; 2006 struct bnge_dev *bd = bn->bd; 2007 u16 tbl_size, i; 2008 2009 tbl_size = bnge_get_rxfh_indir_size(bd); 2010 2011 for (i = 0; i < tbl_size; i++) { 2012 u32 j; 2013 2014 j = bd->rss_indir_tbl[i]; 2015 rxr = &bn->rx_ring[j]; 2016 2017 *ring_tbl++ = cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 2018 *ring_tbl++ = cpu_to_le16(bnge_cp_ring_for_rx(rxr)); 2019 } 2020 } 2021 2022 static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn, 2023 struct bnge_vnic_info *vnic) 2024 { 2025 int rc; 2026 2027 rc = bnge_hwrm_vnic_set_rss(bn, vnic, true); 2028 if (rc) { 2029 netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n", 2030 vnic->vnic_id, rc); 2031 return rc; 2032 } 2033 rc = bnge_hwrm_vnic_cfg(bn, vnic); 2034 if (rc) 2035 netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n", 2036 vnic->vnic_id, rc); 2037 return rc; 2038 } 2039 2040 static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic) 2041 { 2042 struct bnge_dev *bd = bn->bd; 2043 int rc, i, nr_ctxs; 2044 2045 nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings); 2046 for (i = 0; i < nr_ctxs; i++) { 2047 rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i); 2048 if (rc) { 2049 netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n", 2050 vnic->vnic_id, i, rc); 2051 return -ENOMEM; 2052 } 2053 bn->rsscos_nr_ctxs++; 2054 } 2055 2056 rc = bnge_hwrm_vnic_rss_cfg(bn, vnic); 2057 if (rc) 2058 return rc; 2059 2060 if (bnge_is_agg_reqd(bd)) { 2061 rc = bnge_hwrm_vnic_set_hds(bn, vnic); 2062 if (rc) 2063 netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n", 2064 vnic->vnic_id, rc); 2065 } 2066 return rc; 2067 } 2068 2069 static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr) 2070 { 2071 if (!refcount_dec_and_test(&fltr->refcnt)) 2072 return; 2073 hlist_del_rcu(&fltr->base.hash); 2074 kfree_rcu(fltr, base.rcu); 2075 } 2076 2077 static void bnge_init_l2_filter(struct bnge_net *bn, 2078 struct bnge_l2_filter *fltr, 2079 struct bnge_l2_key *key, u32 idx) 2080 { 2081 struct hlist_head *head; 2082 2083 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 2084 fltr->l2_key.vlan = key->vlan; 2085 fltr->base.type = BNGE_FLTR_TYPE_L2; 2086 2087 head = &bn->l2_fltr_hash_tbl[idx]; 2088 hlist_add_head_rcu(&fltr->base.hash, head); 2089 refcount_set(&fltr->refcnt, 1); 2090 } 2091 2092 static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn, 2093 struct bnge_l2_key *key, 2094 u32 idx) 2095 { 2096 struct bnge_l2_filter *fltr; 2097 struct hlist_head *head; 2098 2099 head = &bn->l2_fltr_hash_tbl[idx]; 2100 hlist_for_each_entry_rcu(fltr, head, base.hash) { 2101 struct bnge_l2_key *l2_key = &fltr->l2_key; 2102 2103 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 2104 l2_key->vlan == key->vlan) 2105 return fltr; 2106 } 2107 return NULL; 2108 } 2109 2110 static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn, 2111 struct bnge_l2_key *key, 2112 u32 idx) 2113 { 2114 struct bnge_l2_filter *fltr; 2115 2116 rcu_read_lock(); 2117 fltr = __bnge_lookup_l2_filter(bn, key, idx); 2118 if (fltr) 2119 refcount_inc(&fltr->refcnt); 2120 rcu_read_unlock(); 2121 return fltr; 2122 } 2123 2124 static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn, 2125 struct bnge_l2_key *key, 2126 gfp_t gfp) 2127 { 2128 struct bnge_l2_filter *fltr; 2129 u32 idx; 2130 2131 idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) & 2132 BNGE_L2_FLTR_HASH_MASK; 2133 fltr = bnge_lookup_l2_filter(bn, key, idx); 2134 if (fltr) 2135 return fltr; 2136 2137 fltr = kzalloc_obj(*fltr, gfp); 2138 if (!fltr) 2139 return ERR_PTR(-ENOMEM); 2140 2141 bnge_init_l2_filter(bn, fltr, key, idx); 2142 return fltr; 2143 } 2144 2145 static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx, 2146 const u8 *mac_addr) 2147 { 2148 struct bnge_l2_filter *fltr; 2149 struct bnge_l2_key key; 2150 int rc; 2151 2152 ether_addr_copy(key.dst_mac_addr, mac_addr); 2153 key.vlan = 0; 2154 fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL); 2155 if (IS_ERR(fltr)) 2156 return PTR_ERR(fltr); 2157 2158 fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id; 2159 rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr); 2160 if (rc) 2161 goto err_del_l2_filter; 2162 bn->vnic_info[vnic_id].l2_filters[idx] = fltr; 2163 return rc; 2164 2165 err_del_l2_filter: 2166 bnge_del_l2_filter(bn, fltr); 2167 return rc; 2168 } 2169 2170 static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask, 2171 const struct netdev_hw_addr_list *mc) 2172 { 2173 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2174 struct netdev_hw_addr *ha; 2175 int mc_count = 0, off = 0; 2176 bool update = false; 2177 u8 *haddr; 2178 2179 netdev_hw_addr_list_for_each(ha, mc) { 2180 if (mc_count >= BNGE_MAX_MC_ADDRS) { 2181 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2182 vnic->mc_list_count = 0; 2183 return false; 2184 } 2185 haddr = ha->addr; 2186 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 2187 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 2188 update = true; 2189 } 2190 off += ETH_ALEN; 2191 mc_count++; 2192 } 2193 if (mc_count) 2194 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2195 2196 if (mc_count != vnic->mc_list_count) { 2197 vnic->mc_list_count = mc_count; 2198 update = true; 2199 } 2200 return update; 2201 } 2202 2203 static bool bnge_uc_list_updated(struct bnge_net *bn, 2204 const struct netdev_hw_addr_list *uc) 2205 { 2206 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2207 struct net_device *dev = bn->netdev; 2208 struct netdev_hw_addr *ha; 2209 int off = 0, count = 0; 2210 2211 netdev_hw_addr_list_for_each(ha, uc) { 2212 if (ether_addr_equal(ha->addr, dev->dev_addr)) 2213 continue; 2214 if (count >= vnic->uc_filter_count - 1) 2215 return true; 2216 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 2217 return true; 2218 off += ETH_ALEN; 2219 count++; 2220 } 2221 return count != vnic->uc_filter_count - 1; 2222 } 2223 2224 static bool bnge_promisc_ok(struct bnge_net *bn) 2225 { 2226 return true; 2227 } 2228 2229 static int bnge_cfg_rx_mode(struct bnge_net *bn, struct netdev_hw_addr_list *uc, 2230 bool uc_update, bool snapshot) 2231 { 2232 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2233 struct net_device *dev = bn->netdev; 2234 int i, off = 0, rc, uc_count = 0; 2235 struct bnge_dev *bd = bn->bd; 2236 struct netdev_hw_addr *ha; 2237 2238 if (!uc_update) 2239 goto skip_uc; 2240 2241 for (i = 1; i < vnic->uc_filter_count; i++) { 2242 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2243 2244 bnge_hwrm_l2_filter_free(bd, fltr); 2245 bnge_del_l2_filter(bn, fltr); 2246 } 2247 2248 vnic->uc_filter_count = 1; 2249 2250 if (!snapshot) 2251 netif_addr_lock_bh(dev); 2252 /* dev_addr is already programmed as L2 filter 0, so it takes neither a 2253 * uc_list slot nor an l2_filters[] slot. Count only the entries that 2254 * will actually be programmed so the overflow check matches the fill 2255 * loop below (and bnge_uc_list_updated()). 2256 */ 2257 netdev_hw_addr_list_for_each(ha, uc) 2258 if (!ether_addr_equal(ha->addr, dev->dev_addr)) 2259 uc_count++; 2260 if (uc_count > BNGE_MAX_UC_ADDRS - 1) { 2261 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2262 } else { 2263 netdev_hw_addr_list_for_each(ha, uc) { 2264 if (ether_addr_equal(ha->addr, dev->dev_addr)) 2265 continue; 2266 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 2267 off += ETH_ALEN; 2268 vnic->uc_filter_count++; 2269 } 2270 } 2271 if (!snapshot) 2272 netif_addr_unlock_bh(dev); 2273 2274 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 2275 rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off); 2276 if (rc) { 2277 if (rc == -EAGAIN) 2278 netdev_warn(dev, "FW busy while setting vnic filter, will retry\n"); 2279 else 2280 netdev_err(dev, "HWRM vnic filter failure rc: %d\n", 2281 rc); 2282 vnic->uc_filter_count = i; 2283 return rc; 2284 } 2285 } 2286 2287 skip_uc: 2288 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 2289 !bnge_promisc_ok(bn)) 2290 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2291 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2292 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 2293 netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 2294 rc); 2295 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2296 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2297 vnic->mc_list_count = 0; 2298 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2299 } 2300 if (rc) { 2301 if (rc == -EAGAIN) { 2302 netdev_warn(dev, "FW busy while setting l2 rx mask in CFA, will retry\n"); 2303 vnic->rx_mask &= ~BNGE_RX_MASK_CFG_FLAGS; 2304 } else { 2305 netdev_err(dev, "HWRM CFA L2 rx mask failure rc: %d\n", 2306 rc); 2307 } 2308 } 2309 2310 return rc; 2311 } 2312 2313 static int bnge_set_rx_mode(struct net_device *dev, 2314 struct netdev_hw_addr_list *uc, 2315 struct netdev_hw_addr_list *mc) 2316 { 2317 struct bnge_net *bn = netdev_priv(dev); 2318 struct bnge_vnic_info *vnic; 2319 bool mc_update = false; 2320 bool uc_update; 2321 u32 mask; 2322 2323 if (!test_bit(BNGE_STATE_OPEN, &bn->bd->state)) 2324 return 0; 2325 2326 vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2327 mask = vnic->rx_mask; 2328 mask &= ~BNGE_RX_MASK_CFG_FLAGS; 2329 2330 if (dev->flags & IFF_PROMISC) 2331 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2332 2333 uc_update = bnge_uc_list_updated(bn, uc); 2334 2335 if (dev->flags & IFF_BROADCAST) 2336 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2337 if (dev->flags & IFF_ALLMULTI) { 2338 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2339 vnic->mc_list_count = 0; 2340 } else if (dev->flags & IFF_MULTICAST) { 2341 mc_update = bnge_mc_list_updated(bn, &mask, mc); 2342 } 2343 2344 if (mask != vnic->rx_mask || uc_update || mc_update) { 2345 vnic->rx_mask = mask; 2346 return bnge_cfg_rx_mode(bn, uc, uc_update, true); 2347 } 2348 2349 return 0; 2350 } 2351 2352 static void bnge_disable_int(struct bnge_net *bn) 2353 { 2354 struct bnge_dev *bd = bn->bd; 2355 int i; 2356 2357 if (!bn->bnapi) 2358 return; 2359 2360 for (i = 0; i < bd->nq_nr_rings; i++) { 2361 struct bnge_napi *bnapi = bn->bnapi[i]; 2362 struct bnge_nq_ring_info *nqr; 2363 struct bnge_ring_struct *ring; 2364 2365 nqr = &bnapi->nq_ring; 2366 ring = &nqr->ring_struct; 2367 2368 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) 2369 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 2370 } 2371 } 2372 2373 static void bnge_disable_int_sync(struct bnge_net *bn) 2374 { 2375 struct bnge_dev *bd = bn->bd; 2376 int i; 2377 2378 bnge_disable_int(bn); 2379 for (i = 0; i < bd->nq_nr_rings; i++) { 2380 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2381 2382 synchronize_irq(bd->irq_tbl[map_idx].vector); 2383 } 2384 } 2385 2386 static void bnge_enable_int(struct bnge_net *bn) 2387 { 2388 struct bnge_dev *bd = bn->bd; 2389 int i; 2390 2391 for (i = 0; i < bd->nq_nr_rings; i++) { 2392 struct bnge_napi *bnapi = bn->bnapi[i]; 2393 struct bnge_nq_ring_info *nqr; 2394 2395 nqr = &bnapi->nq_ring; 2396 bnge_db_nq_arm(bn, &nqr->nq_db, nqr->nq_raw_cons); 2397 } 2398 } 2399 2400 static void bnge_disable_napi(struct bnge_net *bn) 2401 { 2402 struct bnge_dev *bd = bn->bd; 2403 int i; 2404 2405 if (test_and_set_bit(BNGE_STATE_NAPI_DISABLED, &bn->state)) 2406 return; 2407 2408 for (i = 0; i < bd->nq_nr_rings; i++) { 2409 struct bnge_napi *bnapi = bn->bnapi[i]; 2410 2411 napi_disable_locked(&bnapi->napi); 2412 } 2413 } 2414 2415 static void bnge_enable_napi(struct bnge_net *bn) 2416 { 2417 struct bnge_dev *bd = bn->bd; 2418 int i; 2419 2420 clear_bit(BNGE_STATE_NAPI_DISABLED, &bn->state); 2421 for (i = 0; i < bd->nq_nr_rings; i++) { 2422 struct bnge_napi *bnapi = bn->bnapi[i]; 2423 2424 bnapi->in_reset = false; 2425 bnapi->tx_fault = 0; 2426 2427 napi_enable_locked(&bnapi->napi); 2428 } 2429 } 2430 2431 static void bnge_hwrm_vnic_free(struct bnge_net *bn) 2432 { 2433 int i; 2434 2435 for (i = 0; i < bn->nr_vnics; i++) 2436 bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]); 2437 } 2438 2439 static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn) 2440 { 2441 int i, j; 2442 2443 for (i = 0; i < bn->nr_vnics; i++) { 2444 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 2445 2446 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) { 2447 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 2448 bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j); 2449 } 2450 } 2451 bn->rsscos_nr_ctxs = 0; 2452 } 2453 2454 static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn) 2455 { 2456 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2457 int i; 2458 2459 for (i = 0; i < vnic->uc_filter_count; i++) { 2460 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2461 2462 bnge_hwrm_l2_filter_free(bn->bd, fltr); 2463 bnge_del_l2_filter(bn, fltr); 2464 } 2465 2466 vnic->uc_filter_count = 0; 2467 } 2468 2469 static void bnge_clear_vnic(struct bnge_net *bn) 2470 { 2471 bnge_hwrm_clear_vnic_filter(bn); 2472 bnge_hwrm_vnic_free(bn); 2473 bnge_hwrm_vnic_ctx_free(bn); 2474 } 2475 2476 static void bnge_hwrm_rx_ring_free(struct bnge_net *bn, 2477 struct bnge_rx_ring_info *rxr, 2478 bool close_path) 2479 { 2480 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 2481 u32 grp_idx = rxr->bnapi->index; 2482 u32 cmpl_ring_id; 2483 2484 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2485 return; 2486 2487 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2488 hwrm_ring_free_send_msg(bn, ring, 2489 RING_FREE_REQ_RING_TYPE_RX, 2490 close_path ? cmpl_ring_id : 2491 INVALID_HW_RING_ID); 2492 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2493 bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 2494 } 2495 2496 static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn, 2497 struct bnge_rx_ring_info *rxr, 2498 bool close_path) 2499 { 2500 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 2501 u32 grp_idx = rxr->bnapi->index; 2502 u32 cmpl_ring_id; 2503 2504 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2505 return; 2506 2507 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2508 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG, 2509 close_path ? cmpl_ring_id : 2510 INVALID_HW_RING_ID); 2511 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2512 bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 2513 } 2514 2515 static void bnge_hwrm_tx_ring_free(struct bnge_net *bn, 2516 struct bnge_tx_ring_info *txr, 2517 bool close_path) 2518 { 2519 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 2520 u32 cmpl_ring_id; 2521 2522 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2523 return; 2524 2525 cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) : 2526 INVALID_HW_RING_ID; 2527 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX, 2528 cmpl_ring_id); 2529 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2530 } 2531 2532 static void bnge_hwrm_cp_ring_free(struct bnge_net *bn, 2533 struct bnge_cp_ring_info *cpr) 2534 { 2535 struct bnge_ring_struct *ring; 2536 2537 ring = &cpr->ring_struct; 2538 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2539 return; 2540 2541 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 2542 INVALID_HW_RING_ID); 2543 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2544 } 2545 2546 static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path) 2547 { 2548 struct bnge_dev *bd = bn->bd; 2549 int i; 2550 2551 if (!bn->bnapi) 2552 return; 2553 2554 for (i = 0; i < bd->tx_nr_rings; i++) 2555 bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path); 2556 2557 for (i = 0; i < bd->rx_nr_rings; i++) { 2558 bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path); 2559 bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path); 2560 } 2561 2562 /* The completion rings are about to be freed. After that the 2563 * IRQ doorbell will not work anymore. So we need to disable 2564 * IRQ here. 2565 */ 2566 bnge_disable_int_sync(bn); 2567 2568 for (i = 0; i < bd->nq_nr_rings; i++) { 2569 struct bnge_napi *bnapi = bn->bnapi[i]; 2570 struct bnge_nq_ring_info *nqr; 2571 struct bnge_ring_struct *ring; 2572 int j; 2573 2574 nqr = &bnapi->nq_ring; 2575 for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++) 2576 bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]); 2577 2578 ring = &nqr->ring_struct; 2579 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) { 2580 hwrm_ring_free_send_msg(bn, ring, 2581 RING_FREE_REQ_RING_TYPE_NQ, 2582 INVALID_HW_RING_ID); 2583 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2584 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 2585 } 2586 } 2587 } 2588 2589 static void bnge_setup_msix(struct bnge_net *bn) 2590 { 2591 struct net_device *dev = bn->netdev; 2592 struct bnge_dev *bd = bn->bd; 2593 int len, i; 2594 2595 len = sizeof(bd->irq_tbl[0].name); 2596 for (i = 0; i < bd->nq_nr_rings; i++) { 2597 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2598 char *attr; 2599 2600 if (bd->flags & BNGE_EN_SHARED_CHNL) 2601 attr = "TxRx"; 2602 else if (i < bd->rx_nr_rings) 2603 attr = "rx"; 2604 else 2605 attr = "tx"; 2606 2607 snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 2608 attr, i); 2609 bd->irq_tbl[map_idx].handler = bnge_msix; 2610 } 2611 } 2612 2613 static int bnge_setup_interrupts(struct bnge_net *bn) 2614 { 2615 struct net_device *dev = bn->netdev; 2616 struct bnge_dev *bd = bn->bd; 2617 2618 bnge_setup_msix(bn); 2619 2620 return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings); 2621 } 2622 2623 static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path) 2624 { 2625 bnge_clear_vnic(bn); 2626 bnge_hwrm_ring_free(bn, close_path); 2627 bnge_hwrm_stat_ctx_free(bn); 2628 } 2629 2630 static void bnge_free_irq(struct bnge_net *bn) 2631 { 2632 struct bnge_dev *bd = bn->bd; 2633 struct bnge_irq *irq; 2634 int i; 2635 2636 for (i = 0; i < bd->nq_nr_rings; i++) { 2637 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2638 2639 irq = &bd->irq_tbl[map_idx]; 2640 if (irq->requested) { 2641 if (irq->have_cpumask) { 2642 irq_set_affinity_hint(irq->vector, NULL); 2643 free_cpumask_var(irq->cpu_mask); 2644 irq->have_cpumask = 0; 2645 } 2646 free_irq(irq->vector, bn->bnapi[i]); 2647 } 2648 2649 irq->requested = 0; 2650 } 2651 } 2652 2653 static int bnge_request_irq(struct bnge_net *bn) 2654 { 2655 struct bnge_dev *bd = bn->bd; 2656 int i, rc; 2657 2658 rc = bnge_setup_interrupts(bn); 2659 if (rc) { 2660 netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc); 2661 return rc; 2662 } 2663 for (i = 0; i < bd->nq_nr_rings; i++) { 2664 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2665 struct bnge_irq *irq = &bd->irq_tbl[map_idx]; 2666 2667 rc = request_irq(irq->vector, irq->handler, 0, irq->name, 2668 bn->bnapi[i]); 2669 if (rc) 2670 goto err_free_irq; 2671 2672 netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector); 2673 irq->requested = 1; 2674 2675 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 2676 int numa_node = dev_to_node(&bd->pdev->dev); 2677 2678 irq->have_cpumask = 1; 2679 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2680 irq->cpu_mask); 2681 if (irq_set_affinity_hint(irq->vector, irq->cpu_mask)) { 2682 netdev_warn(bn->netdev, 2683 "Set affinity failed, IRQ = %d\n", 2684 irq->vector); 2685 } 2686 } 2687 } 2688 return 0; 2689 2690 err_free_irq: 2691 bnge_free_irq(bn); 2692 return rc; 2693 } 2694 2695 static int bnge_set_tpa(struct bnge_net *bn, bool set_tpa) 2696 { 2697 u32 tpa_flags = 0; 2698 int rc, i; 2699 2700 if (set_tpa) 2701 tpa_flags = bn->priv_flags & BNGE_NET_EN_TPA; 2702 else if (BNGE_NO_FW_ACCESS(bn->bd)) 2703 return 0; 2704 for (i = 0; i < bn->nr_vnics; i++) { 2705 rc = bnge_hwrm_vnic_set_tpa(bn->bd, &bn->vnic_info[i], 2706 tpa_flags); 2707 if (rc) { 2708 netdev_err(bn->netdev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 2709 i, rc); 2710 return rc; 2711 } 2712 } 2713 return 0; 2714 } 2715 2716 static int bnge_init_chip(struct bnge_net *bn) 2717 { 2718 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2719 struct bnge_dev *bd = bn->bd; 2720 int rc; 2721 2722 #define BNGE_DEF_STATS_COAL_TICKS 1000000 2723 bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS; 2724 2725 rc = bnge_hwrm_stat_ctx_alloc(bn); 2726 if (rc) { 2727 netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc); 2728 goto err_out; 2729 } 2730 2731 rc = bnge_hwrm_ring_alloc(bn); 2732 if (rc) { 2733 netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc); 2734 goto err_out; 2735 } 2736 2737 rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings); 2738 if (rc) { 2739 netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc); 2740 goto err_out; 2741 } 2742 2743 rc = bnge_setup_vnic(bn, vnic); 2744 if (rc) 2745 goto err_out; 2746 2747 if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA) 2748 bnge_hwrm_update_rss_hash_cfg(bn); 2749 2750 if (bn->priv_flags & BNGE_NET_EN_TPA) { 2751 rc = bnge_set_tpa(bn, true); 2752 if (rc) 2753 goto err_out; 2754 } 2755 2756 /* Filter for default vnic 0 */ 2757 rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr); 2758 if (rc) { 2759 netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc); 2760 goto err_out; 2761 } 2762 vnic->uc_filter_count = 1; 2763 2764 vnic->rx_mask = 0; 2765 2766 if (bn->netdev->flags & IFF_BROADCAST) 2767 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2768 2769 if (bn->netdev->flags & IFF_PROMISC) 2770 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2771 2772 if (bn->netdev->flags & IFF_ALLMULTI) { 2773 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2774 vnic->mc_list_count = 0; 2775 } else if (bn->netdev->flags & IFF_MULTICAST) { 2776 u32 mask = 0; 2777 2778 bnge_mc_list_updated(bn, &mask, &bn->netdev->mc); 2779 vnic->rx_mask |= mask; 2780 } 2781 2782 rc = bnge_cfg_rx_mode(bn, &bn->netdev->uc, true, false); 2783 if (rc == -EAGAIN) { 2784 netif_rx_mode_schedule_retry(bn->netdev); 2785 rc = 0; 2786 } else if (rc) { 2787 goto err_out; 2788 } 2789 return 0; 2790 2791 err_out: 2792 bnge_hwrm_resource_free(bn, 0); 2793 return rc; 2794 } 2795 2796 static void bnge_init_napi(struct bnge_net *bn) 2797 { 2798 struct bnge_dev *bd = bn->bd; 2799 struct bnge_napi *bnapi; 2800 int i; 2801 2802 for (i = 0; i < bd->nq_nr_rings; i++) { 2803 bnapi = bn->bnapi[i]; 2804 netif_napi_add_config_locked(bn->netdev, &bnapi->napi, 2805 bnge_napi_poll, bnapi->index); 2806 } 2807 } 2808 2809 static void bnge_del_napi(struct bnge_net *bn) 2810 { 2811 struct bnge_dev *bd = bn->bd; 2812 int i; 2813 2814 for (i = 0; i < bd->rx_nr_rings; i++) 2815 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL); 2816 for (i = 0; i < bd->tx_nr_rings; i++) 2817 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL); 2818 2819 for (i = 0; i < bd->nq_nr_rings; i++) { 2820 struct bnge_napi *bnapi = bn->bnapi[i]; 2821 2822 __netif_napi_del_locked(&bnapi->napi); 2823 } 2824 2825 /* Wait for RCU grace period after removing NAPI instances */ 2826 synchronize_net(); 2827 } 2828 2829 static int bnge_init_nic(struct bnge_net *bn) 2830 { 2831 int rc; 2832 2833 bnge_init_nq_tree(bn); 2834 2835 bnge_init_rx_rings(bn); 2836 rc = bnge_alloc_rx_ring_pair_bufs(bn); 2837 if (rc) 2838 return rc; 2839 2840 bnge_init_tx_rings(bn); 2841 2842 rc = bnge_init_ring_grps(bn); 2843 if (rc) 2844 goto err_free_rx_ring_pair_bufs; 2845 2846 bnge_init_vnics(bn); 2847 2848 rc = bnge_init_chip(bn); 2849 if (rc) 2850 goto err_free_ring_grps; 2851 return rc; 2852 2853 err_free_ring_grps: 2854 bnge_free_ring_grps(bn); 2855 err_free_rx_ring_pair_bufs: 2856 bnge_free_rx_ring_pair_bufs(bn); 2857 return rc; 2858 } 2859 2860 static void bnge_tx_disable(struct bnge_net *bn) 2861 { 2862 struct bnge_tx_ring_info *txr; 2863 int i; 2864 2865 if (bn->tx_ring) { 2866 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2867 txr = &bn->tx_ring[i]; 2868 WRITE_ONCE(txr->dev_state, BNGE_DEV_STATE_CLOSING); 2869 } 2870 } 2871 /* Make sure napi polls see @dev_state change */ 2872 synchronize_net(); 2873 2874 if (!bn->netdev) 2875 return; 2876 /* Drop carrier first to prevent TX timeout */ 2877 netif_carrier_off(bn->netdev); 2878 /* Stop all TX queues */ 2879 netif_tx_disable(bn->netdev); 2880 } 2881 2882 static void bnge_tx_enable(struct bnge_net *bn) 2883 { 2884 struct bnge_tx_ring_info *txr; 2885 int i; 2886 2887 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2888 txr = &bn->tx_ring[i]; 2889 WRITE_ONCE(txr->dev_state, 0); 2890 } 2891 /* Make sure napi polls see @dev_state change */ 2892 synchronize_net(); 2893 netif_tx_wake_all_queues(bn->netdev); 2894 if (BNGE_LINK_IS_UP(bn->bd)) 2895 netif_carrier_on(bn->netdev); 2896 } 2897 2898 static int bnge_hwrm_if_change(struct bnge_dev *bd, bool up) 2899 { 2900 struct hwrm_func_drv_if_change_input *req; 2901 int rc; 2902 2903 if (!(bd->fw_cap & BNGE_FW_CAP_IF_CHANGE)) 2904 return 0; 2905 2906 rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_DRV_IF_CHANGE); 2907 if (rc) 2908 return rc; 2909 2910 if (up) 2911 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 2912 2913 return bnge_hwrm_req_send(bd, req); 2914 } 2915 2916 static int bnge_open_core(struct bnge_net *bn) 2917 { 2918 struct bnge_dev *bd = bn->bd; 2919 int rc; 2920 2921 netif_carrier_off(bn->netdev); 2922 2923 rc = bnge_hwrm_if_change(bd, true); 2924 if (rc) { 2925 netdev_err(bn->netdev, "bnge_hwrm_if_change err: %d\n", rc); 2926 return rc; 2927 } 2928 2929 rc = bnge_reserve_rings(bd); 2930 if (rc) { 2931 netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc); 2932 goto err_if_change; 2933 } 2934 2935 rc = bnge_alloc_core(bn); 2936 if (rc) { 2937 netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc); 2938 goto err_if_change; 2939 } 2940 2941 bnge_init_napi(bn); 2942 rc = bnge_request_irq(bn); 2943 if (rc) { 2944 netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc); 2945 goto err_del_napi; 2946 } 2947 2948 rc = bnge_init_nic(bn); 2949 if (rc) { 2950 netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc); 2951 goto err_free_irq; 2952 } 2953 2954 bnge_enable_napi(bn); 2955 2956 rc = bnge_update_phy_setting(bn); 2957 if (rc) { 2958 netdev_warn(bn->netdev, "failed to update PHY settings (rc: %d)\n", 2959 rc); 2960 WRITE_ONCE(bd->link_info.phy_retry_expires, jiffies + 5 * HZ); 2961 WRITE_ONCE(bd->link_info.phy_retry, true); 2962 } else { 2963 WRITE_ONCE(bd->link_info.phy_retry, false); 2964 } 2965 2966 set_bit(BNGE_STATE_OPEN, &bd->state); 2967 set_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 2968 2969 bnge_enable_int(bn); 2970 2971 bnge_tx_enable(bn); 2972 2973 mod_timer(&bn->timer, jiffies + bn->current_interval); 2974 2975 /* Poll link status and check for SFP+ module status */ 2976 bnge_get_port_module_status(bn); 2977 2978 return 0; 2979 2980 err_free_irq: 2981 bnge_free_irq(bn); 2982 err_del_napi: 2983 bnge_del_napi(bn); 2984 bnge_free_core(bn); 2985 err_if_change: 2986 bnge_hwrm_if_change(bd, false); 2987 return rc; 2988 } 2989 2990 static int bnge_open(struct net_device *dev) 2991 { 2992 struct bnge_net *bn = netdev_priv(dev); 2993 int rc; 2994 2995 rc = bnge_open_core(bn); 2996 if (rc) 2997 netdev_err(dev, "bnge_open_core err: %d\n", rc); 2998 2999 return rc; 3000 } 3001 3002 static int bnge_shutdown_nic(struct bnge_net *bn) 3003 { 3004 bnge_hwrm_resource_free(bn, 1); 3005 return 0; 3006 } 3007 3008 static void bnge_add_prev_ring_stats64(struct bnge_net *bn, 3009 struct rtnl_link_stats64 *stats) 3010 { 3011 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3012 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3013 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3014 3015 stats->rx_packets += rx_save->packets; 3016 stats->tx_packets += tx_save->packets; 3017 stats->rx_bytes += rx_save->bytes; 3018 stats->tx_bytes += tx_save->bytes; 3019 stats->rx_missed_errors += rx_save->hw_drop_overruns; 3020 stats->tx_dropped += tx_save->hw_drop_errors; 3021 3022 stats->multicast += stats64_save->multicast; 3023 } 3024 3025 static void bnge_get_ring_stats64(struct bnge_dev *bd, 3026 struct rtnl_link_stats64 *stats) 3027 { 3028 struct bnge_net *bn = netdev_priv(bd->netdev); 3029 int i; 3030 3031 for (i = 0; i < bd->nq_nr_rings; i++) { 3032 struct bnge_napi *bnapi = bn->bnapi[i]; 3033 u64 tx_bytes, tx_packets, tx_dropped; 3034 u64 multicast, rx_missed_errors; 3035 struct bnge_nq_ring_info *nqr; 3036 u64 rx_bytes, rx_packets; 3037 unsigned int start; 3038 u64 *sw; 3039 3040 nqr = &bnapi->nq_ring; 3041 sw = nqr->stats.sw_stats; 3042 3043 do { 3044 start = u64_stats_fetch_begin(&nqr->stats.syncp); 3045 3046 rx_packets = BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3047 rx_packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3048 rx_packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3049 3050 tx_packets = BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3051 tx_packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3052 tx_packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3053 3054 rx_bytes = BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3055 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3056 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3057 3058 tx_bytes = BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3059 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3060 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3061 3062 multicast = BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3063 rx_missed_errors = 3064 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3065 tx_dropped = 3066 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3067 } while (u64_stats_fetch_retry(&nqr->stats.syncp, start)); 3068 3069 stats->rx_packets += rx_packets; 3070 stats->tx_packets += tx_packets; 3071 stats->rx_bytes += rx_bytes; 3072 stats->tx_bytes += tx_bytes; 3073 stats->multicast += multicast; 3074 stats->rx_missed_errors += rx_missed_errors; 3075 stats->tx_dropped += tx_dropped; 3076 } 3077 } 3078 3079 static void bnge_get_port_stats64(struct bnge_net *bn, 3080 struct rtnl_link_stats64 *stats) 3081 { 3082 unsigned int start; 3083 u64 *tx, *rx; 3084 3085 rx = bn->port_stats.sw_stats; 3086 tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 3087 3088 do { 3089 start = u64_stats_fetch_begin(&bn->port_stats.syncp); 3090 3091 stats->rx_crc_errors = 3092 BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 3093 stats->rx_frame_errors = 3094 BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 3095 stats->rx_length_errors = 3096 BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 3097 BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 3098 BNGE_GET_RX_PORT_STATS64(rx, rx_runt_frames); 3099 stats->rx_errors = 3100 BNGE_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 3101 BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 3102 stats->collisions = 3103 BNGE_GET_TX_PORT_STATS64(tx, tx_total_collisions); 3104 stats->tx_fifo_errors = 3105 BNGE_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 3106 stats->tx_errors = BNGE_GET_TX_PORT_STATS64(tx, tx_err); 3107 } while (u64_stats_fetch_retry(&bn->port_stats.syncp, start)); 3108 } 3109 3110 static void bnge_fill_prev_stats64(struct bnge_net *bn, 3111 struct rtnl_link_stats64 *stats) 3112 { 3113 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3114 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3115 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3116 3117 stats->rx_packets = rx_save->packets; 3118 stats->tx_packets = tx_save->packets; 3119 stats->rx_bytes = rx_save->bytes; 3120 stats->tx_bytes = tx_save->bytes; 3121 stats->rx_missed_errors = rx_save->hw_drop_overruns; 3122 stats->tx_dropped = tx_save->hw_drop_errors; 3123 stats->multicast = stats64_save->multicast; 3124 } 3125 3126 static void bnge_get_stats64(struct net_device *dev, 3127 struct rtnl_link_stats64 *stats) 3128 { 3129 struct bnge_net *bn = netdev_priv(dev); 3130 3131 if (bn->flags & BNGE_FLAG_PORT_STATS) 3132 bnge_get_port_stats64(bn, stats); 3133 3134 spin_lock_bh(&bn->stats_lock); 3135 if (!test_bit(BNGE_STATE_STATS_ENABLE, &bn->state)) { 3136 bnge_fill_prev_stats64(bn, stats); 3137 spin_unlock_bh(&bn->stats_lock); 3138 return; 3139 } 3140 3141 bnge_get_ring_stats64(bn->bd, stats); 3142 bnge_add_prev_ring_stats64(bn, stats); 3143 spin_unlock_bh(&bn->stats_lock); 3144 } 3145 3146 static void bnge_save_ring_stats(struct bnge_net *bn) 3147 { 3148 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3149 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3150 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3151 int i; 3152 3153 for (i = 0; i < bn->bd->nq_nr_rings; i++) { 3154 struct bnge_napi *bnapi = bn->bnapi[i]; 3155 struct bnge_nq_ring_info *nqr; 3156 u64 *sw; 3157 3158 nqr = &bnapi->nq_ring; 3159 sw = nqr->stats.sw_stats; 3160 3161 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3162 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3163 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3164 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3165 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3166 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3167 rx_save->hw_drop_overruns += BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3168 rx_save->hw_drops += BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3169 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3170 3171 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3172 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3173 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3174 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3175 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3176 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3177 tx_save->hw_drop_errors += BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3178 tx_save->hw_drops += BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3179 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3180 3181 stats64_save->multicast += 3182 BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3183 } 3184 } 3185 3186 static void bnge_close_core(struct bnge_net *bn) 3187 { 3188 struct bnge_dev *bd = bn->bd; 3189 3190 bnge_tx_disable(bn); 3191 3192 clear_bit(BNGE_STATE_OPEN, &bd->state); 3193 3194 timer_delete_sync(&bn->timer); 3195 bnge_shutdown_nic(bn); 3196 bnge_disable_napi(bn); 3197 3198 /* Save ring stats before shutdown */ 3199 spin_lock_bh(&bn->stats_lock); 3200 bnge_save_ring_stats(bn); 3201 clear_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 3202 spin_unlock_bh(&bn->stats_lock); 3203 3204 bnge_free_all_rings_bufs(bn); 3205 bnge_free_irq(bn); 3206 bnge_del_napi(bn); 3207 3208 bnge_free_core(bn); 3209 } 3210 3211 static int bnge_close(struct net_device *dev) 3212 { 3213 struct bnge_net *bn = netdev_priv(dev); 3214 3215 bnge_close_core(bn); 3216 bnge_hwrm_shutdown_link(bn->bd); 3217 bnge_hwrm_if_change(bn->bd, false); 3218 bn->sp_event = 0; 3219 3220 return 0; 3221 } 3222 3223 static void bnge_get_queue_stats_rx(struct net_device *dev, int i, 3224 struct netdev_queue_stats_rx *stats) 3225 { 3226 struct bnge_net *bn = netdev_priv(dev); 3227 struct bnge_nq_ring_info *nqr; 3228 u64 *sw; 3229 3230 if (!bn->bnapi) 3231 return; 3232 3233 nqr = &bn->bnapi[i]->nq_ring; 3234 sw = nqr->stats.sw_stats; 3235 3236 stats->packets = 0; 3237 stats->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3238 stats->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3239 stats->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3240 3241 stats->bytes = 0; 3242 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3243 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3244 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3245 3246 stats->hw_drop_overruns = BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3247 stats->hw_drops = BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3248 stats->hw_drop_overruns; 3249 } 3250 3251 static void bnge_get_queue_stats_tx(struct net_device *dev, int i, 3252 struct netdev_queue_stats_tx *stats) 3253 { 3254 struct bnge_net *bn = netdev_priv(dev); 3255 struct bnge_napi *bnapi; 3256 u64 *sw; 3257 3258 if (!bn->tx_ring) 3259 return; 3260 3261 bnapi = bn->tx_ring[bn->tx_ring_map[i]].bnapi; 3262 sw = bnapi->nq_ring.stats.sw_stats; 3263 3264 stats->packets = 0; 3265 stats->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3266 stats->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3267 stats->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3268 3269 stats->bytes = 0; 3270 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3271 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3272 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3273 3274 stats->hw_drop_errors = BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3275 stats->hw_drops = BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3276 stats->hw_drop_errors; 3277 } 3278 3279 static void bnge_get_base_stats(struct net_device *dev, 3280 struct netdev_queue_stats_rx *rx, 3281 struct netdev_queue_stats_tx *tx) 3282 { 3283 struct bnge_net *bn = netdev_priv(dev); 3284 3285 rx->packets = bn->rxq_prv_stats.packets; 3286 rx->bytes = bn->rxq_prv_stats.bytes; 3287 rx->hw_drops = bn->rxq_prv_stats.hw_drops; 3288 rx->hw_drop_overruns = bn->rxq_prv_stats.hw_drop_overruns; 3289 3290 tx->packets = bn->txq_prv_stats.packets; 3291 tx->bytes = bn->txq_prv_stats.bytes; 3292 tx->hw_drops = bn->txq_prv_stats.hw_drops; 3293 tx->hw_drop_errors = bn->txq_prv_stats.hw_drop_errors; 3294 } 3295 3296 static const struct netdev_stat_ops bnge_stat_ops = { 3297 .get_queue_stats_rx = bnge_get_queue_stats_rx, 3298 .get_queue_stats_tx = bnge_get_queue_stats_tx, 3299 .get_base_stats = bnge_get_base_stats, 3300 }; 3301 3302 static const struct net_device_ops bnge_netdev_ops = { 3303 .ndo_open = bnge_open, 3304 .ndo_stop = bnge_close, 3305 .ndo_start_xmit = bnge_start_xmit, 3306 .ndo_get_stats64 = bnge_get_stats64, 3307 .ndo_set_rx_mode_async = bnge_set_rx_mode, 3308 .ndo_features_check = bnge_features_check, 3309 }; 3310 3311 static void bnge_init_mac_addr(struct bnge_dev *bd) 3312 { 3313 eth_hw_addr_set(bd->netdev, bd->pf.mac_addr); 3314 } 3315 3316 static void bnge_set_tpa_flags(struct bnge_dev *bd) 3317 { 3318 struct bnge_net *bn = netdev_priv(bd->netdev); 3319 3320 bn->priv_flags &= ~BNGE_NET_EN_TPA; 3321 3322 if (bd->netdev->features & NETIF_F_LRO) 3323 bn->priv_flags |= BNGE_NET_EN_LRO; 3324 else if (bd->netdev->features & NETIF_F_GRO_HW) 3325 bn->priv_flags |= BNGE_NET_EN_GRO; 3326 } 3327 3328 static void bnge_init_l2_fltr_tbl(struct bnge_net *bn) 3329 { 3330 int i; 3331 3332 for (i = 0; i < BNGE_L2_FLTR_HASH_SIZE; i++) 3333 INIT_HLIST_HEAD(&bn->l2_fltr_hash_tbl[i]); 3334 get_random_bytes(&bn->hash_seed, sizeof(bn->hash_seed)); 3335 } 3336 3337 void bnge_set_ring_params(struct bnge_dev *bd) 3338 { 3339 struct bnge_net *bn = netdev_priv(bd->netdev); 3340 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3341 u32 agg_factor = 0, agg_ring_size = 0; 3342 3343 /* 8 for CRC and VLAN */ 3344 rx_size = SKB_DATA_ALIGN(bn->netdev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3345 3346 rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) + 3347 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3348 3349 ring_size = bn->rx_ring_size; 3350 bn->rx_agg_ring_size = 0; 3351 bn->rx_agg_nr_pages = 0; 3352 3353 if (bn->priv_flags & BNGE_NET_EN_TPA) 3354 agg_factor = min_t(u32, 4, 65536 / BNGE_RX_PAGE_SIZE); 3355 3356 bn->priv_flags &= ~BNGE_NET_EN_JUMBO; 3357 if (rx_space > PAGE_SIZE) { 3358 u32 jumbo_factor; 3359 3360 bn->priv_flags |= BNGE_NET_EN_JUMBO; 3361 jumbo_factor = PAGE_ALIGN(bn->netdev->mtu - 40) >> PAGE_SHIFT; 3362 if (jumbo_factor > agg_factor) 3363 agg_factor = jumbo_factor; 3364 } 3365 if (agg_factor) { 3366 if (ring_size > BNGE_MAX_RX_DESC_CNT_JUM_ENA) { 3367 ring_size = BNGE_MAX_RX_DESC_CNT_JUM_ENA; 3368 netdev_warn(bn->netdev, "RX ring size reduced from %d to %d due to jumbo ring\n", 3369 bn->rx_ring_size, ring_size); 3370 bn->rx_ring_size = ring_size; 3371 } 3372 agg_ring_size = ring_size * agg_factor; 3373 3374 bn->rx_agg_nr_pages = bnge_adjust_pow_two(agg_ring_size, 3375 RX_DESC_CNT); 3376 if (bn->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3377 u32 tmp = agg_ring_size; 3378 3379 bn->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3380 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3381 netdev_warn(bn->netdev, "RX agg ring size %d reduced to %d.\n", 3382 tmp, agg_ring_size); 3383 } 3384 bn->rx_agg_ring_size = agg_ring_size; 3385 bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3386 3387 rx_size = max3(BNGE_DEFAULT_RX_COPYBREAK, 3388 bn->rx_copybreak, 3389 bn->netdev->cfg_pending->hds_thresh); 3390 rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); 3391 rx_space = rx_size + NET_SKB_PAD + 3392 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3393 } 3394 3395 bn->rx_buf_use_size = rx_size; 3396 bn->rx_buf_size = rx_space; 3397 3398 bn->rx_nr_pages = bnge_adjust_pow_two(ring_size, RX_DESC_CNT); 3399 bn->rx_ring_mask = (bn->rx_nr_pages * RX_DESC_CNT) - 1; 3400 3401 ring_size = bn->tx_ring_size; 3402 bn->tx_nr_pages = bnge_adjust_pow_two(ring_size, TX_DESC_CNT); 3403 bn->tx_ring_mask = (bn->tx_nr_pages * TX_DESC_CNT) - 1; 3404 3405 max_rx_cmpl = bn->rx_ring_size; 3406 3407 if (bn->priv_flags & BNGE_NET_EN_TPA) 3408 max_rx_cmpl += bd->max_tpa_v2; 3409 ring_size = max_rx_cmpl * 2 + agg_ring_size + bn->tx_ring_size; 3410 bn->cp_ring_size = ring_size; 3411 3412 bn->cp_nr_pages = bnge_adjust_pow_two(ring_size, CP_DESC_CNT); 3413 if (bn->cp_nr_pages > MAX_CP_PAGES) { 3414 bn->cp_nr_pages = MAX_CP_PAGES; 3415 bn->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3416 netdev_warn(bn->netdev, "completion ring size %d reduced to %d.\n", 3417 ring_size, bn->cp_ring_size); 3418 } 3419 bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT; 3420 bn->cp_ring_mask = bn->cp_bit - 1; 3421 } 3422 3423 static void bnge_init_ring_params(struct bnge_net *bn) 3424 { 3425 u32 rx_size; 3426 3427 bn->rx_copybreak = BNGE_DEFAULT_RX_COPYBREAK; 3428 /* Try to fit 4 chunks into a 4k page */ 3429 rx_size = SZ_1K - 3430 NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3431 bn->netdev->cfg->hds_thresh = max(BNGE_DEFAULT_RX_COPYBREAK, rx_size); 3432 } 3433 3434 int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) 3435 { 3436 struct net_device *netdev; 3437 struct bnge_net *bn; 3438 int rc; 3439 3440 netdev = alloc_etherdev_mqs(sizeof(*bn), max_irqs * BNGE_MAX_QUEUE, 3441 max_irqs); 3442 if (!netdev) 3443 return -ENOMEM; 3444 3445 SET_NETDEV_DEV(netdev, bd->dev); 3446 bd->netdev = netdev; 3447 3448 netdev->netdev_ops = &bnge_netdev_ops; 3449 netdev->stat_ops = &bnge_stat_ops; 3450 3451 bnge_set_ethtool_ops(netdev); 3452 3453 bn = netdev_priv(netdev); 3454 bn->netdev = netdev; 3455 bn->bd = bd; 3456 3457 netdev->min_mtu = ETH_ZLEN; 3458 netdev->max_mtu = bd->max_mtu; 3459 3460 netdev->hw_features = NETIF_F_IP_CSUM | 3461 NETIF_F_IPV6_CSUM | 3462 NETIF_F_SG | 3463 NETIF_F_TSO | 3464 NETIF_F_TSO6 | 3465 NETIF_F_GSO_UDP_TUNNEL | 3466 NETIF_F_GSO_GRE | 3467 NETIF_F_GSO_IPXIP4 | 3468 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3469 NETIF_F_GSO_GRE_CSUM | 3470 NETIF_F_GSO_PARTIAL | 3471 NETIF_F_RXHASH | 3472 NETIF_F_RXCSUM | 3473 NETIF_F_GRO; 3474 3475 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3476 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 3477 3478 if (BNGE_SUPPORTS_TPA(bd)) 3479 netdev->hw_features |= NETIF_F_LRO; 3480 3481 netdev->hw_enc_features = NETIF_F_IP_CSUM | 3482 NETIF_F_IPV6_CSUM | 3483 NETIF_F_SG | 3484 NETIF_F_TSO | 3485 NETIF_F_TSO6 | 3486 NETIF_F_GSO_UDP_TUNNEL | 3487 NETIF_F_GSO_GRE | 3488 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3489 NETIF_F_GSO_GRE_CSUM | 3490 NETIF_F_GSO_IPXIP4 | 3491 NETIF_F_GSO_PARTIAL; 3492 3493 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3494 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 3495 3496 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 3497 NETIF_F_GSO_GRE_CSUM; 3498 3499 netdev->vlan_features = netdev->hw_features | NETIF_F_HIGHDMA; 3500 if (bd->fw_cap & BNGE_FW_CAP_VLAN_RX_STRIP) 3501 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_RX; 3502 if (bd->fw_cap & BNGE_FW_CAP_VLAN_TX_INSERT) 3503 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_TX; 3504 3505 if (BNGE_SUPPORTS_TPA(bd)) 3506 netdev->hw_features |= NETIF_F_GRO_HW; 3507 3508 netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA; 3509 3510 if (netdev->features & NETIF_F_GRO_HW) 3511 netdev->features &= ~NETIF_F_LRO; 3512 3513 netdev->priv_flags |= IFF_UNICAST_FLT; 3514 3515 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 3516 if (bd->tso_max_segs) 3517 netif_set_tso_max_segs(netdev, bd->tso_max_segs); 3518 3519 INIT_WORK(&bn->sp_task, bnge_sp_task); 3520 timer_setup(&bn->timer, bnge_timer, 0); 3521 bn->current_interval = BNGE_TIMER_INTERVAL; 3522 3523 bn->bnge_pf_wq = alloc_ordered_workqueue("bnge_pf_wq-%s", 0, 3524 dev_name(bd->dev)); 3525 if (!bn->bnge_pf_wq) { 3526 netdev_err(netdev, "Unable to create workqueue.\n"); 3527 rc = -ENOMEM; 3528 goto err_netdev; 3529 } 3530 3531 bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE; 3532 bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE; 3533 bn->rx_dir = DMA_FROM_DEVICE; 3534 3535 bnge_set_tpa_flags(bd); 3536 bnge_init_ring_params(bn); 3537 bnge_set_ring_params(bd); 3538 3539 bnge_init_l2_fltr_tbl(bn); 3540 bnge_init_mac_addr(bd); 3541 3542 rc = bnge_probe_phy(bn, true); 3543 if (rc) 3544 goto err_free_workq; 3545 3546 rc = bnge_alloc_port_stats(bn); 3547 if (rc) 3548 goto err_free_workq; 3549 spin_lock_init(&bn->stats_lock); 3550 3551 netdev->request_ops_lock = true; 3552 rc = register_netdev(netdev); 3553 if (rc) { 3554 dev_err(bd->dev, "Register netdev failed rc: %d\n", rc); 3555 goto err_free_port_stats; 3556 } 3557 3558 return 0; 3559 3560 err_free_port_stats: 3561 bnge_free_port_stats(bn); 3562 err_free_workq: 3563 destroy_workqueue(bn->bnge_pf_wq); 3564 err_netdev: 3565 free_netdev(netdev); 3566 bd->netdev = NULL; 3567 return rc; 3568 } 3569 3570 void bnge_netdev_free(struct bnge_dev *bd) 3571 { 3572 struct net_device *netdev = bd->netdev; 3573 struct bnge_net *bn; 3574 3575 bn = netdev_priv(netdev); 3576 3577 unregister_netdev(netdev); 3578 3579 timer_shutdown_sync(&bn->timer); 3580 cancel_work_sync(&bn->sp_task); 3581 bn->sp_event = 0; 3582 destroy_workqueue(bn->bnge_pf_wq); 3583 3584 bnge_free_port_stats(bn); 3585 3586 free_netdev(netdev); 3587 bd->netdev = NULL; 3588 } 3589