1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <asm/byteorder.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/dmapool.h> 7 #include <linux/delay.h> 8 #include <linux/errno.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/netdevice.h> 13 #include <net/netdev_lock.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if.h> 18 #include <net/ip.h> 19 #include <linux/skbuff.h> 20 #include <net/page_pool/helpers.h> 21 22 #include "bnge.h" 23 #include "bnge_hwrm_lib.h" 24 #include "bnge_ethtool.h" 25 #include "bnge_rmem.h" 26 #include "bnge_txrx.h" 27 28 #define BNGE_RING_TO_TC_OFF(bd, tx) \ 29 ((tx) % (bd)->tx_nr_rings_per_tc) 30 31 #define BNGE_RING_TO_TC(bd, tx) \ 32 ((tx) / (bd)->tx_nr_rings_per_tc) 33 34 #define BNGE_TC_TO_RING_BASE(bd, tc) \ 35 ((tc) * (bd)->tx_nr_rings_per_tc) 36 37 static void bnge_free_stats_mem(struct bnge_net *bn, 38 struct bnge_stats_mem *stats) 39 { 40 struct bnge_dev *bd = bn->bd; 41 42 kfree(stats->hw_masks); 43 stats->hw_masks = NULL; 44 kfree(stats->sw_stats); 45 stats->sw_stats = NULL; 46 if (stats->hw_stats) { 47 dma_free_coherent(bd->dev, stats->len, stats->hw_stats, 48 stats->hw_stats_map); 49 stats->hw_stats = NULL; 50 } 51 } 52 53 static int bnge_alloc_stats_mem(struct bnge_net *bn, 54 struct bnge_stats_mem *stats, bool alloc_masks) 55 { 56 struct bnge_dev *bd = bn->bd; 57 58 stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len, 59 &stats->hw_stats_map, GFP_KERNEL); 60 if (!stats->hw_stats) 61 return -ENOMEM; 62 63 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 64 if (!stats->sw_stats) 65 goto stats_mem_err; 66 67 if (alloc_masks) { 68 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 69 if (!stats->hw_masks) 70 goto stats_mem_err; 71 } 72 73 u64_stats_init(&stats->syncp); 74 75 return 0; 76 77 stats_mem_err: 78 bnge_free_stats_mem(bn, stats); 79 return -ENOMEM; 80 } 81 82 static void bnge_free_ring_stats(struct bnge_net *bn) 83 { 84 struct bnge_dev *bd = bn->bd; 85 int i; 86 87 if (!bn->bnapi) 88 return; 89 90 for (i = 0; i < bd->nq_nr_rings; i++) { 91 struct bnge_napi *bnapi = bn->bnapi[i]; 92 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 93 94 bnge_free_stats_mem(bn, &nqr->stats); 95 } 96 } 97 98 static void bnge_fill_masks(u64 *mask_arr, u64 mask, int count) 99 { 100 int i; 101 102 for (i = 0; i < count; i++) 103 mask_arr[i] = mask; 104 } 105 106 void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 107 { 108 int i; 109 110 for (i = 0; i < count; i++) 111 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 112 } 113 114 static void bnge_init_stats(struct bnge_net *bn) 115 { 116 struct bnge_napi *bnapi = bn->bnapi[0]; 117 struct bnge_nq_ring_info *nqr; 118 struct bnge_stats_mem *stats; 119 struct bnge_dev *bd = bn->bd; 120 __le64 *rx_stats, *tx_stats; 121 int rc, rx_count, tx_count; 122 u64 *rx_masks, *tx_masks; 123 u8 flags; 124 125 nqr = &bnapi->nq_ring; 126 stats = &nqr->stats; 127 rc = bnge_hwrm_func_qstat_ext(bd, stats); 128 if (rc) { 129 u64 mask = (1ULL << 48) - 1; 130 131 bnge_fill_masks(stats->hw_masks, mask, stats->len / 8); 132 } 133 134 if (bn->flags & BNGE_FLAG_PORT_STATS) { 135 stats = &bn->port_stats; 136 rx_stats = stats->hw_stats; 137 rx_masks = stats->hw_masks; 138 rx_count = sizeof(struct rx_port_stats) / 8; 139 tx_stats = rx_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 140 tx_masks = rx_masks + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 141 tx_count = sizeof(struct tx_port_stats) / 8; 142 143 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 144 rc = bnge_hwrm_port_qstats(bd, flags); 145 if (rc) { 146 u64 mask = (1ULL << 40) - 1; 147 148 bnge_fill_masks(rx_masks, mask, rx_count); 149 bnge_fill_masks(tx_masks, mask, tx_count); 150 } else { 151 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 152 bnge_copy_hw_masks(tx_masks, tx_stats, tx_count); 153 } 154 bnge_hwrm_port_qstats(bd, 0); 155 } 156 157 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 158 stats = &bn->rx_port_stats_ext; 159 rx_stats = stats->hw_stats; 160 rx_masks = stats->hw_masks; 161 rx_count = sizeof(struct rx_port_stats_ext) / 8; 162 stats = &bn->tx_port_stats_ext; 163 tx_stats = stats->hw_stats; 164 tx_masks = stats->hw_masks; 165 tx_count = sizeof(struct tx_port_stats_ext) / 8; 166 167 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 168 rc = bnge_hwrm_port_qstats_ext(bd, flags); 169 if (rc) { 170 u64 mask = (1ULL << 40) - 1; 171 172 bnge_fill_masks(rx_masks, mask, rx_count); 173 if (tx_stats) 174 bnge_fill_masks(tx_masks, mask, tx_count); 175 } else { 176 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 177 if (tx_stats) 178 bnge_copy_hw_masks(tx_masks, tx_stats, 179 tx_count); 180 } 181 bnge_hwrm_port_qstats_ext(bd, 0); 182 } 183 } 184 185 static void bnge_free_port_ext_stats(struct bnge_net *bn) 186 { 187 bn->flags &= ~BNGE_FLAG_PORT_STATS_EXT; 188 bnge_free_stats_mem(bn, &bn->rx_port_stats_ext); 189 bnge_free_stats_mem(bn, &bn->tx_port_stats_ext); 190 } 191 192 static void bnge_free_port_stats(struct bnge_net *bn) 193 { 194 bn->flags &= ~BNGE_FLAG_PORT_STATS; 195 bnge_free_stats_mem(bn, &bn->port_stats); 196 bnge_free_port_ext_stats(bn); 197 } 198 199 static int bnge_alloc_ring_stats(struct bnge_net *bn) 200 { 201 struct bnge_dev *bd = bn->bd; 202 u32 size, i; 203 int rc; 204 205 size = bd->hw_ring_stats_size; 206 207 for (i = 0; i < bd->nq_nr_rings; i++) { 208 struct bnge_napi *bnapi = bn->bnapi[i]; 209 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 210 211 nqr->stats.len = size; 212 rc = bnge_alloc_stats_mem(bn, &nqr->stats, !i); 213 if (rc) 214 goto err_free_ring_stats; 215 216 nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 217 } 218 219 return 0; 220 221 err_free_ring_stats: 222 bnge_free_ring_stats(bn); 223 return rc; 224 } 225 226 static void bnge_alloc_port_ext_stats(struct bnge_net *bn) 227 { 228 struct bnge_dev *bd = bn->bd; 229 int rc; 230 231 if (!(bd->fw_cap & BNGE_FW_CAP_EXT_STATS_SUPPORTED)) 232 return; 233 234 if (!bn->rx_port_stats_ext.hw_stats) { 235 bn->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 236 rc = bnge_alloc_stats_mem(bn, &bn->rx_port_stats_ext, true); 237 /* Extended stats are optional */ 238 if (rc) 239 return; 240 } 241 242 if (!bn->tx_port_stats_ext.hw_stats) { 243 bn->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 244 rc = bnge_alloc_stats_mem(bn, &bn->tx_port_stats_ext, true); 245 /* Extended stats are optional */ 246 if (rc) { 247 bnge_free_port_ext_stats(bn); 248 return; 249 } 250 } 251 bn->flags |= BNGE_FLAG_PORT_STATS_EXT; 252 } 253 254 static int bnge_alloc_port_stats(struct bnge_net *bn) 255 { 256 int rc; 257 258 if (!bn->port_stats.hw_stats) { 259 bn->port_stats.len = BNGE_PORT_STATS_SIZE; 260 rc = bnge_alloc_stats_mem(bn, &bn->port_stats, true); 261 if (rc) 262 return rc; 263 264 bn->flags |= BNGE_FLAG_PORT_STATS; 265 } 266 267 bnge_alloc_port_ext_stats(bn); 268 return 0; 269 } 270 271 void __bnge_queue_sp_work(struct bnge_net *bn) 272 { 273 queue_work(bn->bnge_pf_wq, &bn->sp_task); 274 } 275 276 static void bnge_queue_sp_work(struct bnge_net *bn, unsigned int event) 277 { 278 set_bit(event, &bn->sp_event); 279 __bnge_queue_sp_work(bn); 280 } 281 282 static void bnge_timer(struct timer_list *t) 283 { 284 struct bnge_net *bn = timer_container_of(bn, t, timer); 285 struct bnge_dev *bd = bn->bd; 286 287 if (!netif_running(bn->netdev) || 288 !test_bit(BNGE_STATE_OPEN, &bd->state)) 289 return; 290 291 if (READ_ONCE(bd->link_info.phy_retry)) { 292 if (time_after(jiffies, READ_ONCE(bd->link_info.phy_retry_expires))) { 293 WRITE_ONCE(bd->link_info.phy_retry, false); 294 netdev_warn(bn->netdev, "failed to update PHY settings after maximum retries.\n"); 295 } else { 296 bnge_queue_sp_work(bn, BNGE_UPDATE_PHY_SP_EVENT); 297 } 298 } 299 300 if (BNGE_LINK_IS_UP(bd) && bn->stats_coal_ticks) 301 bnge_queue_sp_work(bn, BNGE_PERIODIC_STATS_SP_EVENT); 302 303 mod_timer(&bn->timer, jiffies + bn->current_interval); 304 } 305 306 static void bnge_add_one_ctr(u64 hw, u64 *sw, u64 mask) 307 { 308 u64 sw_tmp, sw_val; 309 310 hw &= mask; 311 sw_val = READ_ONCE(*sw); 312 sw_tmp = (sw_val & ~mask) | hw; 313 if (hw < (sw_val & mask)) 314 sw_tmp += mask + 1; 315 WRITE_ONCE(*sw, sw_tmp); 316 } 317 318 static void __bnge_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 319 int count, struct u64_stats_sync *syncp) 320 { 321 unsigned long flags; 322 int i; 323 324 flags = u64_stats_update_begin_irqsave(syncp); 325 for (i = 0; i < count; i++) { 326 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 327 328 if (masks[i] == -1ULL) 329 WRITE_ONCE(sw_stats[i], hw); 330 else 331 bnge_add_one_ctr(hw, &sw_stats[i], masks[i]); 332 } 333 u64_stats_update_end_irqrestore(syncp, flags); 334 } 335 336 static void bnge_accumulate_stats(struct bnge_stats_mem *stats) 337 { 338 if (!stats->hw_stats) 339 return; 340 341 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 342 stats->hw_masks, stats->len / 8, &stats->syncp); 343 } 344 345 static void bnge_accumulate_all_stats(struct bnge_dev *bd) 346 { 347 struct bnge_net *bn = netdev_priv(bd->netdev); 348 struct bnge_stats_mem *ring0_stats = NULL; 349 int i; 350 351 for (i = 0; i < bd->nq_nr_rings; i++) { 352 struct bnge_napi *bnapi = bn->bnapi[i]; 353 struct bnge_nq_ring_info *nqr; 354 struct bnge_stats_mem *stats; 355 356 nqr = &bnapi->nq_ring; 357 stats = &nqr->stats; 358 359 if (!ring0_stats) 360 ring0_stats = &bn->bnapi[0]->nq_ring.stats; 361 362 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 363 ring0_stats->hw_masks, 364 ring0_stats->len / 8, &stats->syncp); 365 } 366 367 if (bn->flags & BNGE_FLAG_PORT_STATS) { 368 struct bnge_stats_mem *stats = &bn->port_stats; 369 __le64 *hw_stats = stats->hw_stats; 370 u64 *sw_stats = stats->sw_stats; 371 u64 *masks = stats->hw_masks; 372 u16 cnt; 373 374 cnt = sizeof(struct rx_port_stats) / 8; 375 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 376 &stats->syncp); 377 378 hw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 379 sw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 380 masks += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 381 cnt = sizeof(struct tx_port_stats) / 8; 382 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 383 &stats->syncp); 384 } 385 386 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 387 bnge_accumulate_stats(&bn->rx_port_stats_ext); 388 bnge_accumulate_stats(&bn->tx_port_stats_ext); 389 } 390 } 391 392 static void bnge_sp_task(struct work_struct *work) 393 { 394 struct bnge_net *bn = container_of(work, struct bnge_net, sp_task); 395 bool speed_chng, cfg_chng, link_chng; 396 struct bnge_dev *bd = bn->bd; 397 398 netdev_lock(bn->netdev); 399 if (!test_bit(BNGE_STATE_OPEN, &bd->state)) { 400 netdev_unlock(bn->netdev); 401 return; 402 } 403 404 if (test_and_clear_bit(BNGE_PERIODIC_STATS_SP_EVENT, &bn->sp_event)) { 405 bnge_hwrm_port_qstats(bd, 0); 406 bnge_hwrm_port_qstats_ext(bd, 0); 407 bnge_accumulate_all_stats(bd); 408 } 409 410 if (test_and_clear_bit(BNGE_UPDATE_PHY_SP_EVENT, &bn->sp_event)) { 411 int rc; 412 413 rc = bnge_update_phy_setting(bn); 414 if (rc) { 415 netdev_warn(bn->netdev, "update PHY settings retry failed\n"); 416 } else { 417 WRITE_ONCE(bd->link_info.phy_retry, false); 418 netdev_info(bn->netdev, "update PHY settings retry succeeded\n"); 419 } 420 } 421 422 speed_chng = test_and_clear_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, 423 &bn->sp_event); 424 cfg_chng = test_and_clear_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, 425 &bn->sp_event); 426 link_chng = test_and_clear_bit(BNGE_LINK_CHNG_SP_EVENT, 427 &bn->sp_event); 428 429 if (speed_chng || cfg_chng || link_chng) { 430 int rc; 431 432 if (speed_chng) 433 bnge_hwrm_phy_qcaps(bd); 434 435 rc = bnge_update_link(bn, true); 436 if (rc) 437 netdev_err(bn->netdev, "SP task cannot update link (rc: %d)\n", 438 rc); 439 440 if (speed_chng || cfg_chng) 441 bnge_init_ethtool_link_settings(bn); 442 } 443 444 netdev_unlock(bn->netdev); 445 } 446 447 static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr) 448 { 449 struct bnge_ring_struct *ring = &nqr->ring_struct; 450 451 kfree(nqr->desc_ring); 452 nqr->desc_ring = NULL; 453 ring->ring_mem.pg_arr = NULL; 454 kfree(nqr->desc_mapping); 455 nqr->desc_mapping = NULL; 456 ring->ring_mem.dma_arr = NULL; 457 } 458 459 static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr) 460 { 461 struct bnge_ring_struct *ring = &cpr->ring_struct; 462 463 kfree(cpr->desc_ring); 464 cpr->desc_ring = NULL; 465 ring->ring_mem.pg_arr = NULL; 466 kfree(cpr->desc_mapping); 467 cpr->desc_mapping = NULL; 468 ring->ring_mem.dma_arr = NULL; 469 } 470 471 static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n) 472 { 473 nqr->desc_ring = kzalloc_objs(*nqr->desc_ring, n); 474 if (!nqr->desc_ring) 475 return -ENOMEM; 476 477 nqr->desc_mapping = kzalloc_objs(*nqr->desc_mapping, n); 478 if (!nqr->desc_mapping) 479 goto err_free_desc_ring; 480 return 0; 481 482 err_free_desc_ring: 483 kfree(nqr->desc_ring); 484 nqr->desc_ring = NULL; 485 return -ENOMEM; 486 } 487 488 static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n) 489 { 490 cpr->desc_ring = kzalloc_objs(*cpr->desc_ring, n); 491 if (!cpr->desc_ring) 492 return -ENOMEM; 493 494 cpr->desc_mapping = kzalloc_objs(*cpr->desc_mapping, n); 495 if (!cpr->desc_mapping) 496 goto err_free_desc_ring; 497 return 0; 498 499 err_free_desc_ring: 500 kfree(cpr->desc_ring); 501 cpr->desc_ring = NULL; 502 return -ENOMEM; 503 } 504 505 static void bnge_free_nq_arrays(struct bnge_net *bn) 506 { 507 struct bnge_dev *bd = bn->bd; 508 int i; 509 510 for (i = 0; i < bd->nq_nr_rings; i++) { 511 struct bnge_napi *bnapi = bn->bnapi[i]; 512 513 bnge_free_nq_desc_arr(&bnapi->nq_ring); 514 } 515 } 516 517 static int bnge_alloc_nq_arrays(struct bnge_net *bn) 518 { 519 struct bnge_dev *bd = bn->bd; 520 int i, rc; 521 522 for (i = 0; i < bd->nq_nr_rings; i++) { 523 struct bnge_napi *bnapi = bn->bnapi[i]; 524 525 rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages); 526 if (rc) 527 goto err_free_nq_arrays; 528 } 529 return 0; 530 531 err_free_nq_arrays: 532 bnge_free_nq_arrays(bn); 533 return rc; 534 } 535 536 static void bnge_free_nq_tree(struct bnge_net *bn) 537 { 538 struct bnge_dev *bd = bn->bd; 539 int i; 540 541 for (i = 0; i < bd->nq_nr_rings; i++) { 542 struct bnge_napi *bnapi = bn->bnapi[i]; 543 struct bnge_nq_ring_info *nqr; 544 struct bnge_ring_struct *ring; 545 int j; 546 547 nqr = &bnapi->nq_ring; 548 ring = &nqr->ring_struct; 549 550 bnge_free_ring(bd, &ring->ring_mem); 551 552 if (!nqr->cp_ring_arr) 553 continue; 554 555 for (j = 0; j < nqr->cp_ring_count; j++) { 556 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 557 558 ring = &cpr->ring_struct; 559 bnge_free_ring(bd, &ring->ring_mem); 560 bnge_free_cp_desc_arr(cpr); 561 } 562 kfree(nqr->cp_ring_arr); 563 nqr->cp_ring_arr = NULL; 564 nqr->cp_ring_count = 0; 565 } 566 } 567 568 static int alloc_one_cp_ring(struct bnge_net *bn, 569 struct bnge_cp_ring_info *cpr) 570 { 571 struct bnge_ring_mem_info *rmem; 572 struct bnge_ring_struct *ring; 573 struct bnge_dev *bd = bn->bd; 574 int rc; 575 576 rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages); 577 if (rc) 578 return -ENOMEM; 579 ring = &cpr->ring_struct; 580 rmem = &ring->ring_mem; 581 rmem->nr_pages = bn->cp_nr_pages; 582 rmem->page_size = HW_CMPD_RING_SIZE; 583 rmem->pg_arr = (void **)cpr->desc_ring; 584 rmem->dma_arr = cpr->desc_mapping; 585 rmem->flags = BNGE_RMEM_RING_PTE_FLAG; 586 rc = bnge_alloc_ring(bd, rmem); 587 if (rc) 588 goto err_free_cp_desc_arr; 589 return rc; 590 591 err_free_cp_desc_arr: 592 bnge_free_cp_desc_arr(cpr); 593 return rc; 594 } 595 596 static int bnge_alloc_nq_tree(struct bnge_net *bn) 597 { 598 struct bnge_dev *bd = bn->bd; 599 int i, j, ulp_msix, rc; 600 int tcs = 1; 601 602 ulp_msix = bnge_aux_get_msix(bd); 603 for (i = 0, j = 0; i < bd->nq_nr_rings; i++) { 604 bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL); 605 struct bnge_napi *bnapi = bn->bnapi[i]; 606 struct bnge_nq_ring_info *nqr; 607 struct bnge_cp_ring_info *cpr; 608 struct bnge_ring_struct *ring; 609 int cp_count = 0, k; 610 int rx = 0, tx = 0; 611 612 nqr = &bnapi->nq_ring; 613 nqr->bnapi = bnapi; 614 ring = &nqr->ring_struct; 615 616 rc = bnge_alloc_ring(bd, &ring->ring_mem); 617 if (rc) 618 goto err_free_nq_tree; 619 620 ring->map_idx = ulp_msix + i; 621 622 if (i < bd->rx_nr_rings) { 623 cp_count++; 624 rx = 1; 625 } 626 627 if ((sh && i < bd->tx_nr_rings) || 628 (!sh && i >= bd->rx_nr_rings)) { 629 cp_count += tcs; 630 tx = 1; 631 } 632 633 nqr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); 634 if (!nqr->cp_ring_arr) { 635 rc = -ENOMEM; 636 goto err_free_nq_tree; 637 } 638 639 nqr->cp_ring_count = cp_count; 640 641 for (k = 0; k < cp_count; k++) { 642 cpr = &nqr->cp_ring_arr[k]; 643 rc = alloc_one_cp_ring(bn, cpr); 644 if (rc) 645 goto err_free_nq_tree; 646 647 cpr->bnapi = bnapi; 648 cpr->cp_idx = k; 649 if (!k && rx) { 650 bn->rx_ring[i].rx_cpr = cpr; 651 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX; 652 } else { 653 int n, tc = k - rx; 654 655 n = BNGE_TC_TO_RING_BASE(bd, tc) + j; 656 bn->tx_ring[n].tx_cpr = cpr; 657 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX; 658 } 659 } 660 if (tx) 661 j++; 662 } 663 return 0; 664 665 err_free_nq_tree: 666 bnge_free_nq_tree(bn); 667 return rc; 668 } 669 670 static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr) 671 { 672 return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE; 673 } 674 675 static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn, 676 struct bnge_rx_ring_info *rxr) 677 { 678 int i, max_idx; 679 680 if (!rxr->rx_buf_ring) 681 return; 682 683 max_idx = bn->rx_nr_pages * RX_DESC_CNT; 684 685 for (i = 0; i < max_idx; i++) { 686 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 687 void *data = rx_buf->data; 688 689 if (!data) 690 continue; 691 692 rx_buf->data = NULL; 693 page_pool_free_va(rxr->head_pool, data, true); 694 } 695 } 696 697 static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn, 698 struct bnge_rx_ring_info *rxr) 699 { 700 int i, max_idx; 701 702 if (!rxr->rx_agg_buf_ring) 703 return; 704 705 max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT; 706 707 for (i = 0; i < max_idx; i++) { 708 struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i]; 709 netmem_ref netmem = rx_agg_buf->netmem; 710 711 if (!netmem) 712 continue; 713 714 rx_agg_buf->netmem = 0; 715 __clear_bit(i, rxr->rx_agg_bmap); 716 717 page_pool_recycle_direct_netmem(rxr->page_pool, netmem); 718 } 719 } 720 721 static void bnge_free_one_tpa_info_data(struct bnge_net *bn, 722 struct bnge_rx_ring_info *rxr) 723 { 724 int i; 725 726 for (i = 0; i < bn->max_tpa; i++) { 727 struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[i]; 728 u8 *data = tpa_info->data; 729 730 if (!data) 731 continue; 732 733 tpa_info->data = NULL; 734 page_pool_free_va(rxr->head_pool, data, false); 735 } 736 } 737 738 static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn, 739 struct bnge_rx_ring_info *rxr) 740 { 741 struct bnge_tpa_idx_map *map; 742 743 if (rxr->rx_tpa) 744 bnge_free_one_tpa_info_data(bn, rxr); 745 746 bnge_free_one_rx_ring_bufs(bn, rxr); 747 bnge_free_one_agg_ring_bufs(bn, rxr); 748 749 map = rxr->rx_tpa_idx_map; 750 if (map) 751 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 752 } 753 754 static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn) 755 { 756 struct bnge_dev *bd = bn->bd; 757 int i; 758 759 if (!bn->rx_ring) 760 return; 761 762 for (i = 0; i < bd->rx_nr_rings; i++) 763 bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]); 764 } 765 766 static void bnge_free_tx_skbs(struct bnge_net *bn) 767 { 768 struct bnge_dev *bd = bn->bd; 769 u16 max_idx; 770 int i; 771 772 max_idx = bn->tx_nr_pages * TX_DESC_CNT; 773 for (i = 0; i < bd->tx_nr_rings; i++) { 774 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 775 int j; 776 777 if (!txr->tx_buf_ring) 778 continue; 779 780 for (j = 0; j < max_idx;) { 781 struct bnge_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 782 struct sk_buff *skb; 783 int k, last; 784 785 skb = tx_buf->skb; 786 if (!skb) { 787 j++; 788 continue; 789 } 790 791 tx_buf->skb = NULL; 792 793 dma_unmap_single(bd->dev, 794 dma_unmap_addr(tx_buf, mapping), 795 skb_headlen(skb), 796 DMA_TO_DEVICE); 797 798 last = tx_buf->nr_frags; 799 j += 2; 800 for (k = 0; k < last; k++, j++) { 801 int ring_idx = j & bn->tx_ring_mask; 802 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 803 804 tx_buf = &txr->tx_buf_ring[ring_idx]; 805 dma_unmap_page(bd->dev, 806 dma_unmap_addr(tx_buf, mapping), 807 skb_frag_size(frag), 808 DMA_TO_DEVICE); 809 } 810 dev_kfree_skb(skb); 811 } 812 netdev_tx_reset_queue(netdev_get_tx_queue(bd->netdev, i)); 813 } 814 } 815 816 static void bnge_free_all_rings_bufs(struct bnge_net *bn) 817 { 818 bnge_free_rx_ring_pair_bufs(bn); 819 bnge_free_tx_skbs(bn); 820 } 821 822 static void bnge_free_tpa_info(struct bnge_net *bn) 823 { 824 struct bnge_dev *bd = bn->bd; 825 int i, j; 826 827 for (i = 0; i < bd->rx_nr_rings; i++) { 828 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 829 830 kfree(rxr->rx_tpa_idx_map); 831 rxr->rx_tpa_idx_map = NULL; 832 if (rxr->rx_tpa) { 833 for (j = 0; j < bn->max_tpa; j++) { 834 kfree(rxr->rx_tpa[j].agg_arr); 835 rxr->rx_tpa[j].agg_arr = NULL; 836 } 837 } 838 kfree(rxr->rx_tpa); 839 rxr->rx_tpa = NULL; 840 } 841 } 842 843 static int bnge_alloc_tpa_info(struct bnge_net *bn) 844 { 845 struct bnge_dev *bd = bn->bd; 846 int i, j; 847 848 if (!bd->max_tpa_v2) 849 return 0; 850 851 bn->max_tpa = max_t(u16, bd->max_tpa_v2, MAX_TPA); 852 for (i = 0; i < bd->rx_nr_rings; i++) { 853 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 854 855 rxr->rx_tpa = kzalloc_objs(struct bnge_tpa_info, bn->max_tpa); 856 if (!rxr->rx_tpa) 857 goto err_free_tpa_info; 858 859 for (j = 0; j < bn->max_tpa; j++) { 860 struct rx_agg_cmp *agg; 861 862 agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); 863 if (!agg) 864 goto err_free_tpa_info; 865 rxr->rx_tpa[j].agg_arr = agg; 866 } 867 rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); 868 if (!rxr->rx_tpa_idx_map) 869 goto err_free_tpa_info; 870 } 871 return 0; 872 873 err_free_tpa_info: 874 bnge_free_tpa_info(bn); 875 return -ENOMEM; 876 } 877 878 static void bnge_free_rx_rings(struct bnge_net *bn) 879 { 880 struct bnge_dev *bd = bn->bd; 881 int i; 882 883 bnge_free_tpa_info(bn); 884 for (i = 0; i < bd->rx_nr_rings; i++) { 885 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 886 struct bnge_ring_struct *ring; 887 888 page_pool_destroy(rxr->page_pool); 889 page_pool_destroy(rxr->head_pool); 890 rxr->page_pool = rxr->head_pool = NULL; 891 892 kfree(rxr->rx_agg_bmap); 893 rxr->rx_agg_bmap = NULL; 894 895 ring = &rxr->rx_ring_struct; 896 bnge_free_ring(bd, &ring->ring_mem); 897 898 ring = &rxr->rx_agg_ring_struct; 899 bnge_free_ring(bd, &ring->ring_mem); 900 } 901 } 902 903 static int bnge_alloc_rx_page_pool(struct bnge_net *bn, 904 struct bnge_rx_ring_info *rxr, 905 int numa_node) 906 { 907 const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE; 908 const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; 909 struct page_pool_params pp = { 0 }; 910 struct bnge_dev *bd = bn->bd; 911 struct page_pool *pool; 912 913 pp.pool_size = bn->rx_agg_ring_size / agg_size_fac; 914 pp.nid = numa_node; 915 pp.netdev = bn->netdev; 916 pp.dev = bd->dev; 917 pp.dma_dir = bn->rx_dir; 918 pp.max_len = PAGE_SIZE; 919 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | 920 PP_FLAG_ALLOW_UNREADABLE_NETMEM; 921 pp.queue_idx = rxr->bnapi->index; 922 923 pool = page_pool_create(&pp); 924 if (IS_ERR(pool)) 925 return PTR_ERR(pool); 926 rxr->page_pool = pool; 927 928 rxr->need_head_pool = page_pool_is_unreadable(pool); 929 if (bnge_separate_head_pool(rxr)) { 930 pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024); 931 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 932 pool = page_pool_create(&pp); 933 if (IS_ERR(pool)) 934 goto err_destroy_pp; 935 } else { 936 page_pool_get(pool); 937 } 938 rxr->head_pool = pool; 939 return 0; 940 941 err_destroy_pp: 942 page_pool_destroy(rxr->page_pool); 943 rxr->page_pool = NULL; 944 return PTR_ERR(pool); 945 } 946 947 static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr) 948 { 949 page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi); 950 page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); 951 } 952 953 static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn, 954 struct bnge_rx_ring_info *rxr) 955 { 956 u16 mem_size; 957 958 rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1; 959 mem_size = rxr->rx_agg_bmap_size / 8; 960 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 961 if (!rxr->rx_agg_bmap) 962 return -ENOMEM; 963 964 return 0; 965 } 966 967 static int bnge_alloc_rx_rings(struct bnge_net *bn) 968 { 969 int i, rc = 0, agg_rings = 0, cpu; 970 struct bnge_dev *bd = bn->bd; 971 972 if (bnge_is_agg_reqd(bd)) 973 agg_rings = 1; 974 975 for (i = 0; i < bd->rx_nr_rings; i++) { 976 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 977 struct bnge_ring_struct *ring; 978 int cpu_node; 979 980 ring = &rxr->rx_ring_struct; 981 982 cpu = cpumask_local_spread(i, dev_to_node(bd->dev)); 983 cpu_node = cpu_to_node(cpu); 984 netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", 985 i, cpu_node); 986 rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node); 987 if (rc) 988 goto err_free_rx_rings; 989 bnge_enable_rx_page_pool(rxr); 990 991 rc = bnge_alloc_ring(bd, &ring->ring_mem); 992 if (rc) 993 goto err_free_rx_rings; 994 995 ring->grp_idx = i; 996 if (agg_rings) { 997 ring = &rxr->rx_agg_ring_struct; 998 rc = bnge_alloc_ring(bd, &ring->ring_mem); 999 if (rc) 1000 goto err_free_rx_rings; 1001 1002 ring->grp_idx = i; 1003 rc = bnge_alloc_rx_agg_bmap(bn, rxr); 1004 if (rc) 1005 goto err_free_rx_rings; 1006 } 1007 } 1008 1009 if (bn->priv_flags & BNGE_NET_EN_TPA) { 1010 rc = bnge_alloc_tpa_info(bn); 1011 if (rc) 1012 goto err_free_rx_rings; 1013 } 1014 return rc; 1015 1016 err_free_rx_rings: 1017 bnge_free_rx_rings(bn); 1018 return rc; 1019 } 1020 1021 static void bnge_free_tx_rings(struct bnge_net *bn) 1022 { 1023 struct bnge_dev *bd = bn->bd; 1024 int i; 1025 1026 for (i = 0; i < bd->tx_nr_rings; i++) { 1027 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1028 struct bnge_ring_struct *ring; 1029 1030 ring = &txr->tx_ring_struct; 1031 1032 bnge_free_ring(bd, &ring->ring_mem); 1033 } 1034 } 1035 1036 static int bnge_alloc_tx_rings(struct bnge_net *bn) 1037 { 1038 struct bnge_dev *bd = bn->bd; 1039 int i, j, rc; 1040 1041 for (i = 0, j = 0; i < bd->tx_nr_rings; i++) { 1042 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1043 struct bnge_ring_struct *ring; 1044 u8 qidx; 1045 1046 ring = &txr->tx_ring_struct; 1047 1048 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1049 if (rc) 1050 goto err_free_tx_rings; 1051 1052 ring->grp_idx = txr->bnapi->index; 1053 qidx = bd->tc_to_qidx[j]; 1054 ring->queue_id = bd->q_info[qidx].queue_id; 1055 if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1)) 1056 j++; 1057 } 1058 return 0; 1059 1060 err_free_tx_rings: 1061 bnge_free_tx_rings(bn); 1062 return rc; 1063 } 1064 1065 static void bnge_free_vnic_attributes(struct bnge_net *bn) 1066 { 1067 struct pci_dev *pdev = bn->bd->pdev; 1068 struct bnge_vnic_info *vnic; 1069 int i; 1070 1071 if (!bn->vnic_info) 1072 return; 1073 1074 for (i = 0; i < bn->nr_vnics; i++) { 1075 vnic = &bn->vnic_info[i]; 1076 1077 kfree(vnic->uc_list); 1078 vnic->uc_list = NULL; 1079 1080 if (vnic->mc_list) { 1081 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 1082 vnic->mc_list, vnic->mc_list_mapping); 1083 vnic->mc_list = NULL; 1084 } 1085 1086 if (vnic->rss_table) { 1087 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 1088 vnic->rss_table, 1089 vnic->rss_table_dma_addr); 1090 vnic->rss_table = NULL; 1091 } 1092 1093 vnic->rss_hash_key = NULL; 1094 vnic->flags = 0; 1095 } 1096 } 1097 1098 static int bnge_alloc_vnic_attributes(struct bnge_net *bn) 1099 { 1100 struct bnge_dev *bd = bn->bd; 1101 struct bnge_vnic_info *vnic; 1102 int i, size; 1103 1104 for (i = 0; i < bn->nr_vnics; i++) { 1105 vnic = &bn->vnic_info[i]; 1106 1107 if (vnic->flags & BNGE_VNIC_UCAST_FLAG) { 1108 int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN; 1109 1110 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 1111 if (!vnic->uc_list) 1112 goto err_free_vnic_attributes; 1113 } 1114 1115 if (vnic->flags & BNGE_VNIC_MCAST_FLAG) { 1116 vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN; 1117 vnic->mc_list = 1118 dma_alloc_coherent(bd->dev, 1119 vnic->mc_list_size, 1120 &vnic->mc_list_mapping, 1121 GFP_KERNEL); 1122 if (!vnic->mc_list) 1123 goto err_free_vnic_attributes; 1124 } 1125 1126 /* Allocate rss table and hash key */ 1127 size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE); 1128 1129 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 1130 vnic->rss_table = dma_alloc_coherent(bd->dev, 1131 vnic->rss_table_size, 1132 &vnic->rss_table_dma_addr, 1133 GFP_KERNEL); 1134 if (!vnic->rss_table) 1135 goto err_free_vnic_attributes; 1136 1137 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 1138 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 1139 } 1140 return 0; 1141 1142 err_free_vnic_attributes: 1143 bnge_free_vnic_attributes(bn); 1144 return -ENOMEM; 1145 } 1146 1147 static int bnge_alloc_vnics(struct bnge_net *bn) 1148 { 1149 int num_vnics; 1150 1151 /* Allocate only 1 VNIC for now 1152 * Additional VNICs will be added based on RFS/NTUPLE in future patches 1153 */ 1154 num_vnics = 1; 1155 1156 bn->vnic_info = kzalloc_objs(struct bnge_vnic_info, num_vnics); 1157 if (!bn->vnic_info) 1158 return -ENOMEM; 1159 1160 bn->nr_vnics = num_vnics; 1161 1162 return 0; 1163 } 1164 1165 static void bnge_free_vnics(struct bnge_net *bn) 1166 { 1167 kfree(bn->vnic_info); 1168 bn->vnic_info = NULL; 1169 bn->nr_vnics = 0; 1170 } 1171 1172 static void bnge_free_ring_grps(struct bnge_net *bn) 1173 { 1174 kfree(bn->grp_info); 1175 bn->grp_info = NULL; 1176 } 1177 1178 static int bnge_init_ring_grps(struct bnge_net *bn) 1179 { 1180 struct bnge_dev *bd = bn->bd; 1181 int i; 1182 1183 bn->grp_info = kzalloc_objs(struct bnge_ring_grp_info, bd->nq_nr_rings); 1184 if (!bn->grp_info) 1185 return -ENOMEM; 1186 for (i = 0; i < bd->nq_nr_rings; i++) { 1187 bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 1188 bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 1189 bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 1190 bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 1191 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 1192 } 1193 1194 return 0; 1195 } 1196 1197 static void bnge_free_core(struct bnge_net *bn) 1198 { 1199 bnge_free_vnic_attributes(bn); 1200 bnge_free_tx_rings(bn); 1201 bnge_free_rx_rings(bn); 1202 bnge_free_nq_tree(bn); 1203 bnge_free_nq_arrays(bn); 1204 bnge_free_ring_stats(bn); 1205 bnge_free_ring_grps(bn); 1206 bnge_free_vnics(bn); 1207 kfree(bn->tx_ring_map); 1208 bn->tx_ring_map = NULL; 1209 kfree(bn->tx_ring); 1210 bn->tx_ring = NULL; 1211 kfree(bn->rx_ring); 1212 bn->rx_ring = NULL; 1213 kfree(bn->bnapi); 1214 bn->bnapi = NULL; 1215 } 1216 1217 static int bnge_alloc_core(struct bnge_net *bn) 1218 { 1219 struct bnge_dev *bd = bn->bd; 1220 int i, j, size, arr_size; 1221 int rc = -ENOMEM; 1222 void *bnapi; 1223 1224 arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) * 1225 bd->nq_nr_rings); 1226 size = L1_CACHE_ALIGN(sizeof(struct bnge_napi)); 1227 bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL); 1228 if (!bnapi) 1229 return rc; 1230 1231 bn->bnapi = bnapi; 1232 bnapi += arr_size; 1233 for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) { 1234 struct bnge_nq_ring_info *nqr; 1235 1236 bn->bnapi[i] = bnapi; 1237 bn->bnapi[i]->index = i; 1238 bn->bnapi[i]->bn = bn; 1239 nqr = &bn->bnapi[i]->nq_ring; 1240 nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1241 } 1242 1243 bn->rx_ring = kzalloc_objs(struct bnge_rx_ring_info, bd->rx_nr_rings); 1244 if (!bn->rx_ring) 1245 goto err_free_core; 1246 1247 for (i = 0; i < bd->rx_nr_rings; i++) { 1248 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1249 1250 rxr->rx_ring_struct.ring_mem.flags = 1251 BNGE_RMEM_RING_PTE_FLAG; 1252 rxr->rx_agg_ring_struct.ring_mem.flags = 1253 BNGE_RMEM_RING_PTE_FLAG; 1254 rxr->bnapi = bn->bnapi[i]; 1255 bn->bnapi[i]->rx_ring = &bn->rx_ring[i]; 1256 } 1257 1258 bn->tx_ring = kzalloc_objs(struct bnge_tx_ring_info, bd->tx_nr_rings); 1259 if (!bn->tx_ring) 1260 goto err_free_core; 1261 1262 bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16), 1263 GFP_KERNEL); 1264 if (!bn->tx_ring_map) 1265 goto err_free_core; 1266 1267 if (bd->flags & BNGE_EN_SHARED_CHNL) 1268 j = 0; 1269 else 1270 j = bd->rx_nr_rings; 1271 1272 for (i = 0; i < bd->tx_nr_rings; i++) { 1273 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1274 struct bnge_napi *bnapi2; 1275 int k; 1276 1277 txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1278 bn->tx_ring_map[i] = i; 1279 k = j + BNGE_RING_TO_TC_OFF(bd, i); 1280 1281 bnapi2 = bn->bnapi[k]; 1282 txr->txq_index = i; 1283 txr->tx_napi_idx = 1284 BNGE_RING_TO_TC(bd, txr->txq_index); 1285 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 1286 txr->bnapi = bnapi2; 1287 } 1288 1289 rc = bnge_alloc_ring_stats(bn); 1290 if (rc) 1291 goto err_free_core; 1292 1293 bnge_init_stats(bn); 1294 1295 rc = bnge_alloc_vnics(bn); 1296 if (rc) 1297 goto err_free_core; 1298 1299 rc = bnge_alloc_nq_arrays(bn); 1300 if (rc) 1301 goto err_free_core; 1302 1303 bnge_init_ring_struct(bn); 1304 1305 rc = bnge_alloc_rx_rings(bn); 1306 if (rc) 1307 goto err_free_core; 1308 1309 rc = bnge_alloc_tx_rings(bn); 1310 if (rc) 1311 goto err_free_core; 1312 1313 rc = bnge_alloc_nq_tree(bn); 1314 if (rc) 1315 goto err_free_core; 1316 1317 bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG | 1318 BNGE_VNIC_MCAST_FLAG | 1319 BNGE_VNIC_UCAST_FLAG; 1320 rc = bnge_alloc_vnic_attributes(bn); 1321 if (rc) 1322 goto err_free_core; 1323 return 0; 1324 1325 err_free_core: 1326 bnge_free_core(bn); 1327 return rc; 1328 } 1329 1330 u16 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr) 1331 { 1332 return rxr->rx_cpr->ring_struct.fw_ring_id; 1333 } 1334 1335 u16 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr) 1336 { 1337 return txr->tx_cpr->ring_struct.fw_ring_id; 1338 } 1339 1340 static void bnge_db_nq_arm(struct bnge_net *bn, 1341 struct bnge_db_info *db, u32 idx) 1342 { 1343 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_ARM | 1344 DB_RING_IDX(db, idx), db->doorbell); 1345 } 1346 1347 static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1348 { 1349 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK | 1350 DB_RING_IDX(db, idx), db->doorbell); 1351 } 1352 1353 static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1354 { 1355 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL | 1356 DB_RING_IDX(db, idx), db->doorbell); 1357 } 1358 1359 static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n) 1360 { 1361 struct bnge_napi *bnapi = bn->bnapi[n]; 1362 struct bnge_nq_ring_info *nqr; 1363 1364 nqr = &bnapi->nq_ring; 1365 1366 return nqr->ring_struct.map_idx; 1367 } 1368 1369 static void bnge_init_nq_tree(struct bnge_net *bn) 1370 { 1371 struct bnge_dev *bd = bn->bd; 1372 int i, j; 1373 1374 for (i = 0; i < bd->nq_nr_rings; i++) { 1375 struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring; 1376 struct bnge_ring_struct *ring = &nqr->ring_struct; 1377 1378 ring->fw_ring_id = INVALID_HW_RING_ID; 1379 for (j = 0; j < nqr->cp_ring_count; j++) { 1380 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 1381 1382 ring = &cpr->ring_struct; 1383 ring->fw_ring_id = INVALID_HW_RING_ID; 1384 } 1385 } 1386 } 1387 1388 static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn, 1389 dma_addr_t *mapping, 1390 struct bnge_rx_ring_info *rxr, 1391 unsigned int *offset, 1392 gfp_t gfp) 1393 { 1394 netmem_ref netmem; 1395 1396 if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) { 1397 netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, 1398 BNGE_RX_PAGE_SIZE, gfp); 1399 } else { 1400 netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); 1401 *offset = 0; 1402 } 1403 if (!netmem) 1404 return 0; 1405 1406 *mapping = page_pool_get_dma_addr_netmem(netmem) + *offset; 1407 return netmem; 1408 } 1409 1410 u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping, 1411 struct bnge_rx_ring_info *rxr, 1412 gfp_t gfp) 1413 { 1414 unsigned int offset; 1415 struct page *page; 1416 1417 page = page_pool_alloc_frag(rxr->head_pool, &offset, 1418 bn->rx_buf_size, gfp); 1419 if (!page) 1420 return NULL; 1421 1422 *mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset; 1423 return page_address(page) + offset; 1424 } 1425 1426 int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr, 1427 u16 prod, gfp_t gfp) 1428 { 1429 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)]; 1430 struct rx_bd *rxbd; 1431 dma_addr_t mapping; 1432 u8 *data; 1433 1434 rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)]; 1435 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp); 1436 if (!data) 1437 return -ENOMEM; 1438 1439 rx_buf->data = data; 1440 rx_buf->data_ptr = data + bn->rx_offset; 1441 rx_buf->mapping = mapping; 1442 1443 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1444 1445 return 0; 1446 } 1447 1448 static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn, 1449 struct bnge_rx_ring_info *rxr, 1450 int ring_nr) 1451 { 1452 u32 prod = rxr->rx_prod; 1453 int i, rc = 0; 1454 1455 for (i = 0; i < bn->rx_ring_size; i++) { 1456 rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL); 1457 if (rc) 1458 break; 1459 prod = NEXT_RX(prod); 1460 } 1461 1462 /* Abort if not a single buffer can be allocated */ 1463 if (rc && !i) { 1464 netdev_err(bn->netdev, 1465 "RX ring %d: allocated %d/%d buffers, abort\n", 1466 ring_nr, i, bn->rx_ring_size); 1467 return rc; 1468 } 1469 1470 rxr->rx_prod = prod; 1471 1472 if (i < bn->rx_ring_size) 1473 netdev_warn(bn->netdev, 1474 "RX ring %d: allocated %d/%d buffers, continuing\n", 1475 ring_nr, i, bn->rx_ring_size); 1476 return 0; 1477 } 1478 1479 u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx) 1480 { 1481 u16 next, max = rxr->rx_agg_bmap_size; 1482 1483 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 1484 if (next >= max) 1485 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 1486 return next; 1487 } 1488 1489 int bnge_alloc_rx_netmem(struct bnge_net *bn, 1490 struct bnge_rx_ring_info *rxr, 1491 u16 prod, gfp_t gfp) 1492 { 1493 struct bnge_sw_rx_agg_bd *rx_agg_buf; 1494 u16 sw_prod = rxr->rx_sw_agg_prod; 1495 unsigned int offset = 0; 1496 struct rx_bd *rxbd; 1497 dma_addr_t mapping; 1498 netmem_ref netmem; 1499 1500 rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)]; 1501 netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp); 1502 if (!netmem) 1503 return -ENOMEM; 1504 1505 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1506 sw_prod = bnge_find_next_agg_idx(rxr, sw_prod); 1507 1508 __set_bit(sw_prod, rxr->rx_agg_bmap); 1509 rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod]; 1510 rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod)); 1511 1512 rx_agg_buf->netmem = netmem; 1513 rx_agg_buf->offset = offset; 1514 rx_agg_buf->mapping = mapping; 1515 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1516 rxbd->rx_bd_opaque = sw_prod; 1517 return 0; 1518 } 1519 1520 static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn, 1521 struct bnge_rx_ring_info *rxr, 1522 int ring_nr) 1523 { 1524 u32 prod = rxr->rx_agg_prod; 1525 int i, rc = 0; 1526 1527 for (i = 0; i < bn->rx_agg_ring_size; i++) { 1528 rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL); 1529 if (rc) 1530 break; 1531 prod = NEXT_RX_AGG(prod); 1532 } 1533 1534 if (rc && i < MAX_SKB_FRAGS) { 1535 netdev_err(bn->netdev, 1536 "Agg ring %d: allocated %d/%d buffers (min %d), abort\n", 1537 ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS); 1538 goto err_free_one_agg_ring_bufs; 1539 } 1540 1541 rxr->rx_agg_prod = prod; 1542 1543 if (i < bn->rx_agg_ring_size) 1544 netdev_warn(bn->netdev, 1545 "Agg ring %d: allocated %d/%d buffers, continuing\n", 1546 ring_nr, i, bn->rx_agg_ring_size); 1547 return 0; 1548 1549 err_free_one_agg_ring_bufs: 1550 bnge_free_one_agg_ring_bufs(bn, rxr); 1551 return -ENOMEM; 1552 } 1553 1554 static int bnge_alloc_one_tpa_info_data(struct bnge_net *bn, 1555 struct bnge_rx_ring_info *rxr) 1556 { 1557 dma_addr_t mapping; 1558 u8 *data; 1559 int i; 1560 1561 for (i = 0; i < bn->max_tpa; i++) { 1562 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, 1563 GFP_KERNEL); 1564 if (!data) 1565 goto err_free_tpa_info_data; 1566 1567 rxr->rx_tpa[i].data = data; 1568 rxr->rx_tpa[i].data_ptr = data + bn->rx_offset; 1569 rxr->rx_tpa[i].mapping = mapping; 1570 } 1571 return 0; 1572 1573 err_free_tpa_info_data: 1574 bnge_free_one_tpa_info_data(bn, rxr); 1575 return -ENOMEM; 1576 } 1577 1578 static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr) 1579 { 1580 struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr]; 1581 int rc; 1582 1583 rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr); 1584 if (rc) 1585 return rc; 1586 1587 if (bnge_is_agg_reqd(bn->bd)) { 1588 rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr); 1589 if (rc) 1590 goto err_free_one_rx_ring_bufs; 1591 } 1592 1593 if (rxr->rx_tpa) { 1594 rc = bnge_alloc_one_tpa_info_data(bn, rxr); 1595 if (rc) 1596 goto err_free_one_agg_ring_bufs; 1597 } 1598 1599 return 0; 1600 1601 err_free_one_agg_ring_bufs: 1602 bnge_free_one_agg_ring_bufs(bn, rxr); 1603 err_free_one_rx_ring_bufs: 1604 bnge_free_one_rx_ring_bufs(bn, rxr); 1605 return rc; 1606 } 1607 1608 static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type) 1609 { 1610 struct rx_bd **rx_desc_ring; 1611 u32 prod; 1612 int i; 1613 1614 rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 1615 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 1616 struct rx_bd *rxbd = rx_desc_ring[i]; 1617 int j; 1618 1619 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 1620 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 1621 rxbd->rx_bd_opaque = prod; 1622 } 1623 } 1624 } 1625 1626 static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn, 1627 struct bnge_rx_ring_info *rxr) 1628 { 1629 struct bnge_ring_struct *ring; 1630 u32 type; 1631 1632 type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) | 1633 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 1634 1635 if (NET_IP_ALIGN == 2) 1636 type |= RX_BD_FLAGS_SOP; 1637 1638 ring = &rxr->rx_ring_struct; 1639 bnge_init_rxbd_pages(ring, type); 1640 ring->fw_ring_id = INVALID_HW_RING_ID; 1641 } 1642 1643 static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn, 1644 struct bnge_rx_ring_info *rxr) 1645 { 1646 struct bnge_ring_struct *ring; 1647 u32 type; 1648 1649 ring = &rxr->rx_agg_ring_struct; 1650 ring->fw_ring_id = INVALID_HW_RING_ID; 1651 if (bnge_is_agg_reqd(bn->bd)) { 1652 type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 1653 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 1654 1655 bnge_init_rxbd_pages(ring, type); 1656 } 1657 } 1658 1659 static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr) 1660 { 1661 struct bnge_rx_ring_info *rxr; 1662 1663 rxr = &bn->rx_ring[ring_nr]; 1664 bnge_init_one_rx_ring_rxbd(bn, rxr); 1665 1666 netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX, 1667 &rxr->bnapi->napi); 1668 1669 bnge_init_one_agg_ring_rxbd(bn, rxr); 1670 } 1671 1672 static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn) 1673 { 1674 int i, rc; 1675 1676 for (i = 0; i < bn->bd->rx_nr_rings; i++) { 1677 rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i); 1678 if (rc) 1679 goto err_free_rx_ring_pair_bufs; 1680 } 1681 return 0; 1682 1683 err_free_rx_ring_pair_bufs: 1684 bnge_free_rx_ring_pair_bufs(bn); 1685 return rc; 1686 } 1687 1688 static void bnge_init_rx_rings(struct bnge_net *bn) 1689 { 1690 int i; 1691 1692 #define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 1693 #define BNGE_RX_DMA_OFFSET NET_SKB_PAD 1694 bn->rx_offset = BNGE_RX_OFFSET; 1695 bn->rx_dma_offset = BNGE_RX_DMA_OFFSET; 1696 1697 for (i = 0; i < bn->bd->rx_nr_rings; i++) 1698 bnge_init_one_rx_ring_pair(bn, i); 1699 } 1700 1701 static void bnge_init_tx_rings(struct bnge_net *bn) 1702 { 1703 int i; 1704 1705 bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT); 1706 1707 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 1708 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1709 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1710 1711 ring->fw_ring_id = INVALID_HW_RING_ID; 1712 1713 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, 1714 &txr->bnapi->napi); 1715 } 1716 } 1717 1718 static void bnge_init_vnics(struct bnge_net *bn) 1719 { 1720 struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 1721 int i; 1722 1723 for (i = 0; i < bn->nr_vnics; i++) { 1724 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 1725 int j; 1726 1727 vnic->fw_vnic_id = INVALID_HW_RING_ID; 1728 vnic->vnic_id = i; 1729 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) 1730 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 1731 1732 if (bn->vnic_info[i].rss_hash_key) { 1733 if (i == BNGE_VNIC_DEFAULT) { 1734 u8 *key = (void *)vnic->rss_hash_key; 1735 int k; 1736 1737 if (!bn->rss_hash_key_valid && 1738 !bn->rss_hash_key_updated) { 1739 get_random_bytes(bn->rss_hash_key, 1740 HW_HASH_KEY_SIZE); 1741 bn->rss_hash_key_updated = true; 1742 } 1743 1744 memcpy(vnic->rss_hash_key, bn->rss_hash_key, 1745 HW_HASH_KEY_SIZE); 1746 1747 if (!bn->rss_hash_key_updated) 1748 continue; 1749 1750 bn->rss_hash_key_updated = false; 1751 bn->rss_hash_key_valid = true; 1752 1753 bn->toeplitz_prefix = 0; 1754 for (k = 0; k < 8; k++) { 1755 bn->toeplitz_prefix <<= 8; 1756 bn->toeplitz_prefix |= key[k]; 1757 } 1758 } else { 1759 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 1760 HW_HASH_KEY_SIZE); 1761 } 1762 } 1763 } 1764 } 1765 1766 static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db, 1767 u32 ring_type) 1768 { 1769 switch (ring_type) { 1770 case HWRM_RING_ALLOC_TX: 1771 db->db_ring_mask = bn->tx_ring_mask; 1772 break; 1773 case HWRM_RING_ALLOC_RX: 1774 db->db_ring_mask = bn->rx_ring_mask; 1775 break; 1776 case HWRM_RING_ALLOC_AGG: 1777 db->db_ring_mask = bn->rx_agg_ring_mask; 1778 break; 1779 case HWRM_RING_ALLOC_CMPL: 1780 case HWRM_RING_ALLOC_NQ: 1781 db->db_ring_mask = bn->cp_ring_mask; 1782 break; 1783 } 1784 db->db_epoch_mask = db->db_ring_mask + 1; 1785 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 1786 } 1787 1788 static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db, 1789 u32 ring_type, u32 map_idx, u32 xid) 1790 { 1791 struct bnge_dev *bd = bn->bd; 1792 1793 switch (ring_type) { 1794 case HWRM_RING_ALLOC_TX: 1795 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 1796 break; 1797 case HWRM_RING_ALLOC_RX: 1798 case HWRM_RING_ALLOC_AGG: 1799 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 1800 break; 1801 case HWRM_RING_ALLOC_CMPL: 1802 db->db_key64 = DBR_PATH_L2; 1803 break; 1804 case HWRM_RING_ALLOC_NQ: 1805 db->db_key64 = DBR_PATH_L2; 1806 break; 1807 } 1808 db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID; 1809 1810 db->doorbell = bd->bar1 + bd->db_offset; 1811 bnge_set_db_mask(bn, db, ring_type); 1812 } 1813 1814 static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn, 1815 struct bnge_cp_ring_info *cpr) 1816 { 1817 const u32 type = HWRM_RING_ALLOC_CMPL; 1818 struct bnge_napi *bnapi = cpr->bnapi; 1819 struct bnge_ring_struct *ring; 1820 u32 map_idx = bnapi->index; 1821 int rc; 1822 1823 ring = &cpr->ring_struct; 1824 ring->handle = BNGE_SET_NQ_HDL(cpr); 1825 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1826 if (rc) 1827 return rc; 1828 1829 bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 1830 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1831 1832 return 0; 1833 } 1834 1835 static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn, 1836 struct bnge_tx_ring_info *txr, u32 tx_idx) 1837 { 1838 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1839 const u32 type = HWRM_RING_ALLOC_TX; 1840 int rc; 1841 1842 rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx); 1843 if (rc) 1844 return rc; 1845 1846 bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 1847 1848 return 0; 1849 } 1850 1851 static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn, 1852 struct bnge_rx_ring_info *rxr) 1853 { 1854 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 1855 u32 type = HWRM_RING_ALLOC_AGG; 1856 struct bnge_dev *bd = bn->bd; 1857 u32 grp_idx = ring->grp_idx; 1858 u32 map_idx; 1859 int rc; 1860 1861 map_idx = grp_idx + bd->rx_nr_rings; 1862 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1863 if (rc) 1864 return rc; 1865 1866 bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx, 1867 ring->fw_ring_id); 1868 bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod); 1869 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1870 bn->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 1871 1872 return 0; 1873 } 1874 1875 static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn, 1876 struct bnge_rx_ring_info *rxr) 1877 { 1878 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 1879 struct bnge_napi *bnapi = rxr->bnapi; 1880 u32 type = HWRM_RING_ALLOC_RX; 1881 u32 map_idx = bnapi->index; 1882 int rc; 1883 1884 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1885 if (rc) 1886 return rc; 1887 1888 bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 1889 bn->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 1890 1891 return 0; 1892 } 1893 1894 static int bnge_hwrm_ring_alloc(struct bnge_net *bn) 1895 { 1896 struct bnge_dev *bd = bn->bd; 1897 bool agg_rings; 1898 int i, rc = 0; 1899 1900 agg_rings = !!(bnge_is_agg_reqd(bd)); 1901 for (i = 0; i < bd->nq_nr_rings; i++) { 1902 struct bnge_napi *bnapi = bn->bnapi[i]; 1903 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 1904 struct bnge_ring_struct *ring = &nqr->ring_struct; 1905 u32 type = HWRM_RING_ALLOC_NQ; 1906 u32 map_idx = ring->map_idx; 1907 unsigned int vector; 1908 1909 vector = bd->irq_tbl[map_idx].vector; 1910 disable_irq_nosync(vector); 1911 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1912 if (rc) { 1913 enable_irq(vector); 1914 goto err_out; 1915 } 1916 bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id); 1917 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 1918 enable_irq(vector); 1919 bn->grp_info[i].nq_fw_ring_id = ring->fw_ring_id; 1920 1921 if (!i) { 1922 rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id); 1923 if (rc) 1924 netdev_warn(bn->netdev, "Failed to set async event completion ring.\n"); 1925 } 1926 } 1927 1928 for (i = 0; i < bd->tx_nr_rings; i++) { 1929 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1930 1931 rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr); 1932 if (rc) 1933 goto err_out; 1934 rc = bnge_hwrm_tx_ring_alloc(bn, txr, i); 1935 if (rc) 1936 goto err_out; 1937 } 1938 1939 for (i = 0; i < bd->rx_nr_rings; i++) { 1940 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1941 struct bnge_cp_ring_info *cpr; 1942 struct bnge_ring_struct *ring; 1943 struct bnge_napi *bnapi; 1944 u32 map_idx, type; 1945 1946 rc = bnge_hwrm_rx_ring_alloc(bn, rxr); 1947 if (rc) 1948 goto err_out; 1949 /* If we have agg rings, post agg buffers first. */ 1950 if (!agg_rings) 1951 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1952 1953 cpr = rxr->rx_cpr; 1954 bnapi = rxr->bnapi; 1955 type = HWRM_RING_ALLOC_CMPL; 1956 map_idx = bnapi->index; 1957 1958 ring = &cpr->ring_struct; 1959 ring->handle = BNGE_SET_NQ_HDL(cpr); 1960 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1961 if (rc) 1962 goto err_out; 1963 bnge_set_db(bn, &cpr->cp_db, type, map_idx, 1964 ring->fw_ring_id); 1965 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1966 } 1967 1968 if (agg_rings) { 1969 for (i = 0; i < bd->rx_nr_rings; i++) { 1970 rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]); 1971 if (rc) 1972 goto err_out; 1973 } 1974 } 1975 err_out: 1976 return rc; 1977 } 1978 1979 void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic) 1980 { 1981 __le16 *ring_tbl = vnic->rss_table; 1982 struct bnge_rx_ring_info *rxr; 1983 struct bnge_dev *bd = bn->bd; 1984 u16 tbl_size, i; 1985 1986 tbl_size = bnge_get_rxfh_indir_size(bd); 1987 1988 for (i = 0; i < tbl_size; i++) { 1989 u16 ring_id, j; 1990 1991 j = bd->rss_indir_tbl[i]; 1992 rxr = &bn->rx_ring[j]; 1993 1994 ring_id = rxr->rx_ring_struct.fw_ring_id; 1995 *ring_tbl++ = cpu_to_le16(ring_id); 1996 ring_id = bnge_cp_ring_for_rx(rxr); 1997 *ring_tbl++ = cpu_to_le16(ring_id); 1998 } 1999 } 2000 2001 static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn, 2002 struct bnge_vnic_info *vnic) 2003 { 2004 int rc; 2005 2006 rc = bnge_hwrm_vnic_set_rss(bn, vnic, true); 2007 if (rc) { 2008 netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n", 2009 vnic->vnic_id, rc); 2010 return rc; 2011 } 2012 rc = bnge_hwrm_vnic_cfg(bn, vnic); 2013 if (rc) 2014 netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n", 2015 vnic->vnic_id, rc); 2016 return rc; 2017 } 2018 2019 static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic) 2020 { 2021 struct bnge_dev *bd = bn->bd; 2022 int rc, i, nr_ctxs; 2023 2024 nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings); 2025 for (i = 0; i < nr_ctxs; i++) { 2026 rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i); 2027 if (rc) { 2028 netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n", 2029 vnic->vnic_id, i, rc); 2030 return -ENOMEM; 2031 } 2032 bn->rsscos_nr_ctxs++; 2033 } 2034 2035 rc = bnge_hwrm_vnic_rss_cfg(bn, vnic); 2036 if (rc) 2037 return rc; 2038 2039 if (bnge_is_agg_reqd(bd)) { 2040 rc = bnge_hwrm_vnic_set_hds(bn, vnic); 2041 if (rc) 2042 netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n", 2043 vnic->vnic_id, rc); 2044 } 2045 return rc; 2046 } 2047 2048 static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr) 2049 { 2050 if (!refcount_dec_and_test(&fltr->refcnt)) 2051 return; 2052 hlist_del_rcu(&fltr->base.hash); 2053 kfree_rcu(fltr, base.rcu); 2054 } 2055 2056 static void bnge_init_l2_filter(struct bnge_net *bn, 2057 struct bnge_l2_filter *fltr, 2058 struct bnge_l2_key *key, u32 idx) 2059 { 2060 struct hlist_head *head; 2061 2062 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 2063 fltr->l2_key.vlan = key->vlan; 2064 fltr->base.type = BNGE_FLTR_TYPE_L2; 2065 2066 head = &bn->l2_fltr_hash_tbl[idx]; 2067 hlist_add_head_rcu(&fltr->base.hash, head); 2068 refcount_set(&fltr->refcnt, 1); 2069 } 2070 2071 static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn, 2072 struct bnge_l2_key *key, 2073 u32 idx) 2074 { 2075 struct bnge_l2_filter *fltr; 2076 struct hlist_head *head; 2077 2078 head = &bn->l2_fltr_hash_tbl[idx]; 2079 hlist_for_each_entry_rcu(fltr, head, base.hash) { 2080 struct bnge_l2_key *l2_key = &fltr->l2_key; 2081 2082 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 2083 l2_key->vlan == key->vlan) 2084 return fltr; 2085 } 2086 return NULL; 2087 } 2088 2089 static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn, 2090 struct bnge_l2_key *key, 2091 u32 idx) 2092 { 2093 struct bnge_l2_filter *fltr; 2094 2095 rcu_read_lock(); 2096 fltr = __bnge_lookup_l2_filter(bn, key, idx); 2097 if (fltr) 2098 refcount_inc(&fltr->refcnt); 2099 rcu_read_unlock(); 2100 return fltr; 2101 } 2102 2103 static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn, 2104 struct bnge_l2_key *key, 2105 gfp_t gfp) 2106 { 2107 struct bnge_l2_filter *fltr; 2108 u32 idx; 2109 2110 idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) & 2111 BNGE_L2_FLTR_HASH_MASK; 2112 fltr = bnge_lookup_l2_filter(bn, key, idx); 2113 if (fltr) 2114 return fltr; 2115 2116 fltr = kzalloc_obj(*fltr, gfp); 2117 if (!fltr) 2118 return ERR_PTR(-ENOMEM); 2119 2120 bnge_init_l2_filter(bn, fltr, key, idx); 2121 return fltr; 2122 } 2123 2124 static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx, 2125 const u8 *mac_addr) 2126 { 2127 struct bnge_l2_filter *fltr; 2128 struct bnge_l2_key key; 2129 int rc; 2130 2131 ether_addr_copy(key.dst_mac_addr, mac_addr); 2132 key.vlan = 0; 2133 fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL); 2134 if (IS_ERR(fltr)) 2135 return PTR_ERR(fltr); 2136 2137 fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id; 2138 rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr); 2139 if (rc) 2140 goto err_del_l2_filter; 2141 bn->vnic_info[vnic_id].l2_filters[idx] = fltr; 2142 return rc; 2143 2144 err_del_l2_filter: 2145 bnge_del_l2_filter(bn, fltr); 2146 return rc; 2147 } 2148 2149 static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask) 2150 { 2151 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2152 struct net_device *dev = bn->netdev; 2153 struct netdev_hw_addr *ha; 2154 int mc_count = 0, off = 0; 2155 bool update = false; 2156 u8 *haddr; 2157 2158 netdev_for_each_mc_addr(ha, dev) { 2159 if (mc_count >= BNGE_MAX_MC_ADDRS) { 2160 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2161 vnic->mc_list_count = 0; 2162 return false; 2163 } 2164 haddr = ha->addr; 2165 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 2166 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 2167 update = true; 2168 } 2169 off += ETH_ALEN; 2170 mc_count++; 2171 } 2172 if (mc_count) 2173 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2174 2175 if (mc_count != vnic->mc_list_count) { 2176 vnic->mc_list_count = mc_count; 2177 update = true; 2178 } 2179 return update; 2180 } 2181 2182 static bool bnge_uc_list_updated(struct bnge_net *bn) 2183 { 2184 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2185 struct net_device *dev = bn->netdev; 2186 struct netdev_hw_addr *ha; 2187 int off = 0; 2188 2189 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 2190 return true; 2191 2192 netdev_for_each_uc_addr(ha, dev) { 2193 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 2194 return true; 2195 2196 off += ETH_ALEN; 2197 } 2198 return false; 2199 } 2200 2201 static bool bnge_promisc_ok(struct bnge_net *bn) 2202 { 2203 return true; 2204 } 2205 2206 static int bnge_cfg_def_vnic(struct bnge_net *bn) 2207 { 2208 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2209 struct net_device *dev = bn->netdev; 2210 struct bnge_dev *bd = bn->bd; 2211 struct netdev_hw_addr *ha; 2212 int i, off = 0, rc; 2213 bool uc_update; 2214 2215 netif_addr_lock_bh(dev); 2216 uc_update = bnge_uc_list_updated(bn); 2217 netif_addr_unlock_bh(dev); 2218 2219 if (!uc_update) 2220 goto skip_uc; 2221 2222 for (i = 1; i < vnic->uc_filter_count; i++) { 2223 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2224 2225 bnge_hwrm_l2_filter_free(bd, fltr); 2226 bnge_del_l2_filter(bn, fltr); 2227 } 2228 2229 vnic->uc_filter_count = 1; 2230 2231 netif_addr_lock_bh(dev); 2232 if (netdev_uc_count(dev) > (BNGE_MAX_UC_ADDRS - 1)) { 2233 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2234 } else { 2235 netdev_for_each_uc_addr(ha, dev) { 2236 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 2237 off += ETH_ALEN; 2238 vnic->uc_filter_count++; 2239 } 2240 } 2241 netif_addr_unlock_bh(dev); 2242 2243 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 2244 rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off); 2245 if (rc) { 2246 netdev_err(dev, "HWRM vnic filter failure rc: %d\n", rc); 2247 vnic->uc_filter_count = i; 2248 return rc; 2249 } 2250 } 2251 2252 skip_uc: 2253 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 2254 !bnge_promisc_ok(bn)) 2255 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2256 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2257 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 2258 netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 2259 rc); 2260 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2261 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2262 vnic->mc_list_count = 0; 2263 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2264 } 2265 if (rc) 2266 netdev_err(dev, "HWRM cfa l2 rx mask failure rc: %d\n", 2267 rc); 2268 2269 return rc; 2270 } 2271 2272 static void bnge_disable_int(struct bnge_net *bn) 2273 { 2274 struct bnge_dev *bd = bn->bd; 2275 int i; 2276 2277 if (!bn->bnapi) 2278 return; 2279 2280 for (i = 0; i < bd->nq_nr_rings; i++) { 2281 struct bnge_napi *bnapi = bn->bnapi[i]; 2282 struct bnge_nq_ring_info *nqr; 2283 struct bnge_ring_struct *ring; 2284 2285 nqr = &bnapi->nq_ring; 2286 ring = &nqr->ring_struct; 2287 2288 if (ring->fw_ring_id != INVALID_HW_RING_ID) 2289 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 2290 } 2291 } 2292 2293 static void bnge_disable_int_sync(struct bnge_net *bn) 2294 { 2295 struct bnge_dev *bd = bn->bd; 2296 int i; 2297 2298 bnge_disable_int(bn); 2299 for (i = 0; i < bd->nq_nr_rings; i++) { 2300 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2301 2302 synchronize_irq(bd->irq_tbl[map_idx].vector); 2303 } 2304 } 2305 2306 static void bnge_enable_int(struct bnge_net *bn) 2307 { 2308 struct bnge_dev *bd = bn->bd; 2309 int i; 2310 2311 for (i = 0; i < bd->nq_nr_rings; i++) { 2312 struct bnge_napi *bnapi = bn->bnapi[i]; 2313 struct bnge_nq_ring_info *nqr; 2314 2315 nqr = &bnapi->nq_ring; 2316 bnge_db_nq_arm(bn, &nqr->nq_db, nqr->nq_raw_cons); 2317 } 2318 } 2319 2320 static void bnge_disable_napi(struct bnge_net *bn) 2321 { 2322 struct bnge_dev *bd = bn->bd; 2323 int i; 2324 2325 if (test_and_set_bit(BNGE_STATE_NAPI_DISABLED, &bn->state)) 2326 return; 2327 2328 for (i = 0; i < bd->nq_nr_rings; i++) { 2329 struct bnge_napi *bnapi = bn->bnapi[i]; 2330 2331 napi_disable_locked(&bnapi->napi); 2332 } 2333 } 2334 2335 static void bnge_enable_napi(struct bnge_net *bn) 2336 { 2337 struct bnge_dev *bd = bn->bd; 2338 int i; 2339 2340 clear_bit(BNGE_STATE_NAPI_DISABLED, &bn->state); 2341 for (i = 0; i < bd->nq_nr_rings; i++) { 2342 struct bnge_napi *bnapi = bn->bnapi[i]; 2343 2344 bnapi->in_reset = false; 2345 bnapi->tx_fault = 0; 2346 2347 napi_enable_locked(&bnapi->napi); 2348 } 2349 } 2350 2351 static void bnge_hwrm_vnic_free(struct bnge_net *bn) 2352 { 2353 int i; 2354 2355 for (i = 0; i < bn->nr_vnics; i++) 2356 bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]); 2357 } 2358 2359 static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn) 2360 { 2361 int i, j; 2362 2363 for (i = 0; i < bn->nr_vnics; i++) { 2364 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 2365 2366 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) { 2367 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 2368 bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j); 2369 } 2370 } 2371 bn->rsscos_nr_ctxs = 0; 2372 } 2373 2374 static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn) 2375 { 2376 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2377 int i; 2378 2379 for (i = 0; i < vnic->uc_filter_count; i++) { 2380 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2381 2382 bnge_hwrm_l2_filter_free(bn->bd, fltr); 2383 bnge_del_l2_filter(bn, fltr); 2384 } 2385 2386 vnic->uc_filter_count = 0; 2387 } 2388 2389 static void bnge_clear_vnic(struct bnge_net *bn) 2390 { 2391 bnge_hwrm_clear_vnic_filter(bn); 2392 bnge_hwrm_vnic_free(bn); 2393 bnge_hwrm_vnic_ctx_free(bn); 2394 } 2395 2396 static void bnge_hwrm_rx_ring_free(struct bnge_net *bn, 2397 struct bnge_rx_ring_info *rxr, 2398 bool close_path) 2399 { 2400 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 2401 u32 grp_idx = rxr->bnapi->index; 2402 u32 cmpl_ring_id; 2403 2404 if (ring->fw_ring_id == INVALID_HW_RING_ID) 2405 return; 2406 2407 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2408 hwrm_ring_free_send_msg(bn, ring, 2409 RING_FREE_REQ_RING_TYPE_RX, 2410 close_path ? cmpl_ring_id : 2411 INVALID_HW_RING_ID); 2412 ring->fw_ring_id = INVALID_HW_RING_ID; 2413 bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 2414 } 2415 2416 static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn, 2417 struct bnge_rx_ring_info *rxr, 2418 bool close_path) 2419 { 2420 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 2421 u32 grp_idx = rxr->bnapi->index; 2422 u32 cmpl_ring_id; 2423 2424 if (ring->fw_ring_id == INVALID_HW_RING_ID) 2425 return; 2426 2427 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2428 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG, 2429 close_path ? cmpl_ring_id : 2430 INVALID_HW_RING_ID); 2431 ring->fw_ring_id = INVALID_HW_RING_ID; 2432 bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 2433 } 2434 2435 static void bnge_hwrm_tx_ring_free(struct bnge_net *bn, 2436 struct bnge_tx_ring_info *txr, 2437 bool close_path) 2438 { 2439 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 2440 u32 cmpl_ring_id; 2441 2442 if (ring->fw_ring_id == INVALID_HW_RING_ID) 2443 return; 2444 2445 cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) : 2446 INVALID_HW_RING_ID; 2447 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX, 2448 cmpl_ring_id); 2449 ring->fw_ring_id = INVALID_HW_RING_ID; 2450 } 2451 2452 static void bnge_hwrm_cp_ring_free(struct bnge_net *bn, 2453 struct bnge_cp_ring_info *cpr) 2454 { 2455 struct bnge_ring_struct *ring; 2456 2457 ring = &cpr->ring_struct; 2458 if (ring->fw_ring_id == INVALID_HW_RING_ID) 2459 return; 2460 2461 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 2462 INVALID_HW_RING_ID); 2463 ring->fw_ring_id = INVALID_HW_RING_ID; 2464 } 2465 2466 static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path) 2467 { 2468 struct bnge_dev *bd = bn->bd; 2469 int i; 2470 2471 if (!bn->bnapi) 2472 return; 2473 2474 for (i = 0; i < bd->tx_nr_rings; i++) 2475 bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path); 2476 2477 for (i = 0; i < bd->rx_nr_rings; i++) { 2478 bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path); 2479 bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path); 2480 } 2481 2482 /* The completion rings are about to be freed. After that the 2483 * IRQ doorbell will not work anymore. So we need to disable 2484 * IRQ here. 2485 */ 2486 bnge_disable_int_sync(bn); 2487 2488 for (i = 0; i < bd->nq_nr_rings; i++) { 2489 struct bnge_napi *bnapi = bn->bnapi[i]; 2490 struct bnge_nq_ring_info *nqr; 2491 struct bnge_ring_struct *ring; 2492 int j; 2493 2494 nqr = &bnapi->nq_ring; 2495 for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++) 2496 bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]); 2497 2498 ring = &nqr->ring_struct; 2499 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 2500 hwrm_ring_free_send_msg(bn, ring, 2501 RING_FREE_REQ_RING_TYPE_NQ, 2502 INVALID_HW_RING_ID); 2503 ring->fw_ring_id = INVALID_HW_RING_ID; 2504 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 2505 } 2506 } 2507 } 2508 2509 static void bnge_setup_msix(struct bnge_net *bn) 2510 { 2511 struct net_device *dev = bn->netdev; 2512 struct bnge_dev *bd = bn->bd; 2513 int len, i; 2514 2515 len = sizeof(bd->irq_tbl[0].name); 2516 for (i = 0; i < bd->nq_nr_rings; i++) { 2517 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2518 char *attr; 2519 2520 if (bd->flags & BNGE_EN_SHARED_CHNL) 2521 attr = "TxRx"; 2522 else if (i < bd->rx_nr_rings) 2523 attr = "rx"; 2524 else 2525 attr = "tx"; 2526 2527 snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 2528 attr, i); 2529 bd->irq_tbl[map_idx].handler = bnge_msix; 2530 } 2531 } 2532 2533 static int bnge_setup_interrupts(struct bnge_net *bn) 2534 { 2535 struct net_device *dev = bn->netdev; 2536 struct bnge_dev *bd = bn->bd; 2537 2538 bnge_setup_msix(bn); 2539 2540 return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings); 2541 } 2542 2543 static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path) 2544 { 2545 bnge_clear_vnic(bn); 2546 bnge_hwrm_ring_free(bn, close_path); 2547 bnge_hwrm_stat_ctx_free(bn); 2548 } 2549 2550 static void bnge_free_irq(struct bnge_net *bn) 2551 { 2552 struct bnge_dev *bd = bn->bd; 2553 struct bnge_irq *irq; 2554 int i; 2555 2556 for (i = 0; i < bd->nq_nr_rings; i++) { 2557 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2558 2559 irq = &bd->irq_tbl[map_idx]; 2560 if (irq->requested) { 2561 if (irq->have_cpumask) { 2562 irq_set_affinity_hint(irq->vector, NULL); 2563 free_cpumask_var(irq->cpu_mask); 2564 irq->have_cpumask = 0; 2565 } 2566 free_irq(irq->vector, bn->bnapi[i]); 2567 } 2568 2569 irq->requested = 0; 2570 } 2571 } 2572 2573 static int bnge_request_irq(struct bnge_net *bn) 2574 { 2575 struct bnge_dev *bd = bn->bd; 2576 int i, rc; 2577 2578 rc = bnge_setup_interrupts(bn); 2579 if (rc) { 2580 netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc); 2581 return rc; 2582 } 2583 for (i = 0; i < bd->nq_nr_rings; i++) { 2584 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2585 struct bnge_irq *irq = &bd->irq_tbl[map_idx]; 2586 2587 rc = request_irq(irq->vector, irq->handler, 0, irq->name, 2588 bn->bnapi[i]); 2589 if (rc) 2590 goto err_free_irq; 2591 2592 netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector); 2593 irq->requested = 1; 2594 2595 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 2596 int numa_node = dev_to_node(&bd->pdev->dev); 2597 2598 irq->have_cpumask = 1; 2599 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2600 irq->cpu_mask); 2601 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 2602 if (rc) { 2603 netdev_warn(bn->netdev, 2604 "Set affinity failed, IRQ = %d\n", 2605 irq->vector); 2606 goto err_free_irq; 2607 } 2608 } 2609 } 2610 return 0; 2611 2612 err_free_irq: 2613 bnge_free_irq(bn); 2614 return rc; 2615 } 2616 2617 static int bnge_set_tpa(struct bnge_net *bn, bool set_tpa) 2618 { 2619 u32 tpa_flags = 0; 2620 int rc, i; 2621 2622 if (set_tpa) 2623 tpa_flags = bn->priv_flags & BNGE_NET_EN_TPA; 2624 else if (BNGE_NO_FW_ACCESS(bn->bd)) 2625 return 0; 2626 for (i = 0; i < bn->nr_vnics; i++) { 2627 rc = bnge_hwrm_vnic_set_tpa(bn->bd, &bn->vnic_info[i], 2628 tpa_flags); 2629 if (rc) { 2630 netdev_err(bn->netdev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 2631 i, rc); 2632 return rc; 2633 } 2634 } 2635 return 0; 2636 } 2637 2638 static int bnge_init_chip(struct bnge_net *bn) 2639 { 2640 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2641 struct bnge_dev *bd = bn->bd; 2642 int rc; 2643 2644 #define BNGE_DEF_STATS_COAL_TICKS 1000000 2645 bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS; 2646 2647 rc = bnge_hwrm_stat_ctx_alloc(bn); 2648 if (rc) { 2649 netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc); 2650 goto err_out; 2651 } 2652 2653 rc = bnge_hwrm_ring_alloc(bn); 2654 if (rc) { 2655 netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc); 2656 goto err_out; 2657 } 2658 2659 rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings); 2660 if (rc) { 2661 netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc); 2662 goto err_out; 2663 } 2664 2665 rc = bnge_setup_vnic(bn, vnic); 2666 if (rc) 2667 goto err_out; 2668 2669 if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA) 2670 bnge_hwrm_update_rss_hash_cfg(bn); 2671 2672 if (bn->priv_flags & BNGE_NET_EN_TPA) { 2673 rc = bnge_set_tpa(bn, true); 2674 if (rc) 2675 goto err_out; 2676 } 2677 2678 /* Filter for default vnic 0 */ 2679 rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr); 2680 if (rc) { 2681 netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc); 2682 goto err_out; 2683 } 2684 vnic->uc_filter_count = 1; 2685 2686 vnic->rx_mask = 0; 2687 2688 if (bn->netdev->flags & IFF_BROADCAST) 2689 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2690 2691 if (bn->netdev->flags & IFF_PROMISC) 2692 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2693 2694 if (bn->netdev->flags & IFF_ALLMULTI) { 2695 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2696 vnic->mc_list_count = 0; 2697 } else if (bn->netdev->flags & IFF_MULTICAST) { 2698 u32 mask = 0; 2699 2700 bnge_mc_list_updated(bn, &mask); 2701 vnic->rx_mask |= mask; 2702 } 2703 2704 rc = bnge_cfg_def_vnic(bn); 2705 if (rc) 2706 goto err_out; 2707 return 0; 2708 2709 err_out: 2710 bnge_hwrm_resource_free(bn, 0); 2711 return rc; 2712 } 2713 2714 static void bnge_init_napi(struct bnge_net *bn) 2715 { 2716 struct bnge_dev *bd = bn->bd; 2717 struct bnge_napi *bnapi; 2718 int i; 2719 2720 for (i = 0; i < bd->nq_nr_rings; i++) { 2721 bnapi = bn->bnapi[i]; 2722 netif_napi_add_config_locked(bn->netdev, &bnapi->napi, 2723 bnge_napi_poll, bnapi->index); 2724 } 2725 } 2726 2727 static void bnge_del_napi(struct bnge_net *bn) 2728 { 2729 struct bnge_dev *bd = bn->bd; 2730 int i; 2731 2732 for (i = 0; i < bd->rx_nr_rings; i++) 2733 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL); 2734 for (i = 0; i < bd->tx_nr_rings; i++) 2735 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL); 2736 2737 for (i = 0; i < bd->nq_nr_rings; i++) { 2738 struct bnge_napi *bnapi = bn->bnapi[i]; 2739 2740 __netif_napi_del_locked(&bnapi->napi); 2741 } 2742 2743 /* Wait for RCU grace period after removing NAPI instances */ 2744 synchronize_net(); 2745 } 2746 2747 static int bnge_init_nic(struct bnge_net *bn) 2748 { 2749 int rc; 2750 2751 bnge_init_nq_tree(bn); 2752 2753 bnge_init_rx_rings(bn); 2754 rc = bnge_alloc_rx_ring_pair_bufs(bn); 2755 if (rc) 2756 return rc; 2757 2758 bnge_init_tx_rings(bn); 2759 2760 rc = bnge_init_ring_grps(bn); 2761 if (rc) 2762 goto err_free_rx_ring_pair_bufs; 2763 2764 bnge_init_vnics(bn); 2765 2766 rc = bnge_init_chip(bn); 2767 if (rc) 2768 goto err_free_ring_grps; 2769 return rc; 2770 2771 err_free_ring_grps: 2772 bnge_free_ring_grps(bn); 2773 return rc; 2774 2775 err_free_rx_ring_pair_bufs: 2776 bnge_free_rx_ring_pair_bufs(bn); 2777 return rc; 2778 } 2779 2780 static void bnge_tx_disable(struct bnge_net *bn) 2781 { 2782 struct bnge_tx_ring_info *txr; 2783 int i; 2784 2785 if (bn->tx_ring) { 2786 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2787 txr = &bn->tx_ring[i]; 2788 WRITE_ONCE(txr->dev_state, BNGE_DEV_STATE_CLOSING); 2789 } 2790 } 2791 /* Make sure napi polls see @dev_state change */ 2792 synchronize_net(); 2793 2794 if (!bn->netdev) 2795 return; 2796 /* Drop carrier first to prevent TX timeout */ 2797 netif_carrier_off(bn->netdev); 2798 /* Stop all TX queues */ 2799 netif_tx_disable(bn->netdev); 2800 } 2801 2802 static void bnge_tx_enable(struct bnge_net *bn) 2803 { 2804 struct bnge_tx_ring_info *txr; 2805 int i; 2806 2807 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2808 txr = &bn->tx_ring[i]; 2809 WRITE_ONCE(txr->dev_state, 0); 2810 } 2811 /* Make sure napi polls see @dev_state change */ 2812 synchronize_net(); 2813 netif_tx_wake_all_queues(bn->netdev); 2814 if (BNGE_LINK_IS_UP(bn->bd)) 2815 netif_carrier_on(bn->netdev); 2816 } 2817 2818 static int bnge_open_core(struct bnge_net *bn) 2819 { 2820 struct bnge_dev *bd = bn->bd; 2821 int rc; 2822 2823 netif_carrier_off(bn->netdev); 2824 2825 rc = bnge_reserve_rings(bd); 2826 if (rc) { 2827 netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc); 2828 return rc; 2829 } 2830 2831 rc = bnge_alloc_core(bn); 2832 if (rc) { 2833 netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc); 2834 return rc; 2835 } 2836 2837 bnge_init_napi(bn); 2838 rc = bnge_request_irq(bn); 2839 if (rc) { 2840 netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc); 2841 goto err_del_napi; 2842 } 2843 2844 rc = bnge_init_nic(bn); 2845 if (rc) { 2846 netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc); 2847 goto err_free_irq; 2848 } 2849 2850 bnge_enable_napi(bn); 2851 2852 rc = bnge_update_phy_setting(bn); 2853 if (rc) { 2854 netdev_warn(bn->netdev, "failed to update PHY settings (rc: %d)\n", 2855 rc); 2856 WRITE_ONCE(bd->link_info.phy_retry_expires, jiffies + 5 * HZ); 2857 WRITE_ONCE(bd->link_info.phy_retry, true); 2858 } else { 2859 WRITE_ONCE(bd->link_info.phy_retry, false); 2860 } 2861 2862 set_bit(BNGE_STATE_OPEN, &bd->state); 2863 set_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 2864 2865 bnge_enable_int(bn); 2866 2867 bnge_tx_enable(bn); 2868 2869 mod_timer(&bn->timer, jiffies + bn->current_interval); 2870 2871 /* Poll link status and check for SFP+ module status */ 2872 bnge_get_port_module_status(bn); 2873 2874 return 0; 2875 2876 err_free_irq: 2877 bnge_free_irq(bn); 2878 err_del_napi: 2879 bnge_del_napi(bn); 2880 bnge_free_core(bn); 2881 return rc; 2882 } 2883 2884 static int bnge_open(struct net_device *dev) 2885 { 2886 struct bnge_net *bn = netdev_priv(dev); 2887 int rc; 2888 2889 rc = bnge_open_core(bn); 2890 if (rc) 2891 netdev_err(dev, "bnge_open_core err: %d\n", rc); 2892 2893 return rc; 2894 } 2895 2896 static int bnge_shutdown_nic(struct bnge_net *bn) 2897 { 2898 bnge_hwrm_resource_free(bn, 1); 2899 return 0; 2900 } 2901 2902 static void bnge_add_prev_ring_stats64(struct bnge_net *bn, 2903 struct rtnl_link_stats64 *stats) 2904 { 2905 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 2906 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 2907 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 2908 2909 stats->rx_packets += rx_save->packets; 2910 stats->tx_packets += tx_save->packets; 2911 stats->rx_bytes += rx_save->bytes; 2912 stats->tx_bytes += tx_save->bytes; 2913 stats->rx_missed_errors += rx_save->hw_drop_overruns; 2914 stats->tx_dropped += tx_save->hw_drop_errors; 2915 2916 stats->multicast += stats64_save->multicast; 2917 } 2918 2919 static void bnge_get_ring_stats64(struct bnge_dev *bd, 2920 struct rtnl_link_stats64 *stats) 2921 { 2922 struct bnge_net *bn = netdev_priv(bd->netdev); 2923 int i; 2924 2925 for (i = 0; i < bd->nq_nr_rings; i++) { 2926 struct bnge_napi *bnapi = bn->bnapi[i]; 2927 u64 tx_bytes, tx_packets, tx_dropped; 2928 u64 multicast, rx_missed_errors; 2929 struct bnge_nq_ring_info *nqr; 2930 u64 rx_bytes, rx_packets; 2931 unsigned int start; 2932 u64 *sw; 2933 2934 nqr = &bnapi->nq_ring; 2935 sw = nqr->stats.sw_stats; 2936 2937 do { 2938 start = u64_stats_fetch_begin(&nqr->stats.syncp); 2939 2940 rx_packets = BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 2941 rx_packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 2942 rx_packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 2943 2944 tx_packets = BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 2945 tx_packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 2946 tx_packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 2947 2948 rx_bytes = BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 2949 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 2950 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 2951 2952 tx_bytes = BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 2953 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 2954 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 2955 2956 multicast = BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 2957 rx_missed_errors = 2958 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 2959 tx_dropped = 2960 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 2961 } while (u64_stats_fetch_retry(&nqr->stats.syncp, start)); 2962 2963 stats->rx_packets += rx_packets; 2964 stats->tx_packets += tx_packets; 2965 stats->rx_bytes += rx_bytes; 2966 stats->tx_bytes += tx_bytes; 2967 stats->multicast += multicast; 2968 stats->rx_missed_errors += rx_missed_errors; 2969 stats->tx_dropped += tx_dropped; 2970 } 2971 } 2972 2973 static void bnge_get_port_stats64(struct bnge_net *bn, 2974 struct rtnl_link_stats64 *stats) 2975 { 2976 unsigned int start; 2977 u64 *tx, *rx; 2978 2979 rx = bn->port_stats.sw_stats; 2980 tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 2981 2982 do { 2983 start = u64_stats_fetch_begin(&bn->port_stats.syncp); 2984 2985 stats->rx_crc_errors = 2986 BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 2987 stats->rx_frame_errors = 2988 BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 2989 stats->rx_length_errors = 2990 BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 2991 BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 2992 BNGE_GET_RX_PORT_STATS64(rx, rx_runt_frames); 2993 stats->rx_errors = 2994 BNGE_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 2995 BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 2996 stats->collisions = 2997 BNGE_GET_TX_PORT_STATS64(tx, tx_total_collisions); 2998 stats->tx_fifo_errors = 2999 BNGE_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 3000 stats->tx_errors = BNGE_GET_TX_PORT_STATS64(tx, tx_err); 3001 } while (u64_stats_fetch_retry(&bn->port_stats.syncp, start)); 3002 } 3003 3004 static void bnge_fill_prev_stats64(struct bnge_net *bn, 3005 struct rtnl_link_stats64 *stats) 3006 { 3007 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3008 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3009 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3010 3011 stats->rx_packets = rx_save->packets; 3012 stats->tx_packets = tx_save->packets; 3013 stats->rx_bytes = rx_save->bytes; 3014 stats->tx_bytes = tx_save->bytes; 3015 stats->rx_missed_errors = rx_save->hw_drop_overruns; 3016 stats->tx_dropped = tx_save->hw_drop_errors; 3017 stats->multicast = stats64_save->multicast; 3018 } 3019 3020 static void bnge_get_stats64(struct net_device *dev, 3021 struct rtnl_link_stats64 *stats) 3022 { 3023 struct bnge_net *bn = netdev_priv(dev); 3024 3025 if (bn->flags & BNGE_FLAG_PORT_STATS) 3026 bnge_get_port_stats64(bn, stats); 3027 3028 spin_lock_bh(&bn->stats_lock); 3029 if (!test_bit(BNGE_STATE_STATS_ENABLE, &bn->state)) { 3030 bnge_fill_prev_stats64(bn, stats); 3031 spin_unlock_bh(&bn->stats_lock); 3032 return; 3033 } 3034 3035 bnge_get_ring_stats64(bn->bd, stats); 3036 bnge_add_prev_ring_stats64(bn, stats); 3037 spin_unlock_bh(&bn->stats_lock); 3038 } 3039 3040 static void bnge_save_ring_stats(struct bnge_net *bn) 3041 { 3042 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3043 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3044 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3045 int i; 3046 3047 for (i = 0; i < bn->bd->nq_nr_rings; i++) { 3048 struct bnge_napi *bnapi = bn->bnapi[i]; 3049 struct bnge_nq_ring_info *nqr; 3050 u64 *sw; 3051 3052 nqr = &bnapi->nq_ring; 3053 sw = nqr->stats.sw_stats; 3054 3055 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3056 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3057 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3058 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3059 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3060 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3061 rx_save->hw_drop_overruns += BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3062 rx_save->hw_drops += BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3063 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3064 3065 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3066 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3067 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3068 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3069 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3070 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3071 tx_save->hw_drop_errors += BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3072 tx_save->hw_drops += BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3073 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3074 3075 stats64_save->multicast += 3076 BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3077 } 3078 } 3079 3080 static void bnge_close_core(struct bnge_net *bn) 3081 { 3082 struct bnge_dev *bd = bn->bd; 3083 3084 bnge_tx_disable(bn); 3085 3086 clear_bit(BNGE_STATE_OPEN, &bd->state); 3087 3088 timer_delete_sync(&bn->timer); 3089 bnge_shutdown_nic(bn); 3090 bnge_disable_napi(bn); 3091 3092 /* Save ring stats before shutdown */ 3093 spin_lock_bh(&bn->stats_lock); 3094 bnge_save_ring_stats(bn); 3095 clear_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 3096 spin_unlock_bh(&bn->stats_lock); 3097 3098 bnge_free_all_rings_bufs(bn); 3099 bnge_free_irq(bn); 3100 bnge_del_napi(bn); 3101 3102 bnge_free_core(bn); 3103 } 3104 3105 static int bnge_close(struct net_device *dev) 3106 { 3107 struct bnge_net *bn = netdev_priv(dev); 3108 3109 bnge_close_core(bn); 3110 bnge_hwrm_shutdown_link(bn->bd); 3111 bn->sp_event = 0; 3112 3113 return 0; 3114 } 3115 3116 static void bnge_get_queue_stats_rx(struct net_device *dev, int i, 3117 struct netdev_queue_stats_rx *stats) 3118 { 3119 struct bnge_net *bn = netdev_priv(dev); 3120 struct bnge_nq_ring_info *nqr; 3121 u64 *sw; 3122 3123 if (!bn->bnapi) 3124 return; 3125 3126 nqr = &bn->bnapi[i]->nq_ring; 3127 sw = nqr->stats.sw_stats; 3128 3129 stats->packets = 0; 3130 stats->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3131 stats->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3132 stats->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3133 3134 stats->bytes = 0; 3135 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3136 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3137 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3138 3139 stats->hw_drop_overruns = BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3140 stats->hw_drops = BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3141 stats->hw_drop_overruns; 3142 } 3143 3144 static void bnge_get_queue_stats_tx(struct net_device *dev, int i, 3145 struct netdev_queue_stats_tx *stats) 3146 { 3147 struct bnge_net *bn = netdev_priv(dev); 3148 struct bnge_napi *bnapi; 3149 u64 *sw; 3150 3151 if (!bn->tx_ring) 3152 return; 3153 3154 bnapi = bn->tx_ring[bn->tx_ring_map[i]].bnapi; 3155 sw = bnapi->nq_ring.stats.sw_stats; 3156 3157 stats->packets = 0; 3158 stats->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3159 stats->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3160 stats->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3161 3162 stats->bytes = 0; 3163 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3164 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3165 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3166 3167 stats->hw_drop_errors = BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3168 stats->hw_drops = BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3169 stats->hw_drop_errors; 3170 } 3171 3172 static void bnge_get_base_stats(struct net_device *dev, 3173 struct netdev_queue_stats_rx *rx, 3174 struct netdev_queue_stats_tx *tx) 3175 { 3176 struct bnge_net *bn = netdev_priv(dev); 3177 3178 rx->packets = bn->rxq_prv_stats.packets; 3179 rx->bytes = bn->rxq_prv_stats.bytes; 3180 rx->hw_drops = bn->rxq_prv_stats.hw_drops; 3181 rx->hw_drop_overruns = bn->rxq_prv_stats.hw_drop_overruns; 3182 3183 tx->packets = bn->txq_prv_stats.packets; 3184 tx->bytes = bn->txq_prv_stats.bytes; 3185 tx->hw_drops = bn->txq_prv_stats.hw_drops; 3186 tx->hw_drop_errors = bn->txq_prv_stats.hw_drop_errors; 3187 } 3188 3189 static const struct netdev_stat_ops bnge_stat_ops = { 3190 .get_queue_stats_rx = bnge_get_queue_stats_rx, 3191 .get_queue_stats_tx = bnge_get_queue_stats_tx, 3192 .get_base_stats = bnge_get_base_stats, 3193 }; 3194 3195 static const struct net_device_ops bnge_netdev_ops = { 3196 .ndo_open = bnge_open, 3197 .ndo_stop = bnge_close, 3198 .ndo_start_xmit = bnge_start_xmit, 3199 .ndo_get_stats64 = bnge_get_stats64, 3200 .ndo_features_check = bnge_features_check, 3201 }; 3202 3203 static void bnge_init_mac_addr(struct bnge_dev *bd) 3204 { 3205 eth_hw_addr_set(bd->netdev, bd->pf.mac_addr); 3206 } 3207 3208 static void bnge_set_tpa_flags(struct bnge_dev *bd) 3209 { 3210 struct bnge_net *bn = netdev_priv(bd->netdev); 3211 3212 bn->priv_flags &= ~BNGE_NET_EN_TPA; 3213 3214 if (bd->netdev->features & NETIF_F_LRO) 3215 bn->priv_flags |= BNGE_NET_EN_LRO; 3216 else if (bd->netdev->features & NETIF_F_GRO_HW) 3217 bn->priv_flags |= BNGE_NET_EN_GRO; 3218 } 3219 3220 static void bnge_init_l2_fltr_tbl(struct bnge_net *bn) 3221 { 3222 int i; 3223 3224 for (i = 0; i < BNGE_L2_FLTR_HASH_SIZE; i++) 3225 INIT_HLIST_HEAD(&bn->l2_fltr_hash_tbl[i]); 3226 get_random_bytes(&bn->hash_seed, sizeof(bn->hash_seed)); 3227 } 3228 3229 void bnge_set_ring_params(struct bnge_dev *bd) 3230 { 3231 struct bnge_net *bn = netdev_priv(bd->netdev); 3232 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3233 u32 agg_factor = 0, agg_ring_size = 0; 3234 3235 /* 8 for CRC and VLAN */ 3236 rx_size = SKB_DATA_ALIGN(bn->netdev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3237 3238 rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) + 3239 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3240 3241 ring_size = bn->rx_ring_size; 3242 bn->rx_agg_ring_size = 0; 3243 bn->rx_agg_nr_pages = 0; 3244 3245 if (bn->priv_flags & BNGE_NET_EN_TPA) 3246 agg_factor = min_t(u32, 4, 65536 / BNGE_RX_PAGE_SIZE); 3247 3248 bn->priv_flags &= ~BNGE_NET_EN_JUMBO; 3249 if (rx_space > PAGE_SIZE) { 3250 u32 jumbo_factor; 3251 3252 bn->priv_flags |= BNGE_NET_EN_JUMBO; 3253 jumbo_factor = PAGE_ALIGN(bn->netdev->mtu - 40) >> PAGE_SHIFT; 3254 if (jumbo_factor > agg_factor) 3255 agg_factor = jumbo_factor; 3256 } 3257 if (agg_factor) { 3258 if (ring_size > BNGE_MAX_RX_DESC_CNT_JUM_ENA) { 3259 ring_size = BNGE_MAX_RX_DESC_CNT_JUM_ENA; 3260 netdev_warn(bn->netdev, "RX ring size reduced from %d to %d due to jumbo ring\n", 3261 bn->rx_ring_size, ring_size); 3262 bn->rx_ring_size = ring_size; 3263 } 3264 agg_ring_size = ring_size * agg_factor; 3265 3266 bn->rx_agg_nr_pages = bnge_adjust_pow_two(agg_ring_size, 3267 RX_DESC_CNT); 3268 if (bn->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3269 u32 tmp = agg_ring_size; 3270 3271 bn->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3272 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3273 netdev_warn(bn->netdev, "RX agg ring size %d reduced to %d.\n", 3274 tmp, agg_ring_size); 3275 } 3276 bn->rx_agg_ring_size = agg_ring_size; 3277 bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3278 3279 rx_size = max3(BNGE_DEFAULT_RX_COPYBREAK, 3280 bn->rx_copybreak, 3281 bn->netdev->cfg_pending->hds_thresh); 3282 rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); 3283 rx_space = rx_size + NET_SKB_PAD + 3284 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3285 } 3286 3287 bn->rx_buf_use_size = rx_size; 3288 bn->rx_buf_size = rx_space; 3289 3290 bn->rx_nr_pages = bnge_adjust_pow_two(ring_size, RX_DESC_CNT); 3291 bn->rx_ring_mask = (bn->rx_nr_pages * RX_DESC_CNT) - 1; 3292 3293 ring_size = bn->tx_ring_size; 3294 bn->tx_nr_pages = bnge_adjust_pow_two(ring_size, TX_DESC_CNT); 3295 bn->tx_ring_mask = (bn->tx_nr_pages * TX_DESC_CNT) - 1; 3296 3297 max_rx_cmpl = bn->rx_ring_size; 3298 3299 if (bn->priv_flags & BNGE_NET_EN_TPA) 3300 max_rx_cmpl += bd->max_tpa_v2; 3301 ring_size = max_rx_cmpl * 2 + agg_ring_size + bn->tx_ring_size; 3302 bn->cp_ring_size = ring_size; 3303 3304 bn->cp_nr_pages = bnge_adjust_pow_two(ring_size, CP_DESC_CNT); 3305 if (bn->cp_nr_pages > MAX_CP_PAGES) { 3306 bn->cp_nr_pages = MAX_CP_PAGES; 3307 bn->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3308 netdev_warn(bn->netdev, "completion ring size %d reduced to %d.\n", 3309 ring_size, bn->cp_ring_size); 3310 } 3311 bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT; 3312 bn->cp_ring_mask = bn->cp_bit - 1; 3313 } 3314 3315 static void bnge_init_ring_params(struct bnge_net *bn) 3316 { 3317 u32 rx_size; 3318 3319 bn->rx_copybreak = BNGE_DEFAULT_RX_COPYBREAK; 3320 /* Try to fit 4 chunks into a 4k page */ 3321 rx_size = SZ_1K - 3322 NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3323 bn->netdev->cfg->hds_thresh = max(BNGE_DEFAULT_RX_COPYBREAK, rx_size); 3324 } 3325 3326 int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) 3327 { 3328 struct net_device *netdev; 3329 struct bnge_net *bn; 3330 int rc; 3331 3332 netdev = alloc_etherdev_mqs(sizeof(*bn), max_irqs * BNGE_MAX_QUEUE, 3333 max_irqs); 3334 if (!netdev) 3335 return -ENOMEM; 3336 3337 SET_NETDEV_DEV(netdev, bd->dev); 3338 bd->netdev = netdev; 3339 3340 netdev->netdev_ops = &bnge_netdev_ops; 3341 netdev->stat_ops = &bnge_stat_ops; 3342 3343 bnge_set_ethtool_ops(netdev); 3344 3345 bn = netdev_priv(netdev); 3346 bn->netdev = netdev; 3347 bn->bd = bd; 3348 3349 netdev->min_mtu = ETH_ZLEN; 3350 netdev->max_mtu = bd->max_mtu; 3351 3352 netdev->hw_features = NETIF_F_IP_CSUM | 3353 NETIF_F_IPV6_CSUM | 3354 NETIF_F_SG | 3355 NETIF_F_TSO | 3356 NETIF_F_TSO6 | 3357 NETIF_F_GSO_UDP_TUNNEL | 3358 NETIF_F_GSO_GRE | 3359 NETIF_F_GSO_IPXIP4 | 3360 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3361 NETIF_F_GSO_GRE_CSUM | 3362 NETIF_F_GSO_PARTIAL | 3363 NETIF_F_RXHASH | 3364 NETIF_F_RXCSUM | 3365 NETIF_F_GRO; 3366 3367 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3368 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 3369 3370 if (BNGE_SUPPORTS_TPA(bd)) 3371 netdev->hw_features |= NETIF_F_LRO; 3372 3373 netdev->hw_enc_features = NETIF_F_IP_CSUM | 3374 NETIF_F_IPV6_CSUM | 3375 NETIF_F_SG | 3376 NETIF_F_TSO | 3377 NETIF_F_TSO6 | 3378 NETIF_F_GSO_UDP_TUNNEL | 3379 NETIF_F_GSO_GRE | 3380 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3381 NETIF_F_GSO_GRE_CSUM | 3382 NETIF_F_GSO_IPXIP4 | 3383 NETIF_F_GSO_PARTIAL; 3384 3385 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3386 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 3387 3388 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 3389 NETIF_F_GSO_GRE_CSUM; 3390 3391 netdev->vlan_features = netdev->hw_features | NETIF_F_HIGHDMA; 3392 if (bd->fw_cap & BNGE_FW_CAP_VLAN_RX_STRIP) 3393 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_RX; 3394 if (bd->fw_cap & BNGE_FW_CAP_VLAN_TX_INSERT) 3395 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_TX; 3396 3397 if (BNGE_SUPPORTS_TPA(bd)) 3398 netdev->hw_features |= NETIF_F_GRO_HW; 3399 3400 netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA; 3401 3402 if (netdev->features & NETIF_F_GRO_HW) 3403 netdev->features &= ~NETIF_F_LRO; 3404 3405 netdev->priv_flags |= IFF_UNICAST_FLT; 3406 3407 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 3408 if (bd->tso_max_segs) 3409 netif_set_tso_max_segs(netdev, bd->tso_max_segs); 3410 3411 INIT_WORK(&bn->sp_task, bnge_sp_task); 3412 timer_setup(&bn->timer, bnge_timer, 0); 3413 bn->current_interval = BNGE_TIMER_INTERVAL; 3414 3415 bn->bnge_pf_wq = alloc_ordered_workqueue("bnge_pf_wq-%s", 0, 3416 dev_name(bd->dev)); 3417 if (!bn->bnge_pf_wq) { 3418 netdev_err(netdev, "Unable to create workqueue.\n"); 3419 rc = -ENOMEM; 3420 goto err_netdev; 3421 } 3422 3423 bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE; 3424 bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE; 3425 bn->rx_dir = DMA_FROM_DEVICE; 3426 3427 bnge_set_tpa_flags(bd); 3428 bnge_init_ring_params(bn); 3429 bnge_set_ring_params(bd); 3430 3431 bnge_init_l2_fltr_tbl(bn); 3432 bnge_init_mac_addr(bd); 3433 3434 rc = bnge_probe_phy(bn, true); 3435 if (rc) 3436 goto err_free_workq; 3437 3438 rc = bnge_alloc_port_stats(bn); 3439 if (rc) 3440 goto err_free_workq; 3441 spin_lock_init(&bn->stats_lock); 3442 3443 netdev->request_ops_lock = true; 3444 rc = register_netdev(netdev); 3445 if (rc) { 3446 dev_err(bd->dev, "Register netdev failed rc: %d\n", rc); 3447 goto err_free_port_stats; 3448 } 3449 3450 return 0; 3451 3452 err_free_port_stats: 3453 bnge_free_port_stats(bn); 3454 err_free_workq: 3455 destroy_workqueue(bn->bnge_pf_wq); 3456 err_netdev: 3457 free_netdev(netdev); 3458 return rc; 3459 } 3460 3461 void bnge_netdev_free(struct bnge_dev *bd) 3462 { 3463 struct net_device *netdev = bd->netdev; 3464 struct bnge_net *bn; 3465 3466 bn = netdev_priv(netdev); 3467 3468 unregister_netdev(netdev); 3469 3470 timer_shutdown_sync(&bn->timer); 3471 cancel_work_sync(&bn->sp_task); 3472 bn->sp_event = 0; 3473 destroy_workqueue(bn->bnge_pf_wq); 3474 3475 bnge_free_port_stats(bn); 3476 3477 free_netdev(netdev); 3478 bd->netdev = NULL; 3479 } 3480