1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <asm/byteorder.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/dmapool.h> 7 #include <linux/delay.h> 8 #include <linux/errno.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/netdevice.h> 13 #include <net/netdev_lock.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if.h> 18 #include <net/ip.h> 19 #include <linux/skbuff.h> 20 #include <net/page_pool/helpers.h> 21 22 #include "bnge.h" 23 #include "bnge_hwrm_lib.h" 24 #include "bnge_ethtool.h" 25 #include "bnge_rmem.h" 26 #include "bnge_txrx.h" 27 28 #define BNGE_RING_TO_TC_OFF(bd, tx) \ 29 ((tx) % (bd)->tx_nr_rings_per_tc) 30 31 #define BNGE_RING_TO_TC(bd, tx) \ 32 ((tx) / (bd)->tx_nr_rings_per_tc) 33 34 #define BNGE_TC_TO_RING_BASE(bd, tc) \ 35 ((tc) * (bd)->tx_nr_rings_per_tc) 36 37 static void bnge_free_stats_mem(struct bnge_net *bn, 38 struct bnge_stats_mem *stats) 39 { 40 struct bnge_dev *bd = bn->bd; 41 42 kfree(stats->hw_masks); 43 stats->hw_masks = NULL; 44 kfree(stats->sw_stats); 45 stats->sw_stats = NULL; 46 if (stats->hw_stats) { 47 dma_free_coherent(bd->dev, stats->len, stats->hw_stats, 48 stats->hw_stats_map); 49 stats->hw_stats = NULL; 50 } 51 } 52 53 static int bnge_alloc_stats_mem(struct bnge_net *bn, 54 struct bnge_stats_mem *stats, bool alloc_masks) 55 { 56 struct bnge_dev *bd = bn->bd; 57 58 stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len, 59 &stats->hw_stats_map, GFP_KERNEL); 60 if (!stats->hw_stats) 61 return -ENOMEM; 62 63 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 64 if (!stats->sw_stats) 65 goto stats_mem_err; 66 67 if (alloc_masks) { 68 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 69 if (!stats->hw_masks) 70 goto stats_mem_err; 71 } 72 73 u64_stats_init(&stats->syncp); 74 75 return 0; 76 77 stats_mem_err: 78 bnge_free_stats_mem(bn, stats); 79 return -ENOMEM; 80 } 81 82 static void bnge_free_ring_stats(struct bnge_net *bn) 83 { 84 struct bnge_dev *bd = bn->bd; 85 int i; 86 87 if (!bn->bnapi) 88 return; 89 90 for (i = 0; i < bd->nq_nr_rings; i++) { 91 struct bnge_napi *bnapi = bn->bnapi[i]; 92 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 93 94 bnge_free_stats_mem(bn, &nqr->stats); 95 } 96 } 97 98 static void bnge_fill_masks(u64 *mask_arr, u64 mask, int count) 99 { 100 int i; 101 102 for (i = 0; i < count; i++) 103 mask_arr[i] = mask; 104 } 105 106 void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 107 { 108 int i; 109 110 for (i = 0; i < count; i++) 111 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 112 } 113 114 static void bnge_init_stats(struct bnge_net *bn) 115 { 116 struct bnge_napi *bnapi = bn->bnapi[0]; 117 struct bnge_nq_ring_info *nqr; 118 struct bnge_stats_mem *stats; 119 struct bnge_dev *bd = bn->bd; 120 __le64 *rx_stats, *tx_stats; 121 int rc, rx_count, tx_count; 122 u64 *rx_masks, *tx_masks; 123 u8 flags; 124 125 nqr = &bnapi->nq_ring; 126 stats = &nqr->stats; 127 rc = bnge_hwrm_func_qstat_ext(bd, stats); 128 if (rc) { 129 u64 mask = (1ULL << 48) - 1; 130 131 bnge_fill_masks(stats->hw_masks, mask, stats->len / 8); 132 } 133 134 if (bn->flags & BNGE_FLAG_PORT_STATS) { 135 stats = &bn->port_stats; 136 rx_stats = stats->hw_stats; 137 rx_masks = stats->hw_masks; 138 rx_count = sizeof(struct rx_port_stats) / 8; 139 tx_stats = rx_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 140 tx_masks = rx_masks + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 141 tx_count = sizeof(struct tx_port_stats) / 8; 142 143 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 144 rc = bnge_hwrm_port_qstats(bd, flags); 145 if (rc) { 146 u64 mask = (1ULL << 40) - 1; 147 148 bnge_fill_masks(rx_masks, mask, rx_count); 149 bnge_fill_masks(tx_masks, mask, tx_count); 150 } else { 151 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 152 bnge_copy_hw_masks(tx_masks, tx_stats, tx_count); 153 } 154 bnge_hwrm_port_qstats(bd, 0); 155 } 156 157 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 158 stats = &bn->rx_port_stats_ext; 159 rx_stats = stats->hw_stats; 160 rx_masks = stats->hw_masks; 161 rx_count = sizeof(struct rx_port_stats_ext) / 8; 162 stats = &bn->tx_port_stats_ext; 163 tx_stats = stats->hw_stats; 164 tx_masks = stats->hw_masks; 165 tx_count = sizeof(struct tx_port_stats_ext) / 8; 166 167 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 168 rc = bnge_hwrm_port_qstats_ext(bd, flags); 169 if (rc) { 170 u64 mask = (1ULL << 40) - 1; 171 172 bnge_fill_masks(rx_masks, mask, rx_count); 173 if (tx_stats) 174 bnge_fill_masks(tx_masks, mask, tx_count); 175 } else { 176 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 177 if (tx_stats) 178 bnge_copy_hw_masks(tx_masks, tx_stats, 179 tx_count); 180 } 181 bnge_hwrm_port_qstats_ext(bd, 0); 182 } 183 } 184 185 static void bnge_free_port_ext_stats(struct bnge_net *bn) 186 { 187 bn->flags &= ~BNGE_FLAG_PORT_STATS_EXT; 188 bnge_free_stats_mem(bn, &bn->rx_port_stats_ext); 189 bnge_free_stats_mem(bn, &bn->tx_port_stats_ext); 190 } 191 192 static void bnge_free_port_stats(struct bnge_net *bn) 193 { 194 bn->flags &= ~BNGE_FLAG_PORT_STATS; 195 bnge_free_stats_mem(bn, &bn->port_stats); 196 bnge_free_port_ext_stats(bn); 197 } 198 199 static int bnge_alloc_ring_stats(struct bnge_net *bn) 200 { 201 struct bnge_dev *bd = bn->bd; 202 u32 size, i; 203 int rc; 204 205 size = bd->hw_ring_stats_size; 206 207 for (i = 0; i < bd->nq_nr_rings; i++) { 208 struct bnge_napi *bnapi = bn->bnapi[i]; 209 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 210 211 nqr->stats.len = size; 212 rc = bnge_alloc_stats_mem(bn, &nqr->stats, !i); 213 if (rc) 214 goto err_free_ring_stats; 215 216 nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 217 } 218 219 return 0; 220 221 err_free_ring_stats: 222 bnge_free_ring_stats(bn); 223 return rc; 224 } 225 226 static void bnge_alloc_port_ext_stats(struct bnge_net *bn) 227 { 228 struct bnge_dev *bd = bn->bd; 229 int rc; 230 231 if (!(bd->fw_cap & BNGE_FW_CAP_EXT_STATS_SUPPORTED)) 232 return; 233 234 if (!bn->rx_port_stats_ext.hw_stats) { 235 bn->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 236 rc = bnge_alloc_stats_mem(bn, &bn->rx_port_stats_ext, true); 237 /* Extended stats are optional */ 238 if (rc) 239 return; 240 } 241 242 if (!bn->tx_port_stats_ext.hw_stats) { 243 bn->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 244 rc = bnge_alloc_stats_mem(bn, &bn->tx_port_stats_ext, true); 245 /* Extended stats are optional */ 246 if (rc) { 247 bnge_free_port_ext_stats(bn); 248 return; 249 } 250 } 251 bn->flags |= BNGE_FLAG_PORT_STATS_EXT; 252 } 253 254 static int bnge_alloc_port_stats(struct bnge_net *bn) 255 { 256 int rc; 257 258 if (!bn->port_stats.hw_stats) { 259 bn->port_stats.len = BNGE_PORT_STATS_SIZE; 260 rc = bnge_alloc_stats_mem(bn, &bn->port_stats, true); 261 if (rc) 262 return rc; 263 264 bn->flags |= BNGE_FLAG_PORT_STATS; 265 } 266 267 bnge_alloc_port_ext_stats(bn); 268 return 0; 269 } 270 271 void __bnge_queue_sp_work(struct bnge_net *bn) 272 { 273 queue_work(bn->bnge_pf_wq, &bn->sp_task); 274 } 275 276 static void bnge_queue_sp_work(struct bnge_net *bn, unsigned int event) 277 { 278 set_bit(event, &bn->sp_event); 279 __bnge_queue_sp_work(bn); 280 } 281 282 static void bnge_timer(struct timer_list *t) 283 { 284 struct bnge_net *bn = timer_container_of(bn, t, timer); 285 struct bnge_dev *bd = bn->bd; 286 287 if (!netif_running(bn->netdev) || 288 !test_bit(BNGE_STATE_OPEN, &bd->state)) 289 return; 290 291 if (READ_ONCE(bd->link_info.phy_retry)) { 292 if (time_after(jiffies, READ_ONCE(bd->link_info.phy_retry_expires))) { 293 WRITE_ONCE(bd->link_info.phy_retry, false); 294 netdev_warn(bn->netdev, "failed to update PHY settings after maximum retries.\n"); 295 } else { 296 bnge_queue_sp_work(bn, BNGE_UPDATE_PHY_SP_EVENT); 297 } 298 } 299 300 if (BNGE_LINK_IS_UP(bd) && bn->stats_coal_ticks) 301 bnge_queue_sp_work(bn, BNGE_PERIODIC_STATS_SP_EVENT); 302 303 mod_timer(&bn->timer, jiffies + bn->current_interval); 304 } 305 306 static void bnge_add_one_ctr(u64 hw, u64 *sw, u64 mask) 307 { 308 u64 sw_tmp, sw_val; 309 310 hw &= mask; 311 sw_val = READ_ONCE(*sw); 312 sw_tmp = (sw_val & ~mask) | hw; 313 if (hw < (sw_val & mask)) 314 sw_tmp += mask + 1; 315 WRITE_ONCE(*sw, sw_tmp); 316 } 317 318 static void __bnge_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 319 int count, struct u64_stats_sync *syncp) 320 { 321 unsigned long flags; 322 int i; 323 324 flags = u64_stats_update_begin_irqsave(syncp); 325 for (i = 0; i < count; i++) { 326 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 327 328 if (masks[i] == -1ULL) 329 WRITE_ONCE(sw_stats[i], hw); 330 else 331 bnge_add_one_ctr(hw, &sw_stats[i], masks[i]); 332 } 333 u64_stats_update_end_irqrestore(syncp, flags); 334 } 335 336 static void bnge_accumulate_stats(struct bnge_stats_mem *stats) 337 { 338 if (!stats->hw_stats) 339 return; 340 341 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 342 stats->hw_masks, stats->len / 8, &stats->syncp); 343 } 344 345 static void bnge_accumulate_all_stats(struct bnge_dev *bd) 346 { 347 struct bnge_net *bn = netdev_priv(bd->netdev); 348 struct bnge_stats_mem *ring0_stats = NULL; 349 int i; 350 351 for (i = 0; i < bd->nq_nr_rings; i++) { 352 struct bnge_napi *bnapi = bn->bnapi[i]; 353 struct bnge_nq_ring_info *nqr; 354 struct bnge_stats_mem *stats; 355 356 nqr = &bnapi->nq_ring; 357 stats = &nqr->stats; 358 359 if (!ring0_stats) 360 ring0_stats = &bn->bnapi[0]->nq_ring.stats; 361 362 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 363 ring0_stats->hw_masks, 364 ring0_stats->len / 8, &stats->syncp); 365 } 366 367 if (bn->flags & BNGE_FLAG_PORT_STATS) { 368 struct bnge_stats_mem *stats = &bn->port_stats; 369 __le64 *hw_stats = stats->hw_stats; 370 u64 *sw_stats = stats->sw_stats; 371 u64 *masks = stats->hw_masks; 372 u16 cnt; 373 374 cnt = sizeof(struct rx_port_stats) / 8; 375 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 376 &stats->syncp); 377 378 hw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 379 sw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 380 masks += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 381 cnt = sizeof(struct tx_port_stats) / 8; 382 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 383 &stats->syncp); 384 } 385 386 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 387 bnge_accumulate_stats(&bn->rx_port_stats_ext); 388 bnge_accumulate_stats(&bn->tx_port_stats_ext); 389 } 390 } 391 392 static void bnge_sp_task(struct work_struct *work) 393 { 394 struct bnge_net *bn = container_of(work, struct bnge_net, sp_task); 395 bool speed_chng, cfg_chng, link_chng; 396 struct bnge_dev *bd = bn->bd; 397 398 netdev_lock(bn->netdev); 399 if (!test_bit(BNGE_STATE_OPEN, &bd->state)) { 400 netdev_unlock(bn->netdev); 401 return; 402 } 403 404 if (test_and_clear_bit(BNGE_PERIODIC_STATS_SP_EVENT, &bn->sp_event)) { 405 bnge_hwrm_port_qstats(bd, 0); 406 bnge_hwrm_port_qstats_ext(bd, 0); 407 bnge_accumulate_all_stats(bd); 408 } 409 410 if (test_and_clear_bit(BNGE_UPDATE_PHY_SP_EVENT, &bn->sp_event)) { 411 int rc; 412 413 rc = bnge_update_phy_setting(bn); 414 if (rc) { 415 netdev_warn(bn->netdev, "update PHY settings retry failed\n"); 416 } else { 417 WRITE_ONCE(bd->link_info.phy_retry, false); 418 netdev_info(bn->netdev, "update PHY settings retry succeeded\n"); 419 } 420 } 421 422 speed_chng = test_and_clear_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, 423 &bn->sp_event); 424 cfg_chng = test_and_clear_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, 425 &bn->sp_event); 426 link_chng = test_and_clear_bit(BNGE_LINK_CHNG_SP_EVENT, 427 &bn->sp_event); 428 429 if (speed_chng || cfg_chng || link_chng) { 430 int rc; 431 432 if (speed_chng) 433 bnge_hwrm_phy_qcaps(bd); 434 435 rc = bnge_update_link(bn, true); 436 if (rc) 437 netdev_err(bn->netdev, "SP task cannot update link (rc: %d)\n", 438 rc); 439 440 if (speed_chng || cfg_chng) 441 bnge_init_ethtool_link_settings(bn); 442 } 443 444 netdev_unlock(bn->netdev); 445 } 446 447 static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr) 448 { 449 struct bnge_ring_struct *ring = &nqr->ring_struct; 450 451 kfree(nqr->desc_ring); 452 nqr->desc_ring = NULL; 453 ring->ring_mem.pg_arr = NULL; 454 kfree(nqr->desc_mapping); 455 nqr->desc_mapping = NULL; 456 ring->ring_mem.dma_arr = NULL; 457 } 458 459 static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr) 460 { 461 struct bnge_ring_struct *ring = &cpr->ring_struct; 462 463 kfree(cpr->desc_ring); 464 cpr->desc_ring = NULL; 465 ring->ring_mem.pg_arr = NULL; 466 kfree(cpr->desc_mapping); 467 cpr->desc_mapping = NULL; 468 ring->ring_mem.dma_arr = NULL; 469 } 470 471 static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n) 472 { 473 nqr->desc_ring = kzalloc_objs(*nqr->desc_ring, n); 474 if (!nqr->desc_ring) 475 return -ENOMEM; 476 477 nqr->desc_mapping = kzalloc_objs(*nqr->desc_mapping, n); 478 if (!nqr->desc_mapping) 479 goto err_free_desc_ring; 480 return 0; 481 482 err_free_desc_ring: 483 kfree(nqr->desc_ring); 484 nqr->desc_ring = NULL; 485 return -ENOMEM; 486 } 487 488 static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n) 489 { 490 cpr->desc_ring = kzalloc_objs(*cpr->desc_ring, n); 491 if (!cpr->desc_ring) 492 return -ENOMEM; 493 494 cpr->desc_mapping = kzalloc_objs(*cpr->desc_mapping, n); 495 if (!cpr->desc_mapping) 496 goto err_free_desc_ring; 497 return 0; 498 499 err_free_desc_ring: 500 kfree(cpr->desc_ring); 501 cpr->desc_ring = NULL; 502 return -ENOMEM; 503 } 504 505 static void bnge_free_nq_arrays(struct bnge_net *bn) 506 { 507 struct bnge_dev *bd = bn->bd; 508 int i; 509 510 for (i = 0; i < bd->nq_nr_rings; i++) { 511 struct bnge_napi *bnapi = bn->bnapi[i]; 512 513 bnge_free_nq_desc_arr(&bnapi->nq_ring); 514 } 515 } 516 517 static int bnge_alloc_nq_arrays(struct bnge_net *bn) 518 { 519 struct bnge_dev *bd = bn->bd; 520 int i, rc; 521 522 for (i = 0; i < bd->nq_nr_rings; i++) { 523 struct bnge_napi *bnapi = bn->bnapi[i]; 524 525 rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages); 526 if (rc) 527 goto err_free_nq_arrays; 528 } 529 return 0; 530 531 err_free_nq_arrays: 532 bnge_free_nq_arrays(bn); 533 return rc; 534 } 535 536 static void bnge_free_nq_tree(struct bnge_net *bn) 537 { 538 struct bnge_dev *bd = bn->bd; 539 int i; 540 541 for (i = 0; i < bd->nq_nr_rings; i++) { 542 struct bnge_napi *bnapi = bn->bnapi[i]; 543 struct bnge_nq_ring_info *nqr; 544 struct bnge_ring_struct *ring; 545 int j; 546 547 nqr = &bnapi->nq_ring; 548 ring = &nqr->ring_struct; 549 550 bnge_free_ring(bd, &ring->ring_mem); 551 552 if (!nqr->cp_ring_arr) 553 continue; 554 555 for (j = 0; j < nqr->cp_ring_count; j++) { 556 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 557 558 ring = &cpr->ring_struct; 559 bnge_free_ring(bd, &ring->ring_mem); 560 bnge_free_cp_desc_arr(cpr); 561 } 562 kfree(nqr->cp_ring_arr); 563 nqr->cp_ring_arr = NULL; 564 nqr->cp_ring_count = 0; 565 } 566 } 567 568 static int alloc_one_cp_ring(struct bnge_net *bn, 569 struct bnge_cp_ring_info *cpr) 570 { 571 struct bnge_ring_mem_info *rmem; 572 struct bnge_ring_struct *ring; 573 struct bnge_dev *bd = bn->bd; 574 int rc; 575 576 rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages); 577 if (rc) 578 return -ENOMEM; 579 ring = &cpr->ring_struct; 580 rmem = &ring->ring_mem; 581 rmem->nr_pages = bn->cp_nr_pages; 582 rmem->page_size = HW_CMPD_RING_SIZE; 583 rmem->pg_arr = (void **)cpr->desc_ring; 584 rmem->dma_arr = cpr->desc_mapping; 585 rmem->flags = BNGE_RMEM_RING_PTE_FLAG; 586 rc = bnge_alloc_ring(bd, rmem); 587 if (rc) 588 goto err_free_cp_desc_arr; 589 return rc; 590 591 err_free_cp_desc_arr: 592 bnge_free_cp_desc_arr(cpr); 593 return rc; 594 } 595 596 static int bnge_alloc_nq_tree(struct bnge_net *bn) 597 { 598 struct bnge_dev *bd = bn->bd; 599 int i, j, ulp_msix, rc; 600 int tcs = 1; 601 602 ulp_msix = bnge_aux_get_msix(bd); 603 for (i = 0, j = 0; i < bd->nq_nr_rings; i++) { 604 bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL); 605 struct bnge_napi *bnapi = bn->bnapi[i]; 606 struct bnge_nq_ring_info *nqr; 607 struct bnge_cp_ring_info *cpr; 608 struct bnge_ring_struct *ring; 609 int cp_count = 0, k; 610 int rx = 0, tx = 0; 611 612 nqr = &bnapi->nq_ring; 613 nqr->bnapi = bnapi; 614 ring = &nqr->ring_struct; 615 616 rc = bnge_alloc_ring(bd, &ring->ring_mem); 617 if (rc) 618 goto err_free_nq_tree; 619 620 ring->map_idx = ulp_msix + i; 621 622 if (i < bd->rx_nr_rings) { 623 cp_count++; 624 rx = 1; 625 } 626 627 if ((sh && i < bd->tx_nr_rings) || 628 (!sh && i >= bd->rx_nr_rings)) { 629 cp_count += tcs; 630 tx = 1; 631 } 632 633 nqr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); 634 if (!nqr->cp_ring_arr) { 635 rc = -ENOMEM; 636 goto err_free_nq_tree; 637 } 638 639 nqr->cp_ring_count = cp_count; 640 641 for (k = 0; k < cp_count; k++) { 642 cpr = &nqr->cp_ring_arr[k]; 643 rc = alloc_one_cp_ring(bn, cpr); 644 if (rc) 645 goto err_free_nq_tree; 646 647 cpr->bnapi = bnapi; 648 cpr->cp_idx = k; 649 if (!k && rx) { 650 bn->rx_ring[i].rx_cpr = cpr; 651 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX; 652 } else { 653 int n, tc = k - rx; 654 655 n = BNGE_TC_TO_RING_BASE(bd, tc) + j; 656 bn->tx_ring[n].tx_cpr = cpr; 657 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX; 658 } 659 } 660 if (tx) 661 j++; 662 } 663 return 0; 664 665 err_free_nq_tree: 666 bnge_free_nq_tree(bn); 667 return rc; 668 } 669 670 static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr) 671 { 672 return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE; 673 } 674 675 static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn, 676 struct bnge_rx_ring_info *rxr) 677 { 678 int i, max_idx; 679 680 if (!rxr->rx_buf_ring) 681 return; 682 683 max_idx = bn->rx_nr_pages * RX_DESC_CNT; 684 685 for (i = 0; i < max_idx; i++) { 686 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 687 void *data = rx_buf->data; 688 689 if (!data) 690 continue; 691 692 rx_buf->data = NULL; 693 page_pool_free_va(rxr->head_pool, data, true); 694 } 695 } 696 697 static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn, 698 struct bnge_rx_ring_info *rxr) 699 { 700 int i, max_idx; 701 702 if (!rxr->rx_agg_buf_ring) 703 return; 704 705 max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT; 706 707 for (i = 0; i < max_idx; i++) { 708 struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i]; 709 netmem_ref netmem = rx_agg_buf->netmem; 710 711 if (!netmem) 712 continue; 713 714 rx_agg_buf->netmem = 0; 715 __clear_bit(i, rxr->rx_agg_bmap); 716 717 page_pool_recycle_direct_netmem(rxr->page_pool, netmem); 718 } 719 } 720 721 static void bnge_free_one_tpa_info_data(struct bnge_net *bn, 722 struct bnge_rx_ring_info *rxr) 723 { 724 int i; 725 726 for (i = 0; i < bn->max_tpa; i++) { 727 struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[i]; 728 u8 *data = tpa_info->data; 729 730 if (!data) 731 continue; 732 733 tpa_info->data = NULL; 734 page_pool_free_va(rxr->head_pool, data, false); 735 } 736 } 737 738 static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn, 739 struct bnge_rx_ring_info *rxr) 740 { 741 struct bnge_tpa_idx_map *map; 742 743 if (rxr->rx_tpa) 744 bnge_free_one_tpa_info_data(bn, rxr); 745 746 bnge_free_one_rx_ring_bufs(bn, rxr); 747 bnge_free_one_agg_ring_bufs(bn, rxr); 748 749 map = rxr->rx_tpa_idx_map; 750 if (map) 751 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 752 } 753 754 static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn) 755 { 756 struct bnge_dev *bd = bn->bd; 757 int i; 758 759 if (!bn->rx_ring) 760 return; 761 762 for (i = 0; i < bd->rx_nr_rings; i++) 763 bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]); 764 } 765 766 static void bnge_free_tx_skbs(struct bnge_net *bn) 767 { 768 struct bnge_dev *bd = bn->bd; 769 u16 max_idx; 770 int i; 771 772 max_idx = bn->tx_nr_pages * TX_DESC_CNT; 773 for (i = 0; i < bd->tx_nr_rings; i++) { 774 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 775 int j; 776 777 if (!txr->tx_buf_ring) 778 continue; 779 780 for (j = 0; j < max_idx;) { 781 struct bnge_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 782 struct sk_buff *skb; 783 int k, last; 784 785 skb = tx_buf->skb; 786 if (!skb) { 787 j++; 788 continue; 789 } 790 791 tx_buf->skb = NULL; 792 793 dma_unmap_single(bd->dev, 794 dma_unmap_addr(tx_buf, mapping), 795 skb_headlen(skb), 796 DMA_TO_DEVICE); 797 798 last = tx_buf->nr_frags; 799 j += 2; 800 for (k = 0; k < last; k++, j++) { 801 int ring_idx = j & bn->tx_ring_mask; 802 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 803 804 tx_buf = &txr->tx_buf_ring[ring_idx]; 805 dma_unmap_page(bd->dev, 806 dma_unmap_addr(tx_buf, mapping), 807 skb_frag_size(frag), 808 DMA_TO_DEVICE); 809 } 810 dev_kfree_skb(skb); 811 } 812 netdev_tx_reset_queue(netdev_get_tx_queue(bd->netdev, i)); 813 } 814 } 815 816 static void bnge_free_all_rings_bufs(struct bnge_net *bn) 817 { 818 bnge_free_rx_ring_pair_bufs(bn); 819 bnge_free_tx_skbs(bn); 820 } 821 822 static void bnge_free_tpa_info(struct bnge_net *bn) 823 { 824 struct bnge_dev *bd = bn->bd; 825 int i, j; 826 827 for (i = 0; i < bd->rx_nr_rings; i++) { 828 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 829 830 kfree(rxr->rx_tpa_idx_map); 831 rxr->rx_tpa_idx_map = NULL; 832 if (rxr->rx_tpa) { 833 for (j = 0; j < bn->max_tpa; j++) { 834 kfree(rxr->rx_tpa[j].agg_arr); 835 rxr->rx_tpa[j].agg_arr = NULL; 836 } 837 } 838 kfree(rxr->rx_tpa); 839 rxr->rx_tpa = NULL; 840 } 841 } 842 843 static int bnge_alloc_tpa_info(struct bnge_net *bn) 844 { 845 struct bnge_dev *bd = bn->bd; 846 int i, j; 847 848 if (!bd->max_tpa_v2) 849 return 0; 850 851 bn->max_tpa = max_t(u16, bd->max_tpa_v2, MAX_TPA); 852 for (i = 0; i < bd->rx_nr_rings; i++) { 853 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 854 855 rxr->rx_tpa = kzalloc_objs(struct bnge_tpa_info, bn->max_tpa); 856 if (!rxr->rx_tpa) 857 goto err_free_tpa_info; 858 859 for (j = 0; j < bn->max_tpa; j++) { 860 struct rx_agg_cmp *agg; 861 862 agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); 863 if (!agg) 864 goto err_free_tpa_info; 865 rxr->rx_tpa[j].agg_arr = agg; 866 } 867 rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); 868 if (!rxr->rx_tpa_idx_map) 869 goto err_free_tpa_info; 870 } 871 return 0; 872 873 err_free_tpa_info: 874 bnge_free_tpa_info(bn); 875 return -ENOMEM; 876 } 877 878 static void bnge_free_rx_rings(struct bnge_net *bn) 879 { 880 struct bnge_dev *bd = bn->bd; 881 int i; 882 883 bnge_free_tpa_info(bn); 884 for (i = 0; i < bd->rx_nr_rings; i++) { 885 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 886 struct bnge_ring_struct *ring; 887 888 page_pool_destroy(rxr->page_pool); 889 page_pool_destroy(rxr->head_pool); 890 rxr->page_pool = rxr->head_pool = NULL; 891 892 kfree(rxr->rx_agg_bmap); 893 rxr->rx_agg_bmap = NULL; 894 895 ring = &rxr->rx_ring_struct; 896 bnge_free_ring(bd, &ring->ring_mem); 897 898 ring = &rxr->rx_agg_ring_struct; 899 bnge_free_ring(bd, &ring->ring_mem); 900 } 901 } 902 903 static int bnge_alloc_rx_page_pool(struct bnge_net *bn, 904 struct bnge_rx_ring_info *rxr, 905 int numa_node) 906 { 907 const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE; 908 const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; 909 struct page_pool_params pp = { 0 }; 910 struct bnge_dev *bd = bn->bd; 911 struct page_pool *pool; 912 913 pp.pool_size = bn->rx_agg_ring_size / agg_size_fac; 914 pp.nid = numa_node; 915 pp.netdev = bn->netdev; 916 pp.dev = bd->dev; 917 pp.dma_dir = bn->rx_dir; 918 pp.max_len = PAGE_SIZE; 919 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | 920 PP_FLAG_ALLOW_UNREADABLE_NETMEM; 921 pp.queue_idx = rxr->bnapi->index; 922 923 pool = page_pool_create(&pp); 924 if (IS_ERR(pool)) 925 return PTR_ERR(pool); 926 rxr->page_pool = pool; 927 928 rxr->need_head_pool = page_pool_is_unreadable(pool); 929 if (bnge_separate_head_pool(rxr)) { 930 pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024); 931 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 932 pool = page_pool_create(&pp); 933 if (IS_ERR(pool)) 934 goto err_destroy_pp; 935 } else { 936 page_pool_get(pool); 937 } 938 rxr->head_pool = pool; 939 return 0; 940 941 err_destroy_pp: 942 page_pool_destroy(rxr->page_pool); 943 rxr->page_pool = NULL; 944 return PTR_ERR(pool); 945 } 946 947 static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr) 948 { 949 page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi); 950 page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); 951 } 952 953 static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn, 954 struct bnge_rx_ring_info *rxr) 955 { 956 u16 mem_size; 957 958 rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1; 959 mem_size = rxr->rx_agg_bmap_size / 8; 960 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 961 if (!rxr->rx_agg_bmap) 962 return -ENOMEM; 963 964 return 0; 965 } 966 967 static int bnge_alloc_rx_rings(struct bnge_net *bn) 968 { 969 int i, rc = 0, agg_rings = 0, cpu; 970 struct bnge_dev *bd = bn->bd; 971 972 if (bnge_is_agg_reqd(bd)) 973 agg_rings = 1; 974 975 for (i = 0; i < bd->rx_nr_rings; i++) { 976 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 977 struct bnge_ring_struct *ring; 978 int cpu_node; 979 980 ring = &rxr->rx_ring_struct; 981 982 cpu = cpumask_local_spread(i, dev_to_node(bd->dev)); 983 cpu_node = cpu_to_node(cpu); 984 netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", 985 i, cpu_node); 986 rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node); 987 if (rc) 988 goto err_free_rx_rings; 989 bnge_enable_rx_page_pool(rxr); 990 991 rc = bnge_alloc_ring(bd, &ring->ring_mem); 992 if (rc) 993 goto err_free_rx_rings; 994 995 ring->grp_idx = i; 996 if (agg_rings) { 997 ring = &rxr->rx_agg_ring_struct; 998 rc = bnge_alloc_ring(bd, &ring->ring_mem); 999 if (rc) 1000 goto err_free_rx_rings; 1001 1002 ring->grp_idx = i; 1003 rc = bnge_alloc_rx_agg_bmap(bn, rxr); 1004 if (rc) 1005 goto err_free_rx_rings; 1006 } 1007 } 1008 1009 if (bn->priv_flags & BNGE_NET_EN_TPA) { 1010 rc = bnge_alloc_tpa_info(bn); 1011 if (rc) 1012 goto err_free_rx_rings; 1013 } 1014 return rc; 1015 1016 err_free_rx_rings: 1017 bnge_free_rx_rings(bn); 1018 return rc; 1019 } 1020 1021 static void bnge_free_tx_rings(struct bnge_net *bn) 1022 { 1023 struct bnge_dev *bd = bn->bd; 1024 int i; 1025 1026 for (i = 0; i < bd->tx_nr_rings; i++) { 1027 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1028 struct bnge_ring_struct *ring; 1029 1030 ring = &txr->tx_ring_struct; 1031 1032 bnge_free_ring(bd, &ring->ring_mem); 1033 } 1034 } 1035 1036 static int bnge_alloc_tx_rings(struct bnge_net *bn) 1037 { 1038 struct bnge_dev *bd = bn->bd; 1039 int i, j, rc; 1040 1041 for (i = 0, j = 0; i < bd->tx_nr_rings; i++) { 1042 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1043 struct bnge_ring_struct *ring; 1044 u8 qidx; 1045 1046 ring = &txr->tx_ring_struct; 1047 1048 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1049 if (rc) 1050 goto err_free_tx_rings; 1051 1052 ring->grp_idx = txr->bnapi->index; 1053 qidx = bd->tc_to_qidx[j]; 1054 ring->queue_id = bd->q_info[qidx].queue_id; 1055 if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1)) 1056 j++; 1057 } 1058 return 0; 1059 1060 err_free_tx_rings: 1061 bnge_free_tx_rings(bn); 1062 return rc; 1063 } 1064 1065 static void bnge_free_vnic_attributes(struct bnge_net *bn) 1066 { 1067 struct pci_dev *pdev = bn->bd->pdev; 1068 struct bnge_vnic_info *vnic; 1069 int i; 1070 1071 if (!bn->vnic_info) 1072 return; 1073 1074 for (i = 0; i < bn->nr_vnics; i++) { 1075 vnic = &bn->vnic_info[i]; 1076 1077 kfree(vnic->uc_list); 1078 vnic->uc_list = NULL; 1079 1080 if (vnic->mc_list) { 1081 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 1082 vnic->mc_list, vnic->mc_list_mapping); 1083 vnic->mc_list = NULL; 1084 } 1085 1086 if (vnic->rss_table) { 1087 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 1088 vnic->rss_table, 1089 vnic->rss_table_dma_addr); 1090 vnic->rss_table = NULL; 1091 } 1092 1093 vnic->rss_hash_key = NULL; 1094 vnic->flags = 0; 1095 } 1096 } 1097 1098 static int bnge_alloc_vnic_attributes(struct bnge_net *bn) 1099 { 1100 struct bnge_dev *bd = bn->bd; 1101 struct bnge_vnic_info *vnic; 1102 int i, size; 1103 1104 for (i = 0; i < bn->nr_vnics; i++) { 1105 vnic = &bn->vnic_info[i]; 1106 1107 if (vnic->flags & BNGE_VNIC_UCAST_FLAG) { 1108 int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN; 1109 1110 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 1111 if (!vnic->uc_list) 1112 goto err_free_vnic_attributes; 1113 } 1114 1115 if (vnic->flags & BNGE_VNIC_MCAST_FLAG) { 1116 vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN; 1117 vnic->mc_list = 1118 dma_alloc_coherent(bd->dev, 1119 vnic->mc_list_size, 1120 &vnic->mc_list_mapping, 1121 GFP_KERNEL); 1122 if (!vnic->mc_list) 1123 goto err_free_vnic_attributes; 1124 } 1125 1126 /* Allocate rss table and hash key */ 1127 size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE); 1128 1129 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 1130 vnic->rss_table = dma_alloc_coherent(bd->dev, 1131 vnic->rss_table_size, 1132 &vnic->rss_table_dma_addr, 1133 GFP_KERNEL); 1134 if (!vnic->rss_table) 1135 goto err_free_vnic_attributes; 1136 1137 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 1138 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 1139 } 1140 return 0; 1141 1142 err_free_vnic_attributes: 1143 bnge_free_vnic_attributes(bn); 1144 return -ENOMEM; 1145 } 1146 1147 static int bnge_alloc_vnics(struct bnge_net *bn) 1148 { 1149 int num_vnics; 1150 1151 /* Allocate only 1 VNIC for now 1152 * Additional VNICs will be added based on RFS/NTUPLE in future patches 1153 */ 1154 num_vnics = 1; 1155 1156 bn->vnic_info = kzalloc_objs(struct bnge_vnic_info, num_vnics); 1157 if (!bn->vnic_info) 1158 return -ENOMEM; 1159 1160 bn->nr_vnics = num_vnics; 1161 1162 return 0; 1163 } 1164 1165 static void bnge_free_vnics(struct bnge_net *bn) 1166 { 1167 kfree(bn->vnic_info); 1168 bn->vnic_info = NULL; 1169 bn->nr_vnics = 0; 1170 } 1171 1172 static void bnge_free_ring_grps(struct bnge_net *bn) 1173 { 1174 kfree(bn->grp_info); 1175 bn->grp_info = NULL; 1176 } 1177 1178 static int bnge_init_ring_grps(struct bnge_net *bn) 1179 { 1180 struct bnge_dev *bd = bn->bd; 1181 int i; 1182 1183 bn->grp_info = kzalloc_objs(struct bnge_ring_grp_info, bd->nq_nr_rings); 1184 if (!bn->grp_info) 1185 return -ENOMEM; 1186 for (i = 0; i < bd->nq_nr_rings; i++) { 1187 bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 1188 bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 1189 bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 1190 bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 1191 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 1192 } 1193 1194 return 0; 1195 } 1196 1197 static void bnge_free_core(struct bnge_net *bn) 1198 { 1199 bnge_free_vnic_attributes(bn); 1200 bnge_free_tx_rings(bn); 1201 bnge_free_rx_rings(bn); 1202 bnge_free_nq_tree(bn); 1203 bnge_free_nq_arrays(bn); 1204 bnge_free_ring_stats(bn); 1205 bnge_free_ring_grps(bn); 1206 bnge_free_vnics(bn); 1207 kfree(bn->tx_ring_map); 1208 bn->tx_ring_map = NULL; 1209 kfree(bn->tx_ring); 1210 bn->tx_ring = NULL; 1211 kfree(bn->rx_ring); 1212 bn->rx_ring = NULL; 1213 kfree(bn->bnapi); 1214 bn->bnapi = NULL; 1215 } 1216 1217 static int bnge_alloc_core(struct bnge_net *bn) 1218 { 1219 struct bnge_dev *bd = bn->bd; 1220 int i, j, size, arr_size; 1221 int rc = -ENOMEM; 1222 void *bnapi; 1223 1224 arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) * 1225 bd->nq_nr_rings); 1226 size = L1_CACHE_ALIGN(sizeof(struct bnge_napi)); 1227 bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL); 1228 if (!bnapi) 1229 return rc; 1230 1231 bn->bnapi = bnapi; 1232 bnapi += arr_size; 1233 for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) { 1234 struct bnge_nq_ring_info *nqr; 1235 1236 bn->bnapi[i] = bnapi; 1237 bn->bnapi[i]->index = i; 1238 bn->bnapi[i]->bn = bn; 1239 nqr = &bn->bnapi[i]->nq_ring; 1240 nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1241 } 1242 1243 bn->rx_ring = kzalloc_objs(struct bnge_rx_ring_info, bd->rx_nr_rings); 1244 if (!bn->rx_ring) 1245 goto err_free_core; 1246 1247 for (i = 0; i < bd->rx_nr_rings; i++) { 1248 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1249 1250 rxr->rx_ring_struct.ring_mem.flags = 1251 BNGE_RMEM_RING_PTE_FLAG; 1252 rxr->rx_agg_ring_struct.ring_mem.flags = 1253 BNGE_RMEM_RING_PTE_FLAG; 1254 rxr->bnapi = bn->bnapi[i]; 1255 bn->bnapi[i]->rx_ring = &bn->rx_ring[i]; 1256 } 1257 1258 bn->tx_ring = kzalloc_objs(struct bnge_tx_ring_info, bd->tx_nr_rings); 1259 if (!bn->tx_ring) 1260 goto err_free_core; 1261 1262 bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16), 1263 GFP_KERNEL); 1264 if (!bn->tx_ring_map) 1265 goto err_free_core; 1266 1267 if (bd->flags & BNGE_EN_SHARED_CHNL) 1268 j = 0; 1269 else 1270 j = bd->rx_nr_rings; 1271 1272 for (i = 0; i < bd->tx_nr_rings; i++) { 1273 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1274 struct bnge_napi *bnapi2; 1275 int k; 1276 1277 txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1278 bn->tx_ring_map[i] = i; 1279 k = j + BNGE_RING_TO_TC_OFF(bd, i); 1280 1281 bnapi2 = bn->bnapi[k]; 1282 txr->txq_index = i; 1283 txr->tx_napi_idx = 1284 BNGE_RING_TO_TC(bd, txr->txq_index); 1285 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 1286 txr->bnapi = bnapi2; 1287 } 1288 1289 rc = bnge_alloc_ring_stats(bn); 1290 if (rc) 1291 goto err_free_core; 1292 1293 bnge_init_stats(bn); 1294 1295 rc = bnge_alloc_vnics(bn); 1296 if (rc) 1297 goto err_free_core; 1298 1299 rc = bnge_alloc_nq_arrays(bn); 1300 if (rc) 1301 goto err_free_core; 1302 1303 bnge_init_ring_struct(bn); 1304 1305 rc = bnge_alloc_rx_rings(bn); 1306 if (rc) 1307 goto err_free_core; 1308 1309 rc = bnge_alloc_tx_rings(bn); 1310 if (rc) 1311 goto err_free_core; 1312 1313 rc = bnge_alloc_nq_tree(bn); 1314 if (rc) 1315 goto err_free_core; 1316 1317 bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG | 1318 BNGE_VNIC_MCAST_FLAG | 1319 BNGE_VNIC_UCAST_FLAG; 1320 rc = bnge_alloc_vnic_attributes(bn); 1321 if (rc) 1322 goto err_free_core; 1323 return 0; 1324 1325 err_free_core: 1326 bnge_free_core(bn); 1327 return rc; 1328 } 1329 1330 u32 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr) 1331 { 1332 return rxr->rx_cpr->ring_struct.fw_ring_id; 1333 } 1334 1335 u32 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr) 1336 { 1337 return txr->tx_cpr->ring_struct.fw_ring_id; 1338 } 1339 1340 static void bnge_db_nq_arm(struct bnge_net *bn, 1341 struct bnge_db_info *db, u32 idx) 1342 { 1343 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_ARM | 1344 DB_RING_IDX(db, idx), db->doorbell); 1345 } 1346 1347 static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1348 { 1349 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK | 1350 DB_RING_IDX(db, idx), db->doorbell); 1351 } 1352 1353 static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1354 { 1355 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL | 1356 DB_RING_IDX(db, idx), db->doorbell); 1357 } 1358 1359 static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n) 1360 { 1361 struct bnge_napi *bnapi = bn->bnapi[n]; 1362 struct bnge_nq_ring_info *nqr; 1363 1364 nqr = &bnapi->nq_ring; 1365 1366 return nqr->ring_struct.map_idx; 1367 } 1368 1369 static void bnge_init_nq_tree(struct bnge_net *bn) 1370 { 1371 struct bnge_dev *bd = bn->bd; 1372 int i, j; 1373 1374 for (i = 0; i < bd->nq_nr_rings; i++) { 1375 struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring; 1376 struct bnge_ring_struct *ring = &nqr->ring_struct; 1377 1378 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1379 for (j = 0; j < nqr->cp_ring_count; j++) { 1380 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 1381 1382 ring = &cpr->ring_struct; 1383 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1384 } 1385 } 1386 } 1387 1388 static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn, 1389 dma_addr_t *mapping, 1390 struct bnge_rx_ring_info *rxr, 1391 unsigned int *offset, 1392 gfp_t gfp) 1393 { 1394 netmem_ref netmem; 1395 1396 if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) { 1397 netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, 1398 BNGE_RX_PAGE_SIZE, gfp); 1399 } else { 1400 netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); 1401 *offset = 0; 1402 } 1403 if (!netmem) 1404 return 0; 1405 1406 *mapping = page_pool_get_dma_addr_netmem(netmem) + *offset; 1407 return netmem; 1408 } 1409 1410 u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping, 1411 struct bnge_rx_ring_info *rxr, 1412 gfp_t gfp) 1413 { 1414 unsigned int offset; 1415 struct page *page; 1416 1417 page = page_pool_alloc_frag(rxr->head_pool, &offset, 1418 bn->rx_buf_size, gfp); 1419 if (!page) 1420 return NULL; 1421 1422 *mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset; 1423 return page_address(page) + offset; 1424 } 1425 1426 int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr, 1427 u16 prod, gfp_t gfp) 1428 { 1429 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)]; 1430 struct rx_bd *rxbd; 1431 dma_addr_t mapping; 1432 u8 *data; 1433 1434 rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)]; 1435 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp); 1436 if (!data) 1437 return -ENOMEM; 1438 1439 rx_buf->data = data; 1440 rx_buf->data_ptr = data + bn->rx_offset; 1441 rx_buf->mapping = mapping; 1442 1443 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1444 1445 return 0; 1446 } 1447 1448 static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn, 1449 struct bnge_rx_ring_info *rxr, 1450 int ring_nr) 1451 { 1452 u32 prod = rxr->rx_prod; 1453 int i, rc = 0; 1454 1455 for (i = 0; i < bn->rx_ring_size; i++) { 1456 rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL); 1457 if (rc) 1458 break; 1459 prod = NEXT_RX(prod); 1460 } 1461 1462 /* Abort if not a single buffer can be allocated */ 1463 if (rc && !i) { 1464 netdev_err(bn->netdev, 1465 "RX ring %d: allocated %d/%d buffers, abort\n", 1466 ring_nr, i, bn->rx_ring_size); 1467 return rc; 1468 } 1469 1470 rxr->rx_prod = prod; 1471 1472 if (i < bn->rx_ring_size) 1473 netdev_warn(bn->netdev, 1474 "RX ring %d: allocated %d/%d buffers, continuing\n", 1475 ring_nr, i, bn->rx_ring_size); 1476 return 0; 1477 } 1478 1479 u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx) 1480 { 1481 u16 next, max = rxr->rx_agg_bmap_size; 1482 1483 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 1484 if (next >= max) 1485 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 1486 return next; 1487 } 1488 1489 int bnge_alloc_rx_netmem(struct bnge_net *bn, 1490 struct bnge_rx_ring_info *rxr, 1491 u16 prod, gfp_t gfp) 1492 { 1493 struct bnge_sw_rx_agg_bd *rx_agg_buf; 1494 u16 sw_prod = rxr->rx_sw_agg_prod; 1495 unsigned int offset = 0; 1496 struct rx_bd *rxbd; 1497 dma_addr_t mapping; 1498 netmem_ref netmem; 1499 1500 rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)]; 1501 netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp); 1502 if (!netmem) 1503 return -ENOMEM; 1504 1505 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1506 sw_prod = bnge_find_next_agg_idx(rxr, sw_prod); 1507 1508 __set_bit(sw_prod, rxr->rx_agg_bmap); 1509 rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod]; 1510 rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod)); 1511 1512 rx_agg_buf->netmem = netmem; 1513 rx_agg_buf->offset = offset; 1514 rx_agg_buf->mapping = mapping; 1515 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1516 rxbd->rx_bd_opaque = sw_prod; 1517 return 0; 1518 } 1519 1520 static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn, 1521 struct bnge_rx_ring_info *rxr, 1522 int ring_nr) 1523 { 1524 u32 prod = rxr->rx_agg_prod; 1525 int i, rc = 0; 1526 1527 for (i = 0; i < bn->rx_agg_ring_size; i++) { 1528 rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL); 1529 if (rc) 1530 break; 1531 prod = NEXT_RX_AGG(prod); 1532 } 1533 1534 if (rc && i < MAX_SKB_FRAGS) { 1535 netdev_err(bn->netdev, 1536 "Agg ring %d: allocated %d/%d buffers (min %d), abort\n", 1537 ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS); 1538 goto err_free_one_agg_ring_bufs; 1539 } 1540 1541 rxr->rx_agg_prod = prod; 1542 1543 if (i < bn->rx_agg_ring_size) 1544 netdev_warn(bn->netdev, 1545 "Agg ring %d: allocated %d/%d buffers, continuing\n", 1546 ring_nr, i, bn->rx_agg_ring_size); 1547 return 0; 1548 1549 err_free_one_agg_ring_bufs: 1550 bnge_free_one_agg_ring_bufs(bn, rxr); 1551 return -ENOMEM; 1552 } 1553 1554 static int bnge_alloc_one_tpa_info_data(struct bnge_net *bn, 1555 struct bnge_rx_ring_info *rxr) 1556 { 1557 dma_addr_t mapping; 1558 u8 *data; 1559 int i; 1560 1561 for (i = 0; i < bn->max_tpa; i++) { 1562 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, 1563 GFP_KERNEL); 1564 if (!data) 1565 goto err_free_tpa_info_data; 1566 1567 rxr->rx_tpa[i].data = data; 1568 rxr->rx_tpa[i].data_ptr = data + bn->rx_offset; 1569 rxr->rx_tpa[i].mapping = mapping; 1570 } 1571 return 0; 1572 1573 err_free_tpa_info_data: 1574 bnge_free_one_tpa_info_data(bn, rxr); 1575 return -ENOMEM; 1576 } 1577 1578 static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr) 1579 { 1580 struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr]; 1581 int rc; 1582 1583 rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr); 1584 if (rc) 1585 return rc; 1586 1587 if (bnge_is_agg_reqd(bn->bd)) { 1588 rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr); 1589 if (rc) 1590 goto err_free_one_rx_ring_bufs; 1591 } 1592 1593 if (rxr->rx_tpa) { 1594 rc = bnge_alloc_one_tpa_info_data(bn, rxr); 1595 if (rc) 1596 goto err_free_one_agg_ring_bufs; 1597 } 1598 1599 return 0; 1600 1601 err_free_one_agg_ring_bufs: 1602 bnge_free_one_agg_ring_bufs(bn, rxr); 1603 err_free_one_rx_ring_bufs: 1604 bnge_free_one_rx_ring_bufs(bn, rxr); 1605 return rc; 1606 } 1607 1608 static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type) 1609 { 1610 struct rx_bd **rx_desc_ring; 1611 u32 prod; 1612 int i; 1613 1614 rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 1615 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 1616 struct rx_bd *rxbd = rx_desc_ring[i]; 1617 int j; 1618 1619 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 1620 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 1621 rxbd->rx_bd_opaque = prod; 1622 } 1623 } 1624 } 1625 1626 static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn, 1627 struct bnge_rx_ring_info *rxr) 1628 { 1629 struct bnge_ring_struct *ring; 1630 u32 type; 1631 1632 type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) | 1633 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 1634 1635 if (NET_IP_ALIGN == 2) 1636 type |= RX_BD_FLAGS_SOP; 1637 1638 ring = &rxr->rx_ring_struct; 1639 bnge_init_rxbd_pages(ring, type); 1640 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1641 } 1642 1643 static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn, 1644 struct bnge_rx_ring_info *rxr) 1645 { 1646 struct bnge_ring_struct *ring; 1647 u32 type; 1648 1649 ring = &rxr->rx_agg_ring_struct; 1650 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1651 if (bnge_is_agg_reqd(bn->bd)) { 1652 type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 1653 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 1654 1655 bnge_init_rxbd_pages(ring, type); 1656 } 1657 } 1658 1659 static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr) 1660 { 1661 struct bnge_rx_ring_info *rxr; 1662 1663 rxr = &bn->rx_ring[ring_nr]; 1664 bnge_init_one_rx_ring_rxbd(bn, rxr); 1665 1666 netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX, 1667 &rxr->bnapi->napi); 1668 1669 bnge_init_one_agg_ring_rxbd(bn, rxr); 1670 } 1671 1672 static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn) 1673 { 1674 int i, rc; 1675 1676 for (i = 0; i < bn->bd->rx_nr_rings; i++) { 1677 rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i); 1678 if (rc) 1679 goto err_free_rx_ring_pair_bufs; 1680 } 1681 return 0; 1682 1683 err_free_rx_ring_pair_bufs: 1684 bnge_free_rx_ring_pair_bufs(bn); 1685 return rc; 1686 } 1687 1688 static void bnge_init_rx_rings(struct bnge_net *bn) 1689 { 1690 int i; 1691 1692 #define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 1693 #define BNGE_RX_DMA_OFFSET NET_SKB_PAD 1694 bn->rx_offset = BNGE_RX_OFFSET; 1695 bn->rx_dma_offset = BNGE_RX_DMA_OFFSET; 1696 1697 for (i = 0; i < bn->bd->rx_nr_rings; i++) 1698 bnge_init_one_rx_ring_pair(bn, i); 1699 } 1700 1701 static void bnge_init_tx_rings(struct bnge_net *bn) 1702 { 1703 int i; 1704 1705 bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT); 1706 1707 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 1708 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1709 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1710 1711 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1712 1713 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, 1714 &txr->bnapi->napi); 1715 } 1716 } 1717 1718 static void bnge_init_vnics(struct bnge_net *bn) 1719 { 1720 struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 1721 int i; 1722 1723 for (i = 0; i < bn->nr_vnics; i++) { 1724 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 1725 int j; 1726 1727 vnic->fw_vnic_id = INVALID_HW_RING_ID; 1728 vnic->vnic_id = i; 1729 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) 1730 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 1731 1732 if (bn->vnic_info[i].rss_hash_key) { 1733 if (i == BNGE_VNIC_DEFAULT) { 1734 u8 *key = (void *)vnic->rss_hash_key; 1735 int k; 1736 1737 if (!bn->rss_hash_key_valid && 1738 !bn->rss_hash_key_updated) { 1739 get_random_bytes(bn->rss_hash_key, 1740 HW_HASH_KEY_SIZE); 1741 bn->rss_hash_key_updated = true; 1742 } 1743 1744 memcpy(vnic->rss_hash_key, bn->rss_hash_key, 1745 HW_HASH_KEY_SIZE); 1746 1747 if (!bn->rss_hash_key_updated) 1748 continue; 1749 1750 bn->rss_hash_key_updated = false; 1751 bn->rss_hash_key_valid = true; 1752 1753 bn->toeplitz_prefix = 0; 1754 for (k = 0; k < 8; k++) { 1755 bn->toeplitz_prefix <<= 8; 1756 bn->toeplitz_prefix |= key[k]; 1757 } 1758 } else { 1759 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 1760 HW_HASH_KEY_SIZE); 1761 } 1762 } 1763 } 1764 } 1765 1766 static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db, 1767 u32 ring_type) 1768 { 1769 switch (ring_type) { 1770 case HWRM_RING_ALLOC_TX: 1771 db->db_ring_mask = bn->tx_ring_mask; 1772 break; 1773 case HWRM_RING_ALLOC_RX: 1774 db->db_ring_mask = bn->rx_ring_mask; 1775 break; 1776 case HWRM_RING_ALLOC_AGG: 1777 db->db_ring_mask = bn->rx_agg_ring_mask; 1778 break; 1779 case HWRM_RING_ALLOC_CMPL: 1780 case HWRM_RING_ALLOC_NQ: 1781 db->db_ring_mask = bn->cp_ring_mask; 1782 break; 1783 } 1784 db->db_epoch_mask = db->db_ring_mask + 1; 1785 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 1786 } 1787 1788 static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db, 1789 u32 ring_type, u32 map_idx, u32 xid) 1790 { 1791 struct bnge_dev *bd = bn->bd; 1792 1793 switch (ring_type) { 1794 case HWRM_RING_ALLOC_TX: 1795 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 1796 break; 1797 case HWRM_RING_ALLOC_RX: 1798 case HWRM_RING_ALLOC_AGG: 1799 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 1800 break; 1801 case HWRM_RING_ALLOC_CMPL: 1802 db->db_key64 = DBR_PATH_L2; 1803 break; 1804 case HWRM_RING_ALLOC_NQ: 1805 db->db_key64 = DBR_PATH_L2; 1806 break; 1807 } 1808 db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID; 1809 1810 db->doorbell = bd->bar1 + bd->db_offset; 1811 bnge_set_db_mask(bn, db, ring_type); 1812 } 1813 1814 static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn, 1815 struct bnge_cp_ring_info *cpr) 1816 { 1817 const u32 type = HWRM_RING_ALLOC_CMPL; 1818 struct bnge_napi *bnapi = cpr->bnapi; 1819 struct bnge_ring_struct *ring; 1820 u32 map_idx = bnapi->index; 1821 int rc; 1822 1823 ring = &cpr->ring_struct; 1824 ring->handle = BNGE_SET_NQ_HDL(cpr); 1825 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1826 if (rc) 1827 return rc; 1828 1829 bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 1830 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1831 1832 return 0; 1833 } 1834 1835 static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn, 1836 struct bnge_tx_ring_info *txr, u32 tx_idx) 1837 { 1838 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1839 const u32 type = HWRM_RING_ALLOC_TX; 1840 int rc; 1841 1842 rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx); 1843 if (rc) 1844 return rc; 1845 1846 bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 1847 1848 return 0; 1849 } 1850 1851 static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn, 1852 struct bnge_rx_ring_info *rxr) 1853 { 1854 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 1855 u32 type = HWRM_RING_ALLOC_AGG; 1856 struct bnge_dev *bd = bn->bd; 1857 u32 grp_idx = ring->grp_idx; 1858 u32 map_idx; 1859 int rc; 1860 1861 map_idx = grp_idx + bd->rx_nr_rings; 1862 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1863 if (rc) 1864 return rc; 1865 1866 bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx, 1867 ring->fw_ring_id); 1868 bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod); 1869 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1870 bn->grp_info[grp_idx].agg_fw_ring_id = (u16)ring->fw_ring_id; 1871 1872 return 0; 1873 } 1874 1875 static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn, 1876 struct bnge_rx_ring_info *rxr) 1877 { 1878 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 1879 struct bnge_napi *bnapi = rxr->bnapi; 1880 u32 type = HWRM_RING_ALLOC_RX; 1881 u32 map_idx = bnapi->index; 1882 int rc; 1883 1884 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1885 if (rc) 1886 return rc; 1887 1888 bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 1889 bn->grp_info[map_idx].rx_fw_ring_id = (u16)ring->fw_ring_id; 1890 1891 return 0; 1892 } 1893 1894 static int bnge_hwrm_ring_alloc(struct bnge_net *bn) 1895 { 1896 struct bnge_dev *bd = bn->bd; 1897 bool agg_rings; 1898 int i, rc = 0; 1899 1900 agg_rings = !!(bnge_is_agg_reqd(bd)); 1901 for (i = 0; i < bd->nq_nr_rings; i++) { 1902 struct bnge_napi *bnapi = bn->bnapi[i]; 1903 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 1904 struct bnge_ring_struct *ring = &nqr->ring_struct; 1905 u32 type = HWRM_RING_ALLOC_NQ; 1906 u32 map_idx = ring->map_idx; 1907 unsigned int vector; 1908 1909 vector = bd->irq_tbl[map_idx].vector; 1910 disable_irq_nosync(vector); 1911 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1912 if (rc) { 1913 enable_irq(vector); 1914 goto err_out; 1915 } 1916 bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id); 1917 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 1918 enable_irq(vector); 1919 bn->grp_info[i].nq_fw_ring_id = (u16)ring->fw_ring_id; 1920 1921 if (!i) { 1922 rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id); 1923 if (rc) 1924 netdev_warn(bn->netdev, "Failed to set async event completion ring.\n"); 1925 } 1926 } 1927 1928 for (i = 0; i < bd->tx_nr_rings; i++) { 1929 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1930 1931 rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr); 1932 if (rc) 1933 goto err_out; 1934 rc = bnge_hwrm_tx_ring_alloc(bn, txr, i); 1935 if (rc) 1936 goto err_out; 1937 } 1938 1939 for (i = 0; i < bd->rx_nr_rings; i++) { 1940 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1941 struct bnge_cp_ring_info *cpr; 1942 struct bnge_ring_struct *ring; 1943 struct bnge_napi *bnapi; 1944 u32 map_idx, type; 1945 1946 rc = bnge_hwrm_rx_ring_alloc(bn, rxr); 1947 if (rc) 1948 goto err_out; 1949 /* If we have agg rings, post agg buffers first. */ 1950 if (!agg_rings) 1951 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1952 1953 cpr = rxr->rx_cpr; 1954 bnapi = rxr->bnapi; 1955 type = HWRM_RING_ALLOC_CMPL; 1956 map_idx = bnapi->index; 1957 1958 ring = &cpr->ring_struct; 1959 ring->handle = BNGE_SET_NQ_HDL(cpr); 1960 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1961 if (rc) 1962 goto err_out; 1963 bnge_set_db(bn, &cpr->cp_db, type, map_idx, 1964 ring->fw_ring_id); 1965 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1966 } 1967 1968 if (agg_rings) { 1969 for (i = 0; i < bd->rx_nr_rings; i++) { 1970 rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]); 1971 if (rc) 1972 goto err_out; 1973 } 1974 } 1975 err_out: 1976 return rc; 1977 } 1978 1979 void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic) 1980 { 1981 __le16 *ring_tbl = vnic->rss_table; 1982 struct bnge_rx_ring_info *rxr; 1983 struct bnge_dev *bd = bn->bd; 1984 u16 tbl_size, i; 1985 1986 tbl_size = bnge_get_rxfh_indir_size(bd); 1987 1988 for (i = 0; i < tbl_size; i++) { 1989 u32 j; 1990 1991 j = bd->rss_indir_tbl[i]; 1992 rxr = &bn->rx_ring[j]; 1993 1994 *ring_tbl++ = cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 1995 *ring_tbl++ = cpu_to_le16(bnge_cp_ring_for_rx(rxr)); 1996 } 1997 } 1998 1999 static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn, 2000 struct bnge_vnic_info *vnic) 2001 { 2002 int rc; 2003 2004 rc = bnge_hwrm_vnic_set_rss(bn, vnic, true); 2005 if (rc) { 2006 netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n", 2007 vnic->vnic_id, rc); 2008 return rc; 2009 } 2010 rc = bnge_hwrm_vnic_cfg(bn, vnic); 2011 if (rc) 2012 netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n", 2013 vnic->vnic_id, rc); 2014 return rc; 2015 } 2016 2017 static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic) 2018 { 2019 struct bnge_dev *bd = bn->bd; 2020 int rc, i, nr_ctxs; 2021 2022 nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings); 2023 for (i = 0; i < nr_ctxs; i++) { 2024 rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i); 2025 if (rc) { 2026 netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n", 2027 vnic->vnic_id, i, rc); 2028 return -ENOMEM; 2029 } 2030 bn->rsscos_nr_ctxs++; 2031 } 2032 2033 rc = bnge_hwrm_vnic_rss_cfg(bn, vnic); 2034 if (rc) 2035 return rc; 2036 2037 if (bnge_is_agg_reqd(bd)) { 2038 rc = bnge_hwrm_vnic_set_hds(bn, vnic); 2039 if (rc) 2040 netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n", 2041 vnic->vnic_id, rc); 2042 } 2043 return rc; 2044 } 2045 2046 static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr) 2047 { 2048 if (!refcount_dec_and_test(&fltr->refcnt)) 2049 return; 2050 hlist_del_rcu(&fltr->base.hash); 2051 kfree_rcu(fltr, base.rcu); 2052 } 2053 2054 static void bnge_init_l2_filter(struct bnge_net *bn, 2055 struct bnge_l2_filter *fltr, 2056 struct bnge_l2_key *key, u32 idx) 2057 { 2058 struct hlist_head *head; 2059 2060 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 2061 fltr->l2_key.vlan = key->vlan; 2062 fltr->base.type = BNGE_FLTR_TYPE_L2; 2063 2064 head = &bn->l2_fltr_hash_tbl[idx]; 2065 hlist_add_head_rcu(&fltr->base.hash, head); 2066 refcount_set(&fltr->refcnt, 1); 2067 } 2068 2069 static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn, 2070 struct bnge_l2_key *key, 2071 u32 idx) 2072 { 2073 struct bnge_l2_filter *fltr; 2074 struct hlist_head *head; 2075 2076 head = &bn->l2_fltr_hash_tbl[idx]; 2077 hlist_for_each_entry_rcu(fltr, head, base.hash) { 2078 struct bnge_l2_key *l2_key = &fltr->l2_key; 2079 2080 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 2081 l2_key->vlan == key->vlan) 2082 return fltr; 2083 } 2084 return NULL; 2085 } 2086 2087 static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn, 2088 struct bnge_l2_key *key, 2089 u32 idx) 2090 { 2091 struct bnge_l2_filter *fltr; 2092 2093 rcu_read_lock(); 2094 fltr = __bnge_lookup_l2_filter(bn, key, idx); 2095 if (fltr) 2096 refcount_inc(&fltr->refcnt); 2097 rcu_read_unlock(); 2098 return fltr; 2099 } 2100 2101 static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn, 2102 struct bnge_l2_key *key, 2103 gfp_t gfp) 2104 { 2105 struct bnge_l2_filter *fltr; 2106 u32 idx; 2107 2108 idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) & 2109 BNGE_L2_FLTR_HASH_MASK; 2110 fltr = bnge_lookup_l2_filter(bn, key, idx); 2111 if (fltr) 2112 return fltr; 2113 2114 fltr = kzalloc_obj(*fltr, gfp); 2115 if (!fltr) 2116 return ERR_PTR(-ENOMEM); 2117 2118 bnge_init_l2_filter(bn, fltr, key, idx); 2119 return fltr; 2120 } 2121 2122 static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx, 2123 const u8 *mac_addr) 2124 { 2125 struct bnge_l2_filter *fltr; 2126 struct bnge_l2_key key; 2127 int rc; 2128 2129 ether_addr_copy(key.dst_mac_addr, mac_addr); 2130 key.vlan = 0; 2131 fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL); 2132 if (IS_ERR(fltr)) 2133 return PTR_ERR(fltr); 2134 2135 fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id; 2136 rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr); 2137 if (rc) 2138 goto err_del_l2_filter; 2139 bn->vnic_info[vnic_id].l2_filters[idx] = fltr; 2140 return rc; 2141 2142 err_del_l2_filter: 2143 bnge_del_l2_filter(bn, fltr); 2144 return rc; 2145 } 2146 2147 static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask) 2148 { 2149 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2150 struct net_device *dev = bn->netdev; 2151 struct netdev_hw_addr *ha; 2152 int mc_count = 0, off = 0; 2153 bool update = false; 2154 u8 *haddr; 2155 2156 netdev_for_each_mc_addr(ha, dev) { 2157 if (mc_count >= BNGE_MAX_MC_ADDRS) { 2158 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2159 vnic->mc_list_count = 0; 2160 return false; 2161 } 2162 haddr = ha->addr; 2163 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 2164 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 2165 update = true; 2166 } 2167 off += ETH_ALEN; 2168 mc_count++; 2169 } 2170 if (mc_count) 2171 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2172 2173 if (mc_count != vnic->mc_list_count) { 2174 vnic->mc_list_count = mc_count; 2175 update = true; 2176 } 2177 return update; 2178 } 2179 2180 static bool bnge_uc_list_updated(struct bnge_net *bn) 2181 { 2182 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2183 struct net_device *dev = bn->netdev; 2184 struct netdev_hw_addr *ha; 2185 int off = 0; 2186 2187 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 2188 return true; 2189 2190 netdev_for_each_uc_addr(ha, dev) { 2191 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 2192 return true; 2193 2194 off += ETH_ALEN; 2195 } 2196 return false; 2197 } 2198 2199 static bool bnge_promisc_ok(struct bnge_net *bn) 2200 { 2201 return true; 2202 } 2203 2204 static int bnge_cfg_def_vnic(struct bnge_net *bn) 2205 { 2206 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2207 struct net_device *dev = bn->netdev; 2208 struct bnge_dev *bd = bn->bd; 2209 struct netdev_hw_addr *ha; 2210 int i, off = 0, rc; 2211 bool uc_update; 2212 2213 netif_addr_lock_bh(dev); 2214 uc_update = bnge_uc_list_updated(bn); 2215 netif_addr_unlock_bh(dev); 2216 2217 if (!uc_update) 2218 goto skip_uc; 2219 2220 for (i = 1; i < vnic->uc_filter_count; i++) { 2221 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2222 2223 bnge_hwrm_l2_filter_free(bd, fltr); 2224 bnge_del_l2_filter(bn, fltr); 2225 } 2226 2227 vnic->uc_filter_count = 1; 2228 2229 netif_addr_lock_bh(dev); 2230 if (netdev_uc_count(dev) > (BNGE_MAX_UC_ADDRS - 1)) { 2231 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2232 } else { 2233 netdev_for_each_uc_addr(ha, dev) { 2234 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 2235 off += ETH_ALEN; 2236 vnic->uc_filter_count++; 2237 } 2238 } 2239 netif_addr_unlock_bh(dev); 2240 2241 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 2242 rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off); 2243 if (rc) { 2244 netdev_err(dev, "HWRM vnic filter failure rc: %d\n", rc); 2245 vnic->uc_filter_count = i; 2246 return rc; 2247 } 2248 } 2249 2250 skip_uc: 2251 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 2252 !bnge_promisc_ok(bn)) 2253 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2254 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2255 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 2256 netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 2257 rc); 2258 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2259 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2260 vnic->mc_list_count = 0; 2261 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2262 } 2263 if (rc) 2264 netdev_err(dev, "HWRM cfa l2 rx mask failure rc: %d\n", 2265 rc); 2266 2267 return rc; 2268 } 2269 2270 static void bnge_disable_int(struct bnge_net *bn) 2271 { 2272 struct bnge_dev *bd = bn->bd; 2273 int i; 2274 2275 if (!bn->bnapi) 2276 return; 2277 2278 for (i = 0; i < bd->nq_nr_rings; i++) { 2279 struct bnge_napi *bnapi = bn->bnapi[i]; 2280 struct bnge_nq_ring_info *nqr; 2281 struct bnge_ring_struct *ring; 2282 2283 nqr = &bnapi->nq_ring; 2284 ring = &nqr->ring_struct; 2285 2286 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) 2287 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 2288 } 2289 } 2290 2291 static void bnge_disable_int_sync(struct bnge_net *bn) 2292 { 2293 struct bnge_dev *bd = bn->bd; 2294 int i; 2295 2296 bnge_disable_int(bn); 2297 for (i = 0; i < bd->nq_nr_rings; i++) { 2298 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2299 2300 synchronize_irq(bd->irq_tbl[map_idx].vector); 2301 } 2302 } 2303 2304 static void bnge_enable_int(struct bnge_net *bn) 2305 { 2306 struct bnge_dev *bd = bn->bd; 2307 int i; 2308 2309 for (i = 0; i < bd->nq_nr_rings; i++) { 2310 struct bnge_napi *bnapi = bn->bnapi[i]; 2311 struct bnge_nq_ring_info *nqr; 2312 2313 nqr = &bnapi->nq_ring; 2314 bnge_db_nq_arm(bn, &nqr->nq_db, nqr->nq_raw_cons); 2315 } 2316 } 2317 2318 static void bnge_disable_napi(struct bnge_net *bn) 2319 { 2320 struct bnge_dev *bd = bn->bd; 2321 int i; 2322 2323 if (test_and_set_bit(BNGE_STATE_NAPI_DISABLED, &bn->state)) 2324 return; 2325 2326 for (i = 0; i < bd->nq_nr_rings; i++) { 2327 struct bnge_napi *bnapi = bn->bnapi[i]; 2328 2329 napi_disable_locked(&bnapi->napi); 2330 } 2331 } 2332 2333 static void bnge_enable_napi(struct bnge_net *bn) 2334 { 2335 struct bnge_dev *bd = bn->bd; 2336 int i; 2337 2338 clear_bit(BNGE_STATE_NAPI_DISABLED, &bn->state); 2339 for (i = 0; i < bd->nq_nr_rings; i++) { 2340 struct bnge_napi *bnapi = bn->bnapi[i]; 2341 2342 bnapi->in_reset = false; 2343 bnapi->tx_fault = 0; 2344 2345 napi_enable_locked(&bnapi->napi); 2346 } 2347 } 2348 2349 static void bnge_hwrm_vnic_free(struct bnge_net *bn) 2350 { 2351 int i; 2352 2353 for (i = 0; i < bn->nr_vnics; i++) 2354 bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]); 2355 } 2356 2357 static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn) 2358 { 2359 int i, j; 2360 2361 for (i = 0; i < bn->nr_vnics; i++) { 2362 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 2363 2364 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) { 2365 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 2366 bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j); 2367 } 2368 } 2369 bn->rsscos_nr_ctxs = 0; 2370 } 2371 2372 static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn) 2373 { 2374 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2375 int i; 2376 2377 for (i = 0; i < vnic->uc_filter_count; i++) { 2378 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2379 2380 bnge_hwrm_l2_filter_free(bn->bd, fltr); 2381 bnge_del_l2_filter(bn, fltr); 2382 } 2383 2384 vnic->uc_filter_count = 0; 2385 } 2386 2387 static void bnge_clear_vnic(struct bnge_net *bn) 2388 { 2389 bnge_hwrm_clear_vnic_filter(bn); 2390 bnge_hwrm_vnic_free(bn); 2391 bnge_hwrm_vnic_ctx_free(bn); 2392 } 2393 2394 static void bnge_hwrm_rx_ring_free(struct bnge_net *bn, 2395 struct bnge_rx_ring_info *rxr, 2396 bool close_path) 2397 { 2398 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 2399 u32 grp_idx = rxr->bnapi->index; 2400 u32 cmpl_ring_id; 2401 2402 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2403 return; 2404 2405 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2406 hwrm_ring_free_send_msg(bn, ring, 2407 RING_FREE_REQ_RING_TYPE_RX, 2408 close_path ? cmpl_ring_id : 2409 INVALID_HW_RING_ID); 2410 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2411 bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 2412 } 2413 2414 static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn, 2415 struct bnge_rx_ring_info *rxr, 2416 bool close_path) 2417 { 2418 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 2419 u32 grp_idx = rxr->bnapi->index; 2420 u32 cmpl_ring_id; 2421 2422 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2423 return; 2424 2425 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2426 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG, 2427 close_path ? cmpl_ring_id : 2428 INVALID_HW_RING_ID); 2429 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2430 bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 2431 } 2432 2433 static void bnge_hwrm_tx_ring_free(struct bnge_net *bn, 2434 struct bnge_tx_ring_info *txr, 2435 bool close_path) 2436 { 2437 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 2438 u32 cmpl_ring_id; 2439 2440 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2441 return; 2442 2443 cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) : 2444 INVALID_HW_RING_ID; 2445 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX, 2446 cmpl_ring_id); 2447 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2448 } 2449 2450 static void bnge_hwrm_cp_ring_free(struct bnge_net *bn, 2451 struct bnge_cp_ring_info *cpr) 2452 { 2453 struct bnge_ring_struct *ring; 2454 2455 ring = &cpr->ring_struct; 2456 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2457 return; 2458 2459 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 2460 INVALID_HW_RING_ID); 2461 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2462 } 2463 2464 static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path) 2465 { 2466 struct bnge_dev *bd = bn->bd; 2467 int i; 2468 2469 if (!bn->bnapi) 2470 return; 2471 2472 for (i = 0; i < bd->tx_nr_rings; i++) 2473 bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path); 2474 2475 for (i = 0; i < bd->rx_nr_rings; i++) { 2476 bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path); 2477 bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path); 2478 } 2479 2480 /* The completion rings are about to be freed. After that the 2481 * IRQ doorbell will not work anymore. So we need to disable 2482 * IRQ here. 2483 */ 2484 bnge_disable_int_sync(bn); 2485 2486 for (i = 0; i < bd->nq_nr_rings; i++) { 2487 struct bnge_napi *bnapi = bn->bnapi[i]; 2488 struct bnge_nq_ring_info *nqr; 2489 struct bnge_ring_struct *ring; 2490 int j; 2491 2492 nqr = &bnapi->nq_ring; 2493 for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++) 2494 bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]); 2495 2496 ring = &nqr->ring_struct; 2497 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) { 2498 hwrm_ring_free_send_msg(bn, ring, 2499 RING_FREE_REQ_RING_TYPE_NQ, 2500 INVALID_HW_RING_ID); 2501 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2502 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 2503 } 2504 } 2505 } 2506 2507 static void bnge_setup_msix(struct bnge_net *bn) 2508 { 2509 struct net_device *dev = bn->netdev; 2510 struct bnge_dev *bd = bn->bd; 2511 int len, i; 2512 2513 len = sizeof(bd->irq_tbl[0].name); 2514 for (i = 0; i < bd->nq_nr_rings; i++) { 2515 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2516 char *attr; 2517 2518 if (bd->flags & BNGE_EN_SHARED_CHNL) 2519 attr = "TxRx"; 2520 else if (i < bd->rx_nr_rings) 2521 attr = "rx"; 2522 else 2523 attr = "tx"; 2524 2525 snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 2526 attr, i); 2527 bd->irq_tbl[map_idx].handler = bnge_msix; 2528 } 2529 } 2530 2531 static int bnge_setup_interrupts(struct bnge_net *bn) 2532 { 2533 struct net_device *dev = bn->netdev; 2534 struct bnge_dev *bd = bn->bd; 2535 2536 bnge_setup_msix(bn); 2537 2538 return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings); 2539 } 2540 2541 static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path) 2542 { 2543 bnge_clear_vnic(bn); 2544 bnge_hwrm_ring_free(bn, close_path); 2545 bnge_hwrm_stat_ctx_free(bn); 2546 } 2547 2548 static void bnge_free_irq(struct bnge_net *bn) 2549 { 2550 struct bnge_dev *bd = bn->bd; 2551 struct bnge_irq *irq; 2552 int i; 2553 2554 for (i = 0; i < bd->nq_nr_rings; i++) { 2555 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2556 2557 irq = &bd->irq_tbl[map_idx]; 2558 if (irq->requested) { 2559 if (irq->have_cpumask) { 2560 irq_set_affinity_hint(irq->vector, NULL); 2561 free_cpumask_var(irq->cpu_mask); 2562 irq->have_cpumask = 0; 2563 } 2564 free_irq(irq->vector, bn->bnapi[i]); 2565 } 2566 2567 irq->requested = 0; 2568 } 2569 } 2570 2571 static int bnge_request_irq(struct bnge_net *bn) 2572 { 2573 struct bnge_dev *bd = bn->bd; 2574 int i, rc; 2575 2576 rc = bnge_setup_interrupts(bn); 2577 if (rc) { 2578 netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc); 2579 return rc; 2580 } 2581 for (i = 0; i < bd->nq_nr_rings; i++) { 2582 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2583 struct bnge_irq *irq = &bd->irq_tbl[map_idx]; 2584 2585 rc = request_irq(irq->vector, irq->handler, 0, irq->name, 2586 bn->bnapi[i]); 2587 if (rc) 2588 goto err_free_irq; 2589 2590 netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector); 2591 irq->requested = 1; 2592 2593 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 2594 int numa_node = dev_to_node(&bd->pdev->dev); 2595 2596 irq->have_cpumask = 1; 2597 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2598 irq->cpu_mask); 2599 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 2600 if (rc) { 2601 netdev_warn(bn->netdev, 2602 "Set affinity failed, IRQ = %d\n", 2603 irq->vector); 2604 goto err_free_irq; 2605 } 2606 } 2607 } 2608 return 0; 2609 2610 err_free_irq: 2611 bnge_free_irq(bn); 2612 return rc; 2613 } 2614 2615 static int bnge_set_tpa(struct bnge_net *bn, bool set_tpa) 2616 { 2617 u32 tpa_flags = 0; 2618 int rc, i; 2619 2620 if (set_tpa) 2621 tpa_flags = bn->priv_flags & BNGE_NET_EN_TPA; 2622 else if (BNGE_NO_FW_ACCESS(bn->bd)) 2623 return 0; 2624 for (i = 0; i < bn->nr_vnics; i++) { 2625 rc = bnge_hwrm_vnic_set_tpa(bn->bd, &bn->vnic_info[i], 2626 tpa_flags); 2627 if (rc) { 2628 netdev_err(bn->netdev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 2629 i, rc); 2630 return rc; 2631 } 2632 } 2633 return 0; 2634 } 2635 2636 static int bnge_init_chip(struct bnge_net *bn) 2637 { 2638 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2639 struct bnge_dev *bd = bn->bd; 2640 int rc; 2641 2642 #define BNGE_DEF_STATS_COAL_TICKS 1000000 2643 bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS; 2644 2645 rc = bnge_hwrm_stat_ctx_alloc(bn); 2646 if (rc) { 2647 netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc); 2648 goto err_out; 2649 } 2650 2651 rc = bnge_hwrm_ring_alloc(bn); 2652 if (rc) { 2653 netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc); 2654 goto err_out; 2655 } 2656 2657 rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings); 2658 if (rc) { 2659 netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc); 2660 goto err_out; 2661 } 2662 2663 rc = bnge_setup_vnic(bn, vnic); 2664 if (rc) 2665 goto err_out; 2666 2667 if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA) 2668 bnge_hwrm_update_rss_hash_cfg(bn); 2669 2670 if (bn->priv_flags & BNGE_NET_EN_TPA) { 2671 rc = bnge_set_tpa(bn, true); 2672 if (rc) 2673 goto err_out; 2674 } 2675 2676 /* Filter for default vnic 0 */ 2677 rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr); 2678 if (rc) { 2679 netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc); 2680 goto err_out; 2681 } 2682 vnic->uc_filter_count = 1; 2683 2684 vnic->rx_mask = 0; 2685 2686 if (bn->netdev->flags & IFF_BROADCAST) 2687 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2688 2689 if (bn->netdev->flags & IFF_PROMISC) 2690 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2691 2692 if (bn->netdev->flags & IFF_ALLMULTI) { 2693 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2694 vnic->mc_list_count = 0; 2695 } else if (bn->netdev->flags & IFF_MULTICAST) { 2696 u32 mask = 0; 2697 2698 bnge_mc_list_updated(bn, &mask); 2699 vnic->rx_mask |= mask; 2700 } 2701 2702 rc = bnge_cfg_def_vnic(bn); 2703 if (rc) 2704 goto err_out; 2705 return 0; 2706 2707 err_out: 2708 bnge_hwrm_resource_free(bn, 0); 2709 return rc; 2710 } 2711 2712 static void bnge_init_napi(struct bnge_net *bn) 2713 { 2714 struct bnge_dev *bd = bn->bd; 2715 struct bnge_napi *bnapi; 2716 int i; 2717 2718 for (i = 0; i < bd->nq_nr_rings; i++) { 2719 bnapi = bn->bnapi[i]; 2720 netif_napi_add_config_locked(bn->netdev, &bnapi->napi, 2721 bnge_napi_poll, bnapi->index); 2722 } 2723 } 2724 2725 static void bnge_del_napi(struct bnge_net *bn) 2726 { 2727 struct bnge_dev *bd = bn->bd; 2728 int i; 2729 2730 for (i = 0; i < bd->rx_nr_rings; i++) 2731 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL); 2732 for (i = 0; i < bd->tx_nr_rings; i++) 2733 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL); 2734 2735 for (i = 0; i < bd->nq_nr_rings; i++) { 2736 struct bnge_napi *bnapi = bn->bnapi[i]; 2737 2738 __netif_napi_del_locked(&bnapi->napi); 2739 } 2740 2741 /* Wait for RCU grace period after removing NAPI instances */ 2742 synchronize_net(); 2743 } 2744 2745 static int bnge_init_nic(struct bnge_net *bn) 2746 { 2747 int rc; 2748 2749 bnge_init_nq_tree(bn); 2750 2751 bnge_init_rx_rings(bn); 2752 rc = bnge_alloc_rx_ring_pair_bufs(bn); 2753 if (rc) 2754 return rc; 2755 2756 bnge_init_tx_rings(bn); 2757 2758 rc = bnge_init_ring_grps(bn); 2759 if (rc) 2760 goto err_free_rx_ring_pair_bufs; 2761 2762 bnge_init_vnics(bn); 2763 2764 rc = bnge_init_chip(bn); 2765 if (rc) 2766 goto err_free_ring_grps; 2767 return rc; 2768 2769 err_free_ring_grps: 2770 bnge_free_ring_grps(bn); 2771 err_free_rx_ring_pair_bufs: 2772 bnge_free_rx_ring_pair_bufs(bn); 2773 return rc; 2774 } 2775 2776 static void bnge_tx_disable(struct bnge_net *bn) 2777 { 2778 struct bnge_tx_ring_info *txr; 2779 int i; 2780 2781 if (bn->tx_ring) { 2782 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2783 txr = &bn->tx_ring[i]; 2784 WRITE_ONCE(txr->dev_state, BNGE_DEV_STATE_CLOSING); 2785 } 2786 } 2787 /* Make sure napi polls see @dev_state change */ 2788 synchronize_net(); 2789 2790 if (!bn->netdev) 2791 return; 2792 /* Drop carrier first to prevent TX timeout */ 2793 netif_carrier_off(bn->netdev); 2794 /* Stop all TX queues */ 2795 netif_tx_disable(bn->netdev); 2796 } 2797 2798 static void bnge_tx_enable(struct bnge_net *bn) 2799 { 2800 struct bnge_tx_ring_info *txr; 2801 int i; 2802 2803 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2804 txr = &bn->tx_ring[i]; 2805 WRITE_ONCE(txr->dev_state, 0); 2806 } 2807 /* Make sure napi polls see @dev_state change */ 2808 synchronize_net(); 2809 netif_tx_wake_all_queues(bn->netdev); 2810 if (BNGE_LINK_IS_UP(bn->bd)) 2811 netif_carrier_on(bn->netdev); 2812 } 2813 2814 static int bnge_open_core(struct bnge_net *bn) 2815 { 2816 struct bnge_dev *bd = bn->bd; 2817 int rc; 2818 2819 netif_carrier_off(bn->netdev); 2820 2821 rc = bnge_reserve_rings(bd); 2822 if (rc) { 2823 netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc); 2824 return rc; 2825 } 2826 2827 rc = bnge_alloc_core(bn); 2828 if (rc) { 2829 netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc); 2830 return rc; 2831 } 2832 2833 bnge_init_napi(bn); 2834 rc = bnge_request_irq(bn); 2835 if (rc) { 2836 netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc); 2837 goto err_del_napi; 2838 } 2839 2840 rc = bnge_init_nic(bn); 2841 if (rc) { 2842 netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc); 2843 goto err_free_irq; 2844 } 2845 2846 bnge_enable_napi(bn); 2847 2848 rc = bnge_update_phy_setting(bn); 2849 if (rc) { 2850 netdev_warn(bn->netdev, "failed to update PHY settings (rc: %d)\n", 2851 rc); 2852 WRITE_ONCE(bd->link_info.phy_retry_expires, jiffies + 5 * HZ); 2853 WRITE_ONCE(bd->link_info.phy_retry, true); 2854 } else { 2855 WRITE_ONCE(bd->link_info.phy_retry, false); 2856 } 2857 2858 set_bit(BNGE_STATE_OPEN, &bd->state); 2859 set_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 2860 2861 bnge_enable_int(bn); 2862 2863 bnge_tx_enable(bn); 2864 2865 mod_timer(&bn->timer, jiffies + bn->current_interval); 2866 2867 /* Poll link status and check for SFP+ module status */ 2868 bnge_get_port_module_status(bn); 2869 2870 return 0; 2871 2872 err_free_irq: 2873 bnge_free_irq(bn); 2874 err_del_napi: 2875 bnge_del_napi(bn); 2876 bnge_free_core(bn); 2877 return rc; 2878 } 2879 2880 static int bnge_open(struct net_device *dev) 2881 { 2882 struct bnge_net *bn = netdev_priv(dev); 2883 int rc; 2884 2885 rc = bnge_open_core(bn); 2886 if (rc) 2887 netdev_err(dev, "bnge_open_core err: %d\n", rc); 2888 2889 return rc; 2890 } 2891 2892 static int bnge_shutdown_nic(struct bnge_net *bn) 2893 { 2894 bnge_hwrm_resource_free(bn, 1); 2895 return 0; 2896 } 2897 2898 static void bnge_add_prev_ring_stats64(struct bnge_net *bn, 2899 struct rtnl_link_stats64 *stats) 2900 { 2901 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 2902 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 2903 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 2904 2905 stats->rx_packets += rx_save->packets; 2906 stats->tx_packets += tx_save->packets; 2907 stats->rx_bytes += rx_save->bytes; 2908 stats->tx_bytes += tx_save->bytes; 2909 stats->rx_missed_errors += rx_save->hw_drop_overruns; 2910 stats->tx_dropped += tx_save->hw_drop_errors; 2911 2912 stats->multicast += stats64_save->multicast; 2913 } 2914 2915 static void bnge_get_ring_stats64(struct bnge_dev *bd, 2916 struct rtnl_link_stats64 *stats) 2917 { 2918 struct bnge_net *bn = netdev_priv(bd->netdev); 2919 int i; 2920 2921 for (i = 0; i < bd->nq_nr_rings; i++) { 2922 struct bnge_napi *bnapi = bn->bnapi[i]; 2923 u64 tx_bytes, tx_packets, tx_dropped; 2924 u64 multicast, rx_missed_errors; 2925 struct bnge_nq_ring_info *nqr; 2926 u64 rx_bytes, rx_packets; 2927 unsigned int start; 2928 u64 *sw; 2929 2930 nqr = &bnapi->nq_ring; 2931 sw = nqr->stats.sw_stats; 2932 2933 do { 2934 start = u64_stats_fetch_begin(&nqr->stats.syncp); 2935 2936 rx_packets = BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 2937 rx_packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 2938 rx_packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 2939 2940 tx_packets = BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 2941 tx_packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 2942 tx_packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 2943 2944 rx_bytes = BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 2945 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 2946 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 2947 2948 tx_bytes = BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 2949 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 2950 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 2951 2952 multicast = BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 2953 rx_missed_errors = 2954 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 2955 tx_dropped = 2956 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 2957 } while (u64_stats_fetch_retry(&nqr->stats.syncp, start)); 2958 2959 stats->rx_packets += rx_packets; 2960 stats->tx_packets += tx_packets; 2961 stats->rx_bytes += rx_bytes; 2962 stats->tx_bytes += tx_bytes; 2963 stats->multicast += multicast; 2964 stats->rx_missed_errors += rx_missed_errors; 2965 stats->tx_dropped += tx_dropped; 2966 } 2967 } 2968 2969 static void bnge_get_port_stats64(struct bnge_net *bn, 2970 struct rtnl_link_stats64 *stats) 2971 { 2972 unsigned int start; 2973 u64 *tx, *rx; 2974 2975 rx = bn->port_stats.sw_stats; 2976 tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 2977 2978 do { 2979 start = u64_stats_fetch_begin(&bn->port_stats.syncp); 2980 2981 stats->rx_crc_errors = 2982 BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 2983 stats->rx_frame_errors = 2984 BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 2985 stats->rx_length_errors = 2986 BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 2987 BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 2988 BNGE_GET_RX_PORT_STATS64(rx, rx_runt_frames); 2989 stats->rx_errors = 2990 BNGE_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 2991 BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 2992 stats->collisions = 2993 BNGE_GET_TX_PORT_STATS64(tx, tx_total_collisions); 2994 stats->tx_fifo_errors = 2995 BNGE_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 2996 stats->tx_errors = BNGE_GET_TX_PORT_STATS64(tx, tx_err); 2997 } while (u64_stats_fetch_retry(&bn->port_stats.syncp, start)); 2998 } 2999 3000 static void bnge_fill_prev_stats64(struct bnge_net *bn, 3001 struct rtnl_link_stats64 *stats) 3002 { 3003 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3004 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3005 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3006 3007 stats->rx_packets = rx_save->packets; 3008 stats->tx_packets = tx_save->packets; 3009 stats->rx_bytes = rx_save->bytes; 3010 stats->tx_bytes = tx_save->bytes; 3011 stats->rx_missed_errors = rx_save->hw_drop_overruns; 3012 stats->tx_dropped = tx_save->hw_drop_errors; 3013 stats->multicast = stats64_save->multicast; 3014 } 3015 3016 static void bnge_get_stats64(struct net_device *dev, 3017 struct rtnl_link_stats64 *stats) 3018 { 3019 struct bnge_net *bn = netdev_priv(dev); 3020 3021 if (bn->flags & BNGE_FLAG_PORT_STATS) 3022 bnge_get_port_stats64(bn, stats); 3023 3024 spin_lock_bh(&bn->stats_lock); 3025 if (!test_bit(BNGE_STATE_STATS_ENABLE, &bn->state)) { 3026 bnge_fill_prev_stats64(bn, stats); 3027 spin_unlock_bh(&bn->stats_lock); 3028 return; 3029 } 3030 3031 bnge_get_ring_stats64(bn->bd, stats); 3032 bnge_add_prev_ring_stats64(bn, stats); 3033 spin_unlock_bh(&bn->stats_lock); 3034 } 3035 3036 static void bnge_save_ring_stats(struct bnge_net *bn) 3037 { 3038 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3039 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3040 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3041 int i; 3042 3043 for (i = 0; i < bn->bd->nq_nr_rings; i++) { 3044 struct bnge_napi *bnapi = bn->bnapi[i]; 3045 struct bnge_nq_ring_info *nqr; 3046 u64 *sw; 3047 3048 nqr = &bnapi->nq_ring; 3049 sw = nqr->stats.sw_stats; 3050 3051 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3052 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3053 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3054 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3055 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3056 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3057 rx_save->hw_drop_overruns += BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3058 rx_save->hw_drops += BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3059 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3060 3061 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3062 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3063 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3064 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3065 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3066 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3067 tx_save->hw_drop_errors += BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3068 tx_save->hw_drops += BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3069 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3070 3071 stats64_save->multicast += 3072 BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3073 } 3074 } 3075 3076 static void bnge_close_core(struct bnge_net *bn) 3077 { 3078 struct bnge_dev *bd = bn->bd; 3079 3080 bnge_tx_disable(bn); 3081 3082 clear_bit(BNGE_STATE_OPEN, &bd->state); 3083 3084 timer_delete_sync(&bn->timer); 3085 bnge_shutdown_nic(bn); 3086 bnge_disable_napi(bn); 3087 3088 /* Save ring stats before shutdown */ 3089 spin_lock_bh(&bn->stats_lock); 3090 bnge_save_ring_stats(bn); 3091 clear_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 3092 spin_unlock_bh(&bn->stats_lock); 3093 3094 bnge_free_all_rings_bufs(bn); 3095 bnge_free_irq(bn); 3096 bnge_del_napi(bn); 3097 3098 bnge_free_core(bn); 3099 } 3100 3101 static int bnge_close(struct net_device *dev) 3102 { 3103 struct bnge_net *bn = netdev_priv(dev); 3104 3105 bnge_close_core(bn); 3106 bnge_hwrm_shutdown_link(bn->bd); 3107 bn->sp_event = 0; 3108 3109 return 0; 3110 } 3111 3112 static void bnge_get_queue_stats_rx(struct net_device *dev, int i, 3113 struct netdev_queue_stats_rx *stats) 3114 { 3115 struct bnge_net *bn = netdev_priv(dev); 3116 struct bnge_nq_ring_info *nqr; 3117 u64 *sw; 3118 3119 if (!bn->bnapi) 3120 return; 3121 3122 nqr = &bn->bnapi[i]->nq_ring; 3123 sw = nqr->stats.sw_stats; 3124 3125 stats->packets = 0; 3126 stats->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3127 stats->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3128 stats->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3129 3130 stats->bytes = 0; 3131 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3132 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3133 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3134 3135 stats->hw_drop_overruns = BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3136 stats->hw_drops = BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3137 stats->hw_drop_overruns; 3138 } 3139 3140 static void bnge_get_queue_stats_tx(struct net_device *dev, int i, 3141 struct netdev_queue_stats_tx *stats) 3142 { 3143 struct bnge_net *bn = netdev_priv(dev); 3144 struct bnge_napi *bnapi; 3145 u64 *sw; 3146 3147 if (!bn->tx_ring) 3148 return; 3149 3150 bnapi = bn->tx_ring[bn->tx_ring_map[i]].bnapi; 3151 sw = bnapi->nq_ring.stats.sw_stats; 3152 3153 stats->packets = 0; 3154 stats->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3155 stats->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3156 stats->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3157 3158 stats->bytes = 0; 3159 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3160 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3161 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3162 3163 stats->hw_drop_errors = BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3164 stats->hw_drops = BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3165 stats->hw_drop_errors; 3166 } 3167 3168 static void bnge_get_base_stats(struct net_device *dev, 3169 struct netdev_queue_stats_rx *rx, 3170 struct netdev_queue_stats_tx *tx) 3171 { 3172 struct bnge_net *bn = netdev_priv(dev); 3173 3174 rx->packets = bn->rxq_prv_stats.packets; 3175 rx->bytes = bn->rxq_prv_stats.bytes; 3176 rx->hw_drops = bn->rxq_prv_stats.hw_drops; 3177 rx->hw_drop_overruns = bn->rxq_prv_stats.hw_drop_overruns; 3178 3179 tx->packets = bn->txq_prv_stats.packets; 3180 tx->bytes = bn->txq_prv_stats.bytes; 3181 tx->hw_drops = bn->txq_prv_stats.hw_drops; 3182 tx->hw_drop_errors = bn->txq_prv_stats.hw_drop_errors; 3183 } 3184 3185 static const struct netdev_stat_ops bnge_stat_ops = { 3186 .get_queue_stats_rx = bnge_get_queue_stats_rx, 3187 .get_queue_stats_tx = bnge_get_queue_stats_tx, 3188 .get_base_stats = bnge_get_base_stats, 3189 }; 3190 3191 static const struct net_device_ops bnge_netdev_ops = { 3192 .ndo_open = bnge_open, 3193 .ndo_stop = bnge_close, 3194 .ndo_start_xmit = bnge_start_xmit, 3195 .ndo_get_stats64 = bnge_get_stats64, 3196 .ndo_features_check = bnge_features_check, 3197 }; 3198 3199 static void bnge_init_mac_addr(struct bnge_dev *bd) 3200 { 3201 eth_hw_addr_set(bd->netdev, bd->pf.mac_addr); 3202 } 3203 3204 static void bnge_set_tpa_flags(struct bnge_dev *bd) 3205 { 3206 struct bnge_net *bn = netdev_priv(bd->netdev); 3207 3208 bn->priv_flags &= ~BNGE_NET_EN_TPA; 3209 3210 if (bd->netdev->features & NETIF_F_LRO) 3211 bn->priv_flags |= BNGE_NET_EN_LRO; 3212 else if (bd->netdev->features & NETIF_F_GRO_HW) 3213 bn->priv_flags |= BNGE_NET_EN_GRO; 3214 } 3215 3216 static void bnge_init_l2_fltr_tbl(struct bnge_net *bn) 3217 { 3218 int i; 3219 3220 for (i = 0; i < BNGE_L2_FLTR_HASH_SIZE; i++) 3221 INIT_HLIST_HEAD(&bn->l2_fltr_hash_tbl[i]); 3222 get_random_bytes(&bn->hash_seed, sizeof(bn->hash_seed)); 3223 } 3224 3225 void bnge_set_ring_params(struct bnge_dev *bd) 3226 { 3227 struct bnge_net *bn = netdev_priv(bd->netdev); 3228 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3229 u32 agg_factor = 0, agg_ring_size = 0; 3230 3231 /* 8 for CRC and VLAN */ 3232 rx_size = SKB_DATA_ALIGN(bn->netdev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3233 3234 rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) + 3235 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3236 3237 ring_size = bn->rx_ring_size; 3238 bn->rx_agg_ring_size = 0; 3239 bn->rx_agg_nr_pages = 0; 3240 3241 if (bn->priv_flags & BNGE_NET_EN_TPA) 3242 agg_factor = min_t(u32, 4, 65536 / BNGE_RX_PAGE_SIZE); 3243 3244 bn->priv_flags &= ~BNGE_NET_EN_JUMBO; 3245 if (rx_space > PAGE_SIZE) { 3246 u32 jumbo_factor; 3247 3248 bn->priv_flags |= BNGE_NET_EN_JUMBO; 3249 jumbo_factor = PAGE_ALIGN(bn->netdev->mtu - 40) >> PAGE_SHIFT; 3250 if (jumbo_factor > agg_factor) 3251 agg_factor = jumbo_factor; 3252 } 3253 if (agg_factor) { 3254 if (ring_size > BNGE_MAX_RX_DESC_CNT_JUM_ENA) { 3255 ring_size = BNGE_MAX_RX_DESC_CNT_JUM_ENA; 3256 netdev_warn(bn->netdev, "RX ring size reduced from %d to %d due to jumbo ring\n", 3257 bn->rx_ring_size, ring_size); 3258 bn->rx_ring_size = ring_size; 3259 } 3260 agg_ring_size = ring_size * agg_factor; 3261 3262 bn->rx_agg_nr_pages = bnge_adjust_pow_two(agg_ring_size, 3263 RX_DESC_CNT); 3264 if (bn->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3265 u32 tmp = agg_ring_size; 3266 3267 bn->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3268 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3269 netdev_warn(bn->netdev, "RX agg ring size %d reduced to %d.\n", 3270 tmp, agg_ring_size); 3271 } 3272 bn->rx_agg_ring_size = agg_ring_size; 3273 bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3274 3275 rx_size = max3(BNGE_DEFAULT_RX_COPYBREAK, 3276 bn->rx_copybreak, 3277 bn->netdev->cfg_pending->hds_thresh); 3278 rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); 3279 rx_space = rx_size + NET_SKB_PAD + 3280 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3281 } 3282 3283 bn->rx_buf_use_size = rx_size; 3284 bn->rx_buf_size = rx_space; 3285 3286 bn->rx_nr_pages = bnge_adjust_pow_two(ring_size, RX_DESC_CNT); 3287 bn->rx_ring_mask = (bn->rx_nr_pages * RX_DESC_CNT) - 1; 3288 3289 ring_size = bn->tx_ring_size; 3290 bn->tx_nr_pages = bnge_adjust_pow_two(ring_size, TX_DESC_CNT); 3291 bn->tx_ring_mask = (bn->tx_nr_pages * TX_DESC_CNT) - 1; 3292 3293 max_rx_cmpl = bn->rx_ring_size; 3294 3295 if (bn->priv_flags & BNGE_NET_EN_TPA) 3296 max_rx_cmpl += bd->max_tpa_v2; 3297 ring_size = max_rx_cmpl * 2 + agg_ring_size + bn->tx_ring_size; 3298 bn->cp_ring_size = ring_size; 3299 3300 bn->cp_nr_pages = bnge_adjust_pow_two(ring_size, CP_DESC_CNT); 3301 if (bn->cp_nr_pages > MAX_CP_PAGES) { 3302 bn->cp_nr_pages = MAX_CP_PAGES; 3303 bn->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3304 netdev_warn(bn->netdev, "completion ring size %d reduced to %d.\n", 3305 ring_size, bn->cp_ring_size); 3306 } 3307 bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT; 3308 bn->cp_ring_mask = bn->cp_bit - 1; 3309 } 3310 3311 static void bnge_init_ring_params(struct bnge_net *bn) 3312 { 3313 u32 rx_size; 3314 3315 bn->rx_copybreak = BNGE_DEFAULT_RX_COPYBREAK; 3316 /* Try to fit 4 chunks into a 4k page */ 3317 rx_size = SZ_1K - 3318 NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3319 bn->netdev->cfg->hds_thresh = max(BNGE_DEFAULT_RX_COPYBREAK, rx_size); 3320 } 3321 3322 int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) 3323 { 3324 struct net_device *netdev; 3325 struct bnge_net *bn; 3326 int rc; 3327 3328 netdev = alloc_etherdev_mqs(sizeof(*bn), max_irqs * BNGE_MAX_QUEUE, 3329 max_irqs); 3330 if (!netdev) 3331 return -ENOMEM; 3332 3333 SET_NETDEV_DEV(netdev, bd->dev); 3334 bd->netdev = netdev; 3335 3336 netdev->netdev_ops = &bnge_netdev_ops; 3337 netdev->stat_ops = &bnge_stat_ops; 3338 3339 bnge_set_ethtool_ops(netdev); 3340 3341 bn = netdev_priv(netdev); 3342 bn->netdev = netdev; 3343 bn->bd = bd; 3344 3345 netdev->min_mtu = ETH_ZLEN; 3346 netdev->max_mtu = bd->max_mtu; 3347 3348 netdev->hw_features = NETIF_F_IP_CSUM | 3349 NETIF_F_IPV6_CSUM | 3350 NETIF_F_SG | 3351 NETIF_F_TSO | 3352 NETIF_F_TSO6 | 3353 NETIF_F_GSO_UDP_TUNNEL | 3354 NETIF_F_GSO_GRE | 3355 NETIF_F_GSO_IPXIP4 | 3356 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3357 NETIF_F_GSO_GRE_CSUM | 3358 NETIF_F_GSO_PARTIAL | 3359 NETIF_F_RXHASH | 3360 NETIF_F_RXCSUM | 3361 NETIF_F_GRO; 3362 3363 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3364 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 3365 3366 if (BNGE_SUPPORTS_TPA(bd)) 3367 netdev->hw_features |= NETIF_F_LRO; 3368 3369 netdev->hw_enc_features = NETIF_F_IP_CSUM | 3370 NETIF_F_IPV6_CSUM | 3371 NETIF_F_SG | 3372 NETIF_F_TSO | 3373 NETIF_F_TSO6 | 3374 NETIF_F_GSO_UDP_TUNNEL | 3375 NETIF_F_GSO_GRE | 3376 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3377 NETIF_F_GSO_GRE_CSUM | 3378 NETIF_F_GSO_IPXIP4 | 3379 NETIF_F_GSO_PARTIAL; 3380 3381 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3382 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 3383 3384 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 3385 NETIF_F_GSO_GRE_CSUM; 3386 3387 netdev->vlan_features = netdev->hw_features | NETIF_F_HIGHDMA; 3388 if (bd->fw_cap & BNGE_FW_CAP_VLAN_RX_STRIP) 3389 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_RX; 3390 if (bd->fw_cap & BNGE_FW_CAP_VLAN_TX_INSERT) 3391 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_TX; 3392 3393 if (BNGE_SUPPORTS_TPA(bd)) 3394 netdev->hw_features |= NETIF_F_GRO_HW; 3395 3396 netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA; 3397 3398 if (netdev->features & NETIF_F_GRO_HW) 3399 netdev->features &= ~NETIF_F_LRO; 3400 3401 netdev->priv_flags |= IFF_UNICAST_FLT; 3402 3403 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 3404 if (bd->tso_max_segs) 3405 netif_set_tso_max_segs(netdev, bd->tso_max_segs); 3406 3407 INIT_WORK(&bn->sp_task, bnge_sp_task); 3408 timer_setup(&bn->timer, bnge_timer, 0); 3409 bn->current_interval = BNGE_TIMER_INTERVAL; 3410 3411 bn->bnge_pf_wq = alloc_ordered_workqueue("bnge_pf_wq-%s", 0, 3412 dev_name(bd->dev)); 3413 if (!bn->bnge_pf_wq) { 3414 netdev_err(netdev, "Unable to create workqueue.\n"); 3415 rc = -ENOMEM; 3416 goto err_netdev; 3417 } 3418 3419 bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE; 3420 bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE; 3421 bn->rx_dir = DMA_FROM_DEVICE; 3422 3423 bnge_set_tpa_flags(bd); 3424 bnge_init_ring_params(bn); 3425 bnge_set_ring_params(bd); 3426 3427 bnge_init_l2_fltr_tbl(bn); 3428 bnge_init_mac_addr(bd); 3429 3430 rc = bnge_probe_phy(bn, true); 3431 if (rc) 3432 goto err_free_workq; 3433 3434 rc = bnge_alloc_port_stats(bn); 3435 if (rc) 3436 goto err_free_workq; 3437 spin_lock_init(&bn->stats_lock); 3438 3439 netdev->request_ops_lock = true; 3440 rc = register_netdev(netdev); 3441 if (rc) { 3442 dev_err(bd->dev, "Register netdev failed rc: %d\n", rc); 3443 goto err_free_port_stats; 3444 } 3445 3446 return 0; 3447 3448 err_free_port_stats: 3449 bnge_free_port_stats(bn); 3450 err_free_workq: 3451 destroy_workqueue(bn->bnge_pf_wq); 3452 err_netdev: 3453 free_netdev(netdev); 3454 return rc; 3455 } 3456 3457 void bnge_netdev_free(struct bnge_dev *bd) 3458 { 3459 struct net_device *netdev = bd->netdev; 3460 struct bnge_net *bn; 3461 3462 bn = netdev_priv(netdev); 3463 3464 unregister_netdev(netdev); 3465 3466 timer_shutdown_sync(&bn->timer); 3467 cancel_work_sync(&bn->sp_task); 3468 bn->sp_event = 0; 3469 destroy_workqueue(bn->bnge_pf_wq); 3470 3471 bnge_free_port_stats(bn); 3472 3473 free_netdev(netdev); 3474 bd->netdev = NULL; 3475 } 3476