1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <asm/byteorder.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/dmapool.h> 7 #include <linux/delay.h> 8 #include <linux/errno.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/netdevice.h> 13 #include <net/netdev_lock.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if.h> 18 #include <net/ip.h> 19 #include <linux/skbuff.h> 20 #include <net/page_pool/helpers.h> 21 22 #include "bnge.h" 23 #include "bnge_hwrm.h" 24 #include "bnge_hwrm_lib.h" 25 #include "bnge_ethtool.h" 26 #include "bnge_rmem.h" 27 #include "bnge_txrx.h" 28 29 #define BNGE_RING_TO_TC_OFF(bd, tx) \ 30 ((tx) % (bd)->tx_nr_rings_per_tc) 31 32 #define BNGE_RING_TO_TC(bd, tx) \ 33 ((tx) / (bd)->tx_nr_rings_per_tc) 34 35 #define BNGE_TC_TO_RING_BASE(bd, tc) \ 36 ((tc) * (bd)->tx_nr_rings_per_tc) 37 38 static void bnge_free_stats_mem(struct bnge_net *bn, 39 struct bnge_stats_mem *stats) 40 { 41 struct bnge_dev *bd = bn->bd; 42 43 kfree(stats->hw_masks); 44 stats->hw_masks = NULL; 45 kfree(stats->sw_stats); 46 stats->sw_stats = NULL; 47 if (stats->hw_stats) { 48 dma_free_coherent(bd->dev, stats->len, stats->hw_stats, 49 stats->hw_stats_map); 50 stats->hw_stats = NULL; 51 } 52 } 53 54 static int bnge_alloc_stats_mem(struct bnge_net *bn, 55 struct bnge_stats_mem *stats, bool alloc_masks) 56 { 57 struct bnge_dev *bd = bn->bd; 58 59 stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len, 60 &stats->hw_stats_map, GFP_KERNEL); 61 if (!stats->hw_stats) 62 return -ENOMEM; 63 64 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 65 if (!stats->sw_stats) 66 goto stats_mem_err; 67 68 if (alloc_masks) { 69 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 70 if (!stats->hw_masks) 71 goto stats_mem_err; 72 } 73 74 u64_stats_init(&stats->syncp); 75 76 return 0; 77 78 stats_mem_err: 79 bnge_free_stats_mem(bn, stats); 80 return -ENOMEM; 81 } 82 83 static void bnge_free_ring_stats(struct bnge_net *bn) 84 { 85 struct bnge_dev *bd = bn->bd; 86 int i; 87 88 if (!bn->bnapi) 89 return; 90 91 for (i = 0; i < bd->nq_nr_rings; i++) { 92 struct bnge_napi *bnapi = bn->bnapi[i]; 93 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 94 95 bnge_free_stats_mem(bn, &nqr->stats); 96 } 97 } 98 99 static void bnge_fill_masks(u64 *mask_arr, u64 mask, int count) 100 { 101 int i; 102 103 for (i = 0; i < count; i++) 104 mask_arr[i] = mask; 105 } 106 107 void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 108 { 109 int i; 110 111 for (i = 0; i < count; i++) 112 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 113 } 114 115 static void bnge_init_stats(struct bnge_net *bn) 116 { 117 struct bnge_napi *bnapi = bn->bnapi[0]; 118 struct bnge_nq_ring_info *nqr; 119 struct bnge_stats_mem *stats; 120 struct bnge_dev *bd = bn->bd; 121 __le64 *rx_stats, *tx_stats; 122 int rc, rx_count, tx_count; 123 u64 *rx_masks, *tx_masks; 124 u8 flags; 125 126 nqr = &bnapi->nq_ring; 127 stats = &nqr->stats; 128 rc = bnge_hwrm_func_qstat_ext(bd, stats); 129 if (rc) { 130 u64 mask = (1ULL << 48) - 1; 131 132 bnge_fill_masks(stats->hw_masks, mask, stats->len / 8); 133 } 134 135 if (bn->flags & BNGE_FLAG_PORT_STATS) { 136 stats = &bn->port_stats; 137 rx_stats = stats->hw_stats; 138 rx_masks = stats->hw_masks; 139 rx_count = sizeof(struct rx_port_stats) / 8; 140 tx_stats = rx_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 141 tx_masks = rx_masks + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 142 tx_count = sizeof(struct tx_port_stats) / 8; 143 144 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 145 rc = bnge_hwrm_port_qstats(bd, flags); 146 if (rc) { 147 u64 mask = (1ULL << 40) - 1; 148 149 bnge_fill_masks(rx_masks, mask, rx_count); 150 bnge_fill_masks(tx_masks, mask, tx_count); 151 } else { 152 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 153 bnge_copy_hw_masks(tx_masks, tx_stats, tx_count); 154 } 155 bnge_hwrm_port_qstats(bd, 0); 156 } 157 158 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 159 stats = &bn->rx_port_stats_ext; 160 rx_stats = stats->hw_stats; 161 rx_masks = stats->hw_masks; 162 rx_count = sizeof(struct rx_port_stats_ext) / 8; 163 stats = &bn->tx_port_stats_ext; 164 tx_stats = stats->hw_stats; 165 tx_masks = stats->hw_masks; 166 tx_count = sizeof(struct tx_port_stats_ext) / 8; 167 168 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 169 rc = bnge_hwrm_port_qstats_ext(bd, flags); 170 if (rc) { 171 u64 mask = (1ULL << 40) - 1; 172 173 bnge_fill_masks(rx_masks, mask, rx_count); 174 if (tx_stats) 175 bnge_fill_masks(tx_masks, mask, tx_count); 176 } else { 177 bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); 178 if (tx_stats) 179 bnge_copy_hw_masks(tx_masks, tx_stats, 180 tx_count); 181 } 182 bnge_hwrm_port_qstats_ext(bd, 0); 183 } 184 } 185 186 static void bnge_free_port_ext_stats(struct bnge_net *bn) 187 { 188 bn->flags &= ~BNGE_FLAG_PORT_STATS_EXT; 189 bnge_free_stats_mem(bn, &bn->rx_port_stats_ext); 190 bnge_free_stats_mem(bn, &bn->tx_port_stats_ext); 191 } 192 193 static void bnge_free_port_stats(struct bnge_net *bn) 194 { 195 bn->flags &= ~BNGE_FLAG_PORT_STATS; 196 bnge_free_stats_mem(bn, &bn->port_stats); 197 bnge_free_port_ext_stats(bn); 198 } 199 200 static int bnge_alloc_ring_stats(struct bnge_net *bn) 201 { 202 struct bnge_dev *bd = bn->bd; 203 u32 size, i; 204 int rc; 205 206 size = bd->hw_ring_stats_size; 207 208 for (i = 0; i < bd->nq_nr_rings; i++) { 209 struct bnge_napi *bnapi = bn->bnapi[i]; 210 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 211 212 nqr->stats.len = size; 213 rc = bnge_alloc_stats_mem(bn, &nqr->stats, !i); 214 if (rc) 215 goto err_free_ring_stats; 216 217 nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 218 } 219 220 return 0; 221 222 err_free_ring_stats: 223 bnge_free_ring_stats(bn); 224 return rc; 225 } 226 227 static void bnge_alloc_port_ext_stats(struct bnge_net *bn) 228 { 229 struct bnge_dev *bd = bn->bd; 230 int rc; 231 232 if (!(bd->fw_cap & BNGE_FW_CAP_EXT_STATS_SUPPORTED)) 233 return; 234 235 if (!bn->rx_port_stats_ext.hw_stats) { 236 bn->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 237 rc = bnge_alloc_stats_mem(bn, &bn->rx_port_stats_ext, true); 238 /* Extended stats are optional */ 239 if (rc) 240 return; 241 } 242 243 if (!bn->tx_port_stats_ext.hw_stats) { 244 bn->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 245 rc = bnge_alloc_stats_mem(bn, &bn->tx_port_stats_ext, true); 246 /* Extended stats are optional */ 247 if (rc) { 248 bnge_free_port_ext_stats(bn); 249 return; 250 } 251 } 252 bn->flags |= BNGE_FLAG_PORT_STATS_EXT; 253 } 254 255 static int bnge_alloc_port_stats(struct bnge_net *bn) 256 { 257 int rc; 258 259 if (!bn->port_stats.hw_stats) { 260 bn->port_stats.len = BNGE_PORT_STATS_SIZE; 261 rc = bnge_alloc_stats_mem(bn, &bn->port_stats, true); 262 if (rc) 263 return rc; 264 265 bn->flags |= BNGE_FLAG_PORT_STATS; 266 } 267 268 bnge_alloc_port_ext_stats(bn); 269 return 0; 270 } 271 272 void __bnge_queue_sp_work(struct bnge_net *bn) 273 { 274 queue_work(bn->bnge_pf_wq, &bn->sp_task); 275 } 276 277 static void bnge_queue_sp_work(struct bnge_net *bn, unsigned int event) 278 { 279 set_bit(event, &bn->sp_event); 280 __bnge_queue_sp_work(bn); 281 } 282 283 static void bnge_timer(struct timer_list *t) 284 { 285 struct bnge_net *bn = timer_container_of(bn, t, timer); 286 struct bnge_dev *bd = bn->bd; 287 288 if (!netif_running(bn->netdev) || 289 !test_bit(BNGE_STATE_OPEN, &bd->state)) 290 return; 291 292 if (READ_ONCE(bd->link_info.phy_retry)) { 293 if (time_after(jiffies, READ_ONCE(bd->link_info.phy_retry_expires))) { 294 WRITE_ONCE(bd->link_info.phy_retry, false); 295 netdev_warn(bn->netdev, "failed to update PHY settings after maximum retries.\n"); 296 } else { 297 bnge_queue_sp_work(bn, BNGE_UPDATE_PHY_SP_EVENT); 298 } 299 } 300 301 if (BNGE_LINK_IS_UP(bd) && bn->stats_coal_ticks) 302 bnge_queue_sp_work(bn, BNGE_PERIODIC_STATS_SP_EVENT); 303 304 mod_timer(&bn->timer, jiffies + bn->current_interval); 305 } 306 307 static void bnge_add_one_ctr(u64 hw, u64 *sw, u64 mask) 308 { 309 u64 sw_tmp, sw_val; 310 311 hw &= mask; 312 sw_val = READ_ONCE(*sw); 313 sw_tmp = (sw_val & ~mask) | hw; 314 if (hw < (sw_val & mask)) 315 sw_tmp += mask + 1; 316 WRITE_ONCE(*sw, sw_tmp); 317 } 318 319 static void __bnge_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 320 int count, struct u64_stats_sync *syncp) 321 { 322 unsigned long flags; 323 int i; 324 325 flags = u64_stats_update_begin_irqsave(syncp); 326 for (i = 0; i < count; i++) { 327 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 328 329 if (masks[i] == -1ULL) 330 WRITE_ONCE(sw_stats[i], hw); 331 else 332 bnge_add_one_ctr(hw, &sw_stats[i], masks[i]); 333 } 334 u64_stats_update_end_irqrestore(syncp, flags); 335 } 336 337 static void bnge_accumulate_stats(struct bnge_stats_mem *stats) 338 { 339 if (!stats->hw_stats) 340 return; 341 342 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 343 stats->hw_masks, stats->len / 8, &stats->syncp); 344 } 345 346 static void bnge_accumulate_all_stats(struct bnge_dev *bd) 347 { 348 struct bnge_net *bn = netdev_priv(bd->netdev); 349 struct bnge_stats_mem *ring0_stats = NULL; 350 int i; 351 352 for (i = 0; i < bd->nq_nr_rings; i++) { 353 struct bnge_napi *bnapi = bn->bnapi[i]; 354 struct bnge_nq_ring_info *nqr; 355 struct bnge_stats_mem *stats; 356 357 nqr = &bnapi->nq_ring; 358 stats = &nqr->stats; 359 360 if (!ring0_stats) 361 ring0_stats = &bn->bnapi[0]->nq_ring.stats; 362 363 __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, 364 ring0_stats->hw_masks, 365 ring0_stats->len / 8, &stats->syncp); 366 } 367 368 if (bn->flags & BNGE_FLAG_PORT_STATS) { 369 struct bnge_stats_mem *stats = &bn->port_stats; 370 __le64 *hw_stats = stats->hw_stats; 371 u64 *sw_stats = stats->sw_stats; 372 u64 *masks = stats->hw_masks; 373 u16 cnt; 374 375 cnt = sizeof(struct rx_port_stats) / 8; 376 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 377 &stats->syncp); 378 379 hw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 380 sw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 381 masks += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 382 cnt = sizeof(struct tx_port_stats) / 8; 383 __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, 384 &stats->syncp); 385 } 386 387 if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { 388 bnge_accumulate_stats(&bn->rx_port_stats_ext); 389 bnge_accumulate_stats(&bn->tx_port_stats_ext); 390 } 391 } 392 393 static void bnge_sp_task(struct work_struct *work) 394 { 395 struct bnge_net *bn = container_of(work, struct bnge_net, sp_task); 396 bool speed_chng, cfg_chng, link_chng; 397 struct bnge_dev *bd = bn->bd; 398 399 netdev_lock(bn->netdev); 400 if (!test_bit(BNGE_STATE_OPEN, &bd->state)) { 401 netdev_unlock(bn->netdev); 402 return; 403 } 404 405 if (test_and_clear_bit(BNGE_PERIODIC_STATS_SP_EVENT, &bn->sp_event)) { 406 bnge_hwrm_port_qstats(bd, 0); 407 bnge_hwrm_port_qstats_ext(bd, 0); 408 bnge_accumulate_all_stats(bd); 409 } 410 411 if (test_and_clear_bit(BNGE_UPDATE_PHY_SP_EVENT, &bn->sp_event)) { 412 int rc; 413 414 rc = bnge_update_phy_setting(bn); 415 if (rc) { 416 netdev_warn(bn->netdev, "update PHY settings retry failed\n"); 417 } else { 418 WRITE_ONCE(bd->link_info.phy_retry, false); 419 netdev_info(bn->netdev, "update PHY settings retry succeeded\n"); 420 } 421 } 422 423 speed_chng = test_and_clear_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, 424 &bn->sp_event); 425 cfg_chng = test_and_clear_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, 426 &bn->sp_event); 427 link_chng = test_and_clear_bit(BNGE_LINK_CHNG_SP_EVENT, 428 &bn->sp_event); 429 430 if (speed_chng || cfg_chng || link_chng) { 431 int rc; 432 433 if (speed_chng) 434 bnge_hwrm_phy_qcaps(bd); 435 436 rc = bnge_update_link(bn, true); 437 if (rc) 438 netdev_err(bn->netdev, "SP task cannot update link (rc: %d)\n", 439 rc); 440 441 if (speed_chng || cfg_chng) 442 bnge_init_ethtool_link_settings(bn); 443 } 444 445 netdev_unlock(bn->netdev); 446 } 447 448 static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr) 449 { 450 struct bnge_ring_struct *ring = &nqr->ring_struct; 451 452 kfree(nqr->desc_ring); 453 nqr->desc_ring = NULL; 454 ring->ring_mem.pg_arr = NULL; 455 kfree(nqr->desc_mapping); 456 nqr->desc_mapping = NULL; 457 ring->ring_mem.dma_arr = NULL; 458 } 459 460 static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr) 461 { 462 struct bnge_ring_struct *ring = &cpr->ring_struct; 463 464 kfree(cpr->desc_ring); 465 cpr->desc_ring = NULL; 466 ring->ring_mem.pg_arr = NULL; 467 kfree(cpr->desc_mapping); 468 cpr->desc_mapping = NULL; 469 ring->ring_mem.dma_arr = NULL; 470 } 471 472 static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n) 473 { 474 nqr->desc_ring = kzalloc_objs(*nqr->desc_ring, n); 475 if (!nqr->desc_ring) 476 return -ENOMEM; 477 478 nqr->desc_mapping = kzalloc_objs(*nqr->desc_mapping, n); 479 if (!nqr->desc_mapping) 480 goto err_free_desc_ring; 481 return 0; 482 483 err_free_desc_ring: 484 kfree(nqr->desc_ring); 485 nqr->desc_ring = NULL; 486 return -ENOMEM; 487 } 488 489 static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n) 490 { 491 cpr->desc_ring = kzalloc_objs(*cpr->desc_ring, n); 492 if (!cpr->desc_ring) 493 return -ENOMEM; 494 495 cpr->desc_mapping = kzalloc_objs(*cpr->desc_mapping, n); 496 if (!cpr->desc_mapping) 497 goto err_free_desc_ring; 498 return 0; 499 500 err_free_desc_ring: 501 kfree(cpr->desc_ring); 502 cpr->desc_ring = NULL; 503 return -ENOMEM; 504 } 505 506 static void bnge_free_nq_arrays(struct bnge_net *bn) 507 { 508 struct bnge_dev *bd = bn->bd; 509 int i; 510 511 for (i = 0; i < bd->nq_nr_rings; i++) { 512 struct bnge_napi *bnapi = bn->bnapi[i]; 513 514 bnge_free_nq_desc_arr(&bnapi->nq_ring); 515 } 516 } 517 518 static int bnge_alloc_nq_arrays(struct bnge_net *bn) 519 { 520 struct bnge_dev *bd = bn->bd; 521 int i, rc; 522 523 for (i = 0; i < bd->nq_nr_rings; i++) { 524 struct bnge_napi *bnapi = bn->bnapi[i]; 525 526 rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages); 527 if (rc) 528 goto err_free_nq_arrays; 529 } 530 return 0; 531 532 err_free_nq_arrays: 533 bnge_free_nq_arrays(bn); 534 return rc; 535 } 536 537 static void bnge_free_nq_tree(struct bnge_net *bn) 538 { 539 struct bnge_dev *bd = bn->bd; 540 int i; 541 542 for (i = 0; i < bd->nq_nr_rings; i++) { 543 struct bnge_napi *bnapi = bn->bnapi[i]; 544 struct bnge_nq_ring_info *nqr; 545 struct bnge_ring_struct *ring; 546 int j; 547 548 nqr = &bnapi->nq_ring; 549 ring = &nqr->ring_struct; 550 551 bnge_free_ring(bd, &ring->ring_mem); 552 553 if (!nqr->cp_ring_arr) 554 continue; 555 556 for (j = 0; j < nqr->cp_ring_count; j++) { 557 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 558 559 ring = &cpr->ring_struct; 560 bnge_free_ring(bd, &ring->ring_mem); 561 bnge_free_cp_desc_arr(cpr); 562 } 563 kfree(nqr->cp_ring_arr); 564 nqr->cp_ring_arr = NULL; 565 nqr->cp_ring_count = 0; 566 } 567 } 568 569 static int alloc_one_cp_ring(struct bnge_net *bn, 570 struct bnge_cp_ring_info *cpr) 571 { 572 struct bnge_ring_mem_info *rmem; 573 struct bnge_ring_struct *ring; 574 struct bnge_dev *bd = bn->bd; 575 int rc; 576 577 rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages); 578 if (rc) 579 return -ENOMEM; 580 ring = &cpr->ring_struct; 581 rmem = &ring->ring_mem; 582 rmem->nr_pages = bn->cp_nr_pages; 583 rmem->page_size = HW_CMPD_RING_SIZE; 584 rmem->pg_arr = (void **)cpr->desc_ring; 585 rmem->dma_arr = cpr->desc_mapping; 586 rmem->flags = BNGE_RMEM_RING_PTE_FLAG; 587 rc = bnge_alloc_ring(bd, rmem); 588 if (rc) 589 goto err_free_cp_desc_arr; 590 return rc; 591 592 err_free_cp_desc_arr: 593 bnge_free_cp_desc_arr(cpr); 594 return rc; 595 } 596 597 static int bnge_alloc_nq_tree(struct bnge_net *bn) 598 { 599 struct bnge_dev *bd = bn->bd; 600 int i, j, ulp_msix, rc; 601 int tcs = 1; 602 603 ulp_msix = bnge_aux_get_msix(bd); 604 for (i = 0, j = 0; i < bd->nq_nr_rings; i++) { 605 bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL); 606 struct bnge_napi *bnapi = bn->bnapi[i]; 607 struct bnge_nq_ring_info *nqr; 608 struct bnge_cp_ring_info *cpr; 609 struct bnge_ring_struct *ring; 610 int cp_count = 0, k; 611 int rx = 0, tx = 0; 612 613 nqr = &bnapi->nq_ring; 614 nqr->bnapi = bnapi; 615 ring = &nqr->ring_struct; 616 617 rc = bnge_alloc_ring(bd, &ring->ring_mem); 618 if (rc) 619 goto err_free_nq_tree; 620 621 ring->map_idx = ulp_msix + i; 622 623 if (i < bd->rx_nr_rings) { 624 cp_count++; 625 rx = 1; 626 } 627 628 if ((sh && i < bd->tx_nr_rings) || 629 (!sh && i >= bd->rx_nr_rings)) { 630 cp_count += tcs; 631 tx = 1; 632 } 633 634 nqr->cp_ring_arr = kzalloc_objs(*cpr, cp_count); 635 if (!nqr->cp_ring_arr) { 636 rc = -ENOMEM; 637 goto err_free_nq_tree; 638 } 639 640 nqr->cp_ring_count = cp_count; 641 642 for (k = 0; k < cp_count; k++) { 643 cpr = &nqr->cp_ring_arr[k]; 644 rc = alloc_one_cp_ring(bn, cpr); 645 if (rc) 646 goto err_free_nq_tree; 647 648 cpr->bnapi = bnapi; 649 cpr->cp_idx = k; 650 if (!k && rx) { 651 bn->rx_ring[i].rx_cpr = cpr; 652 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX; 653 } else { 654 int n, tc = k - rx; 655 656 n = BNGE_TC_TO_RING_BASE(bd, tc) + j; 657 bn->tx_ring[n].tx_cpr = cpr; 658 cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX; 659 } 660 } 661 if (tx) 662 j++; 663 } 664 return 0; 665 666 err_free_nq_tree: 667 bnge_free_nq_tree(bn); 668 return rc; 669 } 670 671 static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr) 672 { 673 return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE; 674 } 675 676 static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn, 677 struct bnge_rx_ring_info *rxr) 678 { 679 int i, max_idx; 680 681 if (!rxr->rx_buf_ring) 682 return; 683 684 max_idx = bn->rx_nr_pages * RX_DESC_CNT; 685 686 for (i = 0; i < max_idx; i++) { 687 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 688 void *data = rx_buf->data; 689 690 if (!data) 691 continue; 692 693 rx_buf->data = NULL; 694 page_pool_free_va(rxr->head_pool, data, true); 695 } 696 } 697 698 static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn, 699 struct bnge_rx_ring_info *rxr) 700 { 701 int i, max_idx; 702 703 if (!rxr->rx_agg_buf_ring) 704 return; 705 706 max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT; 707 708 for (i = 0; i < max_idx; i++) { 709 struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i]; 710 netmem_ref netmem = rx_agg_buf->netmem; 711 712 if (!netmem) 713 continue; 714 715 rx_agg_buf->netmem = 0; 716 __clear_bit(i, rxr->rx_agg_bmap); 717 718 page_pool_recycle_direct_netmem(rxr->page_pool, netmem); 719 } 720 } 721 722 static void bnge_free_one_tpa_info_data(struct bnge_net *bn, 723 struct bnge_rx_ring_info *rxr) 724 { 725 int i; 726 727 for (i = 0; i < bn->max_tpa; i++) { 728 struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[i]; 729 u8 *data = tpa_info->data; 730 731 if (!data) 732 continue; 733 734 tpa_info->data = NULL; 735 page_pool_free_va(rxr->head_pool, data, false); 736 } 737 } 738 739 static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn, 740 struct bnge_rx_ring_info *rxr) 741 { 742 struct bnge_tpa_idx_map *map; 743 744 if (rxr->rx_tpa) 745 bnge_free_one_tpa_info_data(bn, rxr); 746 747 bnge_free_one_rx_ring_bufs(bn, rxr); 748 bnge_free_one_agg_ring_bufs(bn, rxr); 749 750 map = rxr->rx_tpa_idx_map; 751 if (map) 752 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 753 } 754 755 static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn) 756 { 757 struct bnge_dev *bd = bn->bd; 758 int i; 759 760 if (!bn->rx_ring) 761 return; 762 763 for (i = 0; i < bd->rx_nr_rings; i++) 764 bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]); 765 } 766 767 static void bnge_free_tx_skbs(struct bnge_net *bn) 768 { 769 struct bnge_dev *bd = bn->bd; 770 u16 max_idx; 771 int i; 772 773 max_idx = bn->tx_nr_pages * TX_DESC_CNT; 774 for (i = 0; i < bd->tx_nr_rings; i++) { 775 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 776 int j; 777 778 if (!txr->tx_buf_ring) 779 continue; 780 781 for (j = 0; j < max_idx;) { 782 struct bnge_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; 783 struct sk_buff *skb; 784 int k, last; 785 786 skb = tx_buf->skb; 787 if (!skb) { 788 j++; 789 continue; 790 } 791 792 tx_buf->skb = NULL; 793 794 dma_unmap_single(bd->dev, 795 dma_unmap_addr(tx_buf, mapping), 796 skb_headlen(skb), 797 DMA_TO_DEVICE); 798 799 last = tx_buf->nr_frags; 800 j += 2; 801 for (k = 0; k < last; k++, j++) { 802 int ring_idx = j & bn->tx_ring_mask; 803 skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; 804 805 tx_buf = &txr->tx_buf_ring[ring_idx]; 806 dma_unmap_page(bd->dev, 807 dma_unmap_addr(tx_buf, mapping), 808 skb_frag_size(frag), 809 DMA_TO_DEVICE); 810 } 811 dev_kfree_skb(skb); 812 } 813 netdev_tx_reset_queue(netdev_get_tx_queue(bd->netdev, i)); 814 } 815 } 816 817 static void bnge_free_all_rings_bufs(struct bnge_net *bn) 818 { 819 bnge_free_rx_ring_pair_bufs(bn); 820 bnge_free_tx_skbs(bn); 821 } 822 823 static void bnge_free_tpa_info(struct bnge_net *bn) 824 { 825 struct bnge_dev *bd = bn->bd; 826 int i, j; 827 828 for (i = 0; i < bd->rx_nr_rings; i++) { 829 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 830 831 kfree(rxr->rx_tpa_idx_map); 832 rxr->rx_tpa_idx_map = NULL; 833 if (rxr->rx_tpa) { 834 for (j = 0; j < bn->max_tpa; j++) { 835 kfree(rxr->rx_tpa[j].agg_arr); 836 rxr->rx_tpa[j].agg_arr = NULL; 837 } 838 } 839 kfree(rxr->rx_tpa); 840 rxr->rx_tpa = NULL; 841 } 842 } 843 844 static int bnge_alloc_tpa_info(struct bnge_net *bn) 845 { 846 struct bnge_dev *bd = bn->bd; 847 int i, j; 848 849 if (!bd->max_tpa_v2) 850 return 0; 851 852 bn->max_tpa = max_t(u16, bd->max_tpa_v2, MAX_TPA); 853 for (i = 0; i < bd->rx_nr_rings; i++) { 854 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 855 856 rxr->rx_tpa = kzalloc_objs(struct bnge_tpa_info, bn->max_tpa); 857 if (!rxr->rx_tpa) 858 goto err_free_tpa_info; 859 860 for (j = 0; j < bn->max_tpa; j++) { 861 struct rx_agg_cmp *agg; 862 863 agg = kzalloc_objs(*agg, MAX_SKB_FRAGS); 864 if (!agg) 865 goto err_free_tpa_info; 866 rxr->rx_tpa[j].agg_arr = agg; 867 } 868 rxr->rx_tpa_idx_map = kzalloc_obj(*rxr->rx_tpa_idx_map); 869 if (!rxr->rx_tpa_idx_map) 870 goto err_free_tpa_info; 871 } 872 return 0; 873 874 err_free_tpa_info: 875 bnge_free_tpa_info(bn); 876 return -ENOMEM; 877 } 878 879 static void bnge_free_rx_rings(struct bnge_net *bn) 880 { 881 struct bnge_dev *bd = bn->bd; 882 int i; 883 884 bnge_free_tpa_info(bn); 885 for (i = 0; i < bd->rx_nr_rings; i++) { 886 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 887 struct bnge_ring_struct *ring; 888 889 page_pool_destroy(rxr->page_pool); 890 page_pool_destroy(rxr->head_pool); 891 rxr->page_pool = rxr->head_pool = NULL; 892 893 kfree(rxr->rx_agg_bmap); 894 rxr->rx_agg_bmap = NULL; 895 896 ring = &rxr->rx_ring_struct; 897 bnge_free_ring(bd, &ring->ring_mem); 898 899 ring = &rxr->rx_agg_ring_struct; 900 bnge_free_ring(bd, &ring->ring_mem); 901 } 902 } 903 904 static int bnge_alloc_rx_page_pool(struct bnge_net *bn, 905 struct bnge_rx_ring_info *rxr, 906 int numa_node) 907 { 908 const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE; 909 const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; 910 struct page_pool_params pp = { 0 }; 911 struct bnge_dev *bd = bn->bd; 912 struct page_pool *pool; 913 914 pp.pool_size = bn->rx_agg_ring_size / agg_size_fac; 915 pp.nid = numa_node; 916 pp.netdev = bn->netdev; 917 pp.dev = bd->dev; 918 pp.dma_dir = bn->rx_dir; 919 pp.max_len = PAGE_SIZE; 920 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | 921 PP_FLAG_ALLOW_UNREADABLE_NETMEM; 922 pp.queue_idx = rxr->bnapi->index; 923 924 pool = page_pool_create(&pp); 925 if (IS_ERR(pool)) 926 return PTR_ERR(pool); 927 rxr->page_pool = pool; 928 929 rxr->need_head_pool = page_pool_is_unreadable(pool); 930 if (bnge_separate_head_pool(rxr)) { 931 pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024); 932 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 933 pool = page_pool_create(&pp); 934 if (IS_ERR(pool)) 935 goto err_destroy_pp; 936 } else { 937 page_pool_get(pool); 938 } 939 rxr->head_pool = pool; 940 return 0; 941 942 err_destroy_pp: 943 page_pool_destroy(rxr->page_pool); 944 rxr->page_pool = NULL; 945 return PTR_ERR(pool); 946 } 947 948 static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr) 949 { 950 page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi); 951 page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); 952 } 953 954 static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn, 955 struct bnge_rx_ring_info *rxr) 956 { 957 u16 mem_size; 958 959 rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1; 960 mem_size = rxr->rx_agg_bmap_size / 8; 961 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 962 if (!rxr->rx_agg_bmap) 963 return -ENOMEM; 964 965 return 0; 966 } 967 968 static int bnge_alloc_rx_rings(struct bnge_net *bn) 969 { 970 int i, rc = 0, agg_rings = 0, cpu; 971 struct bnge_dev *bd = bn->bd; 972 973 if (bnge_is_agg_reqd(bd)) 974 agg_rings = 1; 975 976 for (i = 0; i < bd->rx_nr_rings; i++) { 977 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 978 struct bnge_ring_struct *ring; 979 int cpu_node; 980 981 ring = &rxr->rx_ring_struct; 982 983 cpu = cpumask_local_spread(i, dev_to_node(bd->dev)); 984 cpu_node = cpu_to_node(cpu); 985 netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", 986 i, cpu_node); 987 rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node); 988 if (rc) 989 goto err_free_rx_rings; 990 bnge_enable_rx_page_pool(rxr); 991 992 rc = bnge_alloc_ring(bd, &ring->ring_mem); 993 if (rc) 994 goto err_free_rx_rings; 995 996 ring->grp_idx = i; 997 if (agg_rings) { 998 ring = &rxr->rx_agg_ring_struct; 999 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1000 if (rc) 1001 goto err_free_rx_rings; 1002 1003 ring->grp_idx = i; 1004 rc = bnge_alloc_rx_agg_bmap(bn, rxr); 1005 if (rc) 1006 goto err_free_rx_rings; 1007 } 1008 } 1009 1010 if (bn->priv_flags & BNGE_NET_EN_TPA) { 1011 rc = bnge_alloc_tpa_info(bn); 1012 if (rc) 1013 goto err_free_rx_rings; 1014 } 1015 return rc; 1016 1017 err_free_rx_rings: 1018 bnge_free_rx_rings(bn); 1019 return rc; 1020 } 1021 1022 static void bnge_free_tx_rings(struct bnge_net *bn) 1023 { 1024 struct bnge_dev *bd = bn->bd; 1025 int i; 1026 1027 for (i = 0; i < bd->tx_nr_rings; i++) { 1028 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1029 struct bnge_ring_struct *ring; 1030 1031 ring = &txr->tx_ring_struct; 1032 1033 bnge_free_ring(bd, &ring->ring_mem); 1034 } 1035 } 1036 1037 static int bnge_alloc_tx_rings(struct bnge_net *bn) 1038 { 1039 struct bnge_dev *bd = bn->bd; 1040 int i, j, rc; 1041 1042 for (i = 0, j = 0; i < bd->tx_nr_rings; i++) { 1043 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1044 struct bnge_ring_struct *ring; 1045 u8 qidx; 1046 1047 ring = &txr->tx_ring_struct; 1048 1049 rc = bnge_alloc_ring(bd, &ring->ring_mem); 1050 if (rc) 1051 goto err_free_tx_rings; 1052 1053 ring->grp_idx = txr->bnapi->index; 1054 qidx = bd->tc_to_qidx[j]; 1055 ring->queue_id = bd->q_info[qidx].queue_id; 1056 if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1)) 1057 j++; 1058 } 1059 return 0; 1060 1061 err_free_tx_rings: 1062 bnge_free_tx_rings(bn); 1063 return rc; 1064 } 1065 1066 static void bnge_free_vnic_attributes(struct bnge_net *bn) 1067 { 1068 struct pci_dev *pdev = bn->bd->pdev; 1069 struct bnge_vnic_info *vnic; 1070 int i; 1071 1072 if (!bn->vnic_info) 1073 return; 1074 1075 for (i = 0; i < bn->nr_vnics; i++) { 1076 vnic = &bn->vnic_info[i]; 1077 1078 kfree(vnic->uc_list); 1079 vnic->uc_list = NULL; 1080 1081 if (vnic->mc_list) { 1082 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 1083 vnic->mc_list, vnic->mc_list_mapping); 1084 vnic->mc_list = NULL; 1085 } 1086 1087 if (vnic->rss_table) { 1088 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 1089 vnic->rss_table, 1090 vnic->rss_table_dma_addr); 1091 vnic->rss_table = NULL; 1092 } 1093 1094 vnic->rss_hash_key = NULL; 1095 vnic->flags = 0; 1096 } 1097 } 1098 1099 static int bnge_alloc_vnic_attributes(struct bnge_net *bn) 1100 { 1101 struct bnge_dev *bd = bn->bd; 1102 struct bnge_vnic_info *vnic; 1103 int i, size; 1104 1105 for (i = 0; i < bn->nr_vnics; i++) { 1106 vnic = &bn->vnic_info[i]; 1107 1108 if (vnic->flags & BNGE_VNIC_UCAST_FLAG) { 1109 int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN; 1110 1111 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 1112 if (!vnic->uc_list) 1113 goto err_free_vnic_attributes; 1114 } 1115 1116 if (vnic->flags & BNGE_VNIC_MCAST_FLAG) { 1117 vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN; 1118 vnic->mc_list = 1119 dma_alloc_coherent(bd->dev, 1120 vnic->mc_list_size, 1121 &vnic->mc_list_mapping, 1122 GFP_KERNEL); 1123 if (!vnic->mc_list) 1124 goto err_free_vnic_attributes; 1125 } 1126 1127 /* Allocate rss table and hash key */ 1128 size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE); 1129 1130 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 1131 vnic->rss_table = dma_alloc_coherent(bd->dev, 1132 vnic->rss_table_size, 1133 &vnic->rss_table_dma_addr, 1134 GFP_KERNEL); 1135 if (!vnic->rss_table) 1136 goto err_free_vnic_attributes; 1137 1138 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 1139 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 1140 } 1141 return 0; 1142 1143 err_free_vnic_attributes: 1144 bnge_free_vnic_attributes(bn); 1145 return -ENOMEM; 1146 } 1147 1148 static int bnge_alloc_vnics(struct bnge_net *bn) 1149 { 1150 int num_vnics; 1151 1152 /* Allocate only 1 VNIC for now 1153 * Additional VNICs will be added based on RFS/NTUPLE in future patches 1154 */ 1155 num_vnics = 1; 1156 1157 bn->vnic_info = kzalloc_objs(struct bnge_vnic_info, num_vnics); 1158 if (!bn->vnic_info) 1159 return -ENOMEM; 1160 1161 bn->nr_vnics = num_vnics; 1162 1163 return 0; 1164 } 1165 1166 static void bnge_free_vnics(struct bnge_net *bn) 1167 { 1168 kfree(bn->vnic_info); 1169 bn->vnic_info = NULL; 1170 bn->nr_vnics = 0; 1171 } 1172 1173 static void bnge_free_ring_grps(struct bnge_net *bn) 1174 { 1175 kfree(bn->grp_info); 1176 bn->grp_info = NULL; 1177 } 1178 1179 static int bnge_init_ring_grps(struct bnge_net *bn) 1180 { 1181 struct bnge_dev *bd = bn->bd; 1182 int i; 1183 1184 bn->grp_info = kzalloc_objs(struct bnge_ring_grp_info, bd->nq_nr_rings); 1185 if (!bn->grp_info) 1186 return -ENOMEM; 1187 for (i = 0; i < bd->nq_nr_rings; i++) { 1188 bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 1189 bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 1190 bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 1191 bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 1192 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 1193 } 1194 1195 return 0; 1196 } 1197 1198 static void bnge_free_core(struct bnge_net *bn) 1199 { 1200 bnge_free_vnic_attributes(bn); 1201 bnge_free_tx_rings(bn); 1202 bnge_free_rx_rings(bn); 1203 bnge_free_nq_tree(bn); 1204 bnge_free_nq_arrays(bn); 1205 bnge_free_ring_stats(bn); 1206 bnge_free_ring_grps(bn); 1207 bnge_free_vnics(bn); 1208 kfree(bn->tx_ring_map); 1209 bn->tx_ring_map = NULL; 1210 kfree(bn->tx_ring); 1211 bn->tx_ring = NULL; 1212 kfree(bn->rx_ring); 1213 bn->rx_ring = NULL; 1214 kfree(bn->bnapi); 1215 bn->bnapi = NULL; 1216 } 1217 1218 static int bnge_alloc_core(struct bnge_net *bn) 1219 { 1220 struct bnge_dev *bd = bn->bd; 1221 int i, j, size, arr_size; 1222 int rc = -ENOMEM; 1223 void *bnapi; 1224 1225 arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) * 1226 bd->nq_nr_rings); 1227 size = L1_CACHE_ALIGN(sizeof(struct bnge_napi)); 1228 bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL); 1229 if (!bnapi) 1230 return rc; 1231 1232 bn->bnapi = bnapi; 1233 bnapi += arr_size; 1234 for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) { 1235 struct bnge_nq_ring_info *nqr; 1236 1237 bn->bnapi[i] = bnapi; 1238 bn->bnapi[i]->index = i; 1239 bn->bnapi[i]->bn = bn; 1240 nqr = &bn->bnapi[i]->nq_ring; 1241 nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1242 } 1243 1244 bn->rx_ring = kzalloc_objs(struct bnge_rx_ring_info, bd->rx_nr_rings); 1245 if (!bn->rx_ring) 1246 goto err_free_core; 1247 1248 for (i = 0; i < bd->rx_nr_rings; i++) { 1249 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1250 1251 rxr->rx_ring_struct.ring_mem.flags = 1252 BNGE_RMEM_RING_PTE_FLAG; 1253 rxr->rx_agg_ring_struct.ring_mem.flags = 1254 BNGE_RMEM_RING_PTE_FLAG; 1255 rxr->bnapi = bn->bnapi[i]; 1256 bn->bnapi[i]->rx_ring = &bn->rx_ring[i]; 1257 } 1258 1259 bn->tx_ring = kzalloc_objs(struct bnge_tx_ring_info, bd->tx_nr_rings); 1260 if (!bn->tx_ring) 1261 goto err_free_core; 1262 1263 bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16), 1264 GFP_KERNEL); 1265 if (!bn->tx_ring_map) 1266 goto err_free_core; 1267 1268 if (bd->flags & BNGE_EN_SHARED_CHNL) 1269 j = 0; 1270 else 1271 j = bd->rx_nr_rings; 1272 1273 for (i = 0; i < bd->tx_nr_rings; i++) { 1274 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1275 struct bnge_napi *bnapi2; 1276 int k; 1277 1278 txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG; 1279 bn->tx_ring_map[i] = i; 1280 k = j + BNGE_RING_TO_TC_OFF(bd, i); 1281 1282 bnapi2 = bn->bnapi[k]; 1283 txr->txq_index = i; 1284 txr->tx_napi_idx = 1285 BNGE_RING_TO_TC(bd, txr->txq_index); 1286 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 1287 txr->bnapi = bnapi2; 1288 } 1289 1290 rc = bnge_alloc_ring_stats(bn); 1291 if (rc) 1292 goto err_free_core; 1293 1294 bnge_init_stats(bn); 1295 1296 rc = bnge_alloc_vnics(bn); 1297 if (rc) 1298 goto err_free_core; 1299 1300 rc = bnge_alloc_nq_arrays(bn); 1301 if (rc) 1302 goto err_free_core; 1303 1304 bnge_init_ring_struct(bn); 1305 1306 rc = bnge_alloc_rx_rings(bn); 1307 if (rc) 1308 goto err_free_core; 1309 1310 rc = bnge_alloc_tx_rings(bn); 1311 if (rc) 1312 goto err_free_core; 1313 1314 rc = bnge_alloc_nq_tree(bn); 1315 if (rc) 1316 goto err_free_core; 1317 1318 bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG | 1319 BNGE_VNIC_MCAST_FLAG | 1320 BNGE_VNIC_UCAST_FLAG; 1321 rc = bnge_alloc_vnic_attributes(bn); 1322 if (rc) 1323 goto err_free_core; 1324 return 0; 1325 1326 err_free_core: 1327 bnge_free_core(bn); 1328 return rc; 1329 } 1330 1331 u32 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr) 1332 { 1333 return rxr->rx_cpr->ring_struct.fw_ring_id; 1334 } 1335 1336 u32 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr) 1337 { 1338 return txr->tx_cpr->ring_struct.fw_ring_id; 1339 } 1340 1341 static void bnge_db_nq_arm(struct bnge_net *bn, 1342 struct bnge_db_info *db, u32 idx) 1343 { 1344 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_ARM | 1345 DB_RING_IDX(db, idx), db->doorbell); 1346 } 1347 1348 static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1349 { 1350 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK | 1351 DB_RING_IDX(db, idx), db->doorbell); 1352 } 1353 1354 static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx) 1355 { 1356 bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL | 1357 DB_RING_IDX(db, idx), db->doorbell); 1358 } 1359 1360 static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n) 1361 { 1362 struct bnge_napi *bnapi = bn->bnapi[n]; 1363 struct bnge_nq_ring_info *nqr; 1364 1365 nqr = &bnapi->nq_ring; 1366 1367 return nqr->ring_struct.map_idx; 1368 } 1369 1370 static void bnge_init_nq_tree(struct bnge_net *bn) 1371 { 1372 struct bnge_dev *bd = bn->bd; 1373 int i, j; 1374 1375 for (i = 0; i < bd->nq_nr_rings; i++) { 1376 struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring; 1377 struct bnge_ring_struct *ring = &nqr->ring_struct; 1378 1379 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1380 for (j = 0; j < nqr->cp_ring_count; j++) { 1381 struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j]; 1382 1383 ring = &cpr->ring_struct; 1384 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1385 } 1386 } 1387 } 1388 1389 static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn, 1390 dma_addr_t *mapping, 1391 struct bnge_rx_ring_info *rxr, 1392 unsigned int *offset, 1393 gfp_t gfp) 1394 { 1395 netmem_ref netmem; 1396 1397 if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) { 1398 netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, 1399 BNGE_RX_PAGE_SIZE, gfp); 1400 } else { 1401 netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); 1402 *offset = 0; 1403 } 1404 if (!netmem) 1405 return 0; 1406 1407 *mapping = page_pool_get_dma_addr_netmem(netmem) + *offset; 1408 return netmem; 1409 } 1410 1411 u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping, 1412 struct bnge_rx_ring_info *rxr, 1413 gfp_t gfp) 1414 { 1415 unsigned int offset; 1416 struct page *page; 1417 1418 page = page_pool_alloc_frag(rxr->head_pool, &offset, 1419 bn->rx_buf_size, gfp); 1420 if (!page) 1421 return NULL; 1422 1423 *mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset; 1424 return page_address(page) + offset; 1425 } 1426 1427 int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr, 1428 u16 prod, gfp_t gfp) 1429 { 1430 struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)]; 1431 struct rx_bd *rxbd; 1432 dma_addr_t mapping; 1433 u8 *data; 1434 1435 rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)]; 1436 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp); 1437 if (!data) 1438 return -ENOMEM; 1439 1440 rx_buf->data = data; 1441 rx_buf->data_ptr = data + bn->rx_offset; 1442 rx_buf->mapping = mapping; 1443 1444 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1445 1446 return 0; 1447 } 1448 1449 static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn, 1450 struct bnge_rx_ring_info *rxr, 1451 int ring_nr) 1452 { 1453 u32 prod = rxr->rx_prod; 1454 int i, rc = 0; 1455 1456 for (i = 0; i < bn->rx_ring_size; i++) { 1457 rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL); 1458 if (rc) 1459 break; 1460 prod = NEXT_RX(prod); 1461 } 1462 1463 /* Abort if not a single buffer can be allocated */ 1464 if (rc && !i) { 1465 netdev_err(bn->netdev, 1466 "RX ring %d: allocated %d/%d buffers, abort\n", 1467 ring_nr, i, bn->rx_ring_size); 1468 return rc; 1469 } 1470 1471 rxr->rx_prod = prod; 1472 1473 if (i < bn->rx_ring_size) 1474 netdev_warn(bn->netdev, 1475 "RX ring %d: allocated %d/%d buffers, continuing\n", 1476 ring_nr, i, bn->rx_ring_size); 1477 return 0; 1478 } 1479 1480 u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx) 1481 { 1482 u16 next, max = rxr->rx_agg_bmap_size; 1483 1484 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 1485 if (next >= max) 1486 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 1487 return next; 1488 } 1489 1490 int bnge_alloc_rx_netmem(struct bnge_net *bn, 1491 struct bnge_rx_ring_info *rxr, 1492 u16 prod, gfp_t gfp) 1493 { 1494 struct bnge_sw_rx_agg_bd *rx_agg_buf; 1495 u16 sw_prod = rxr->rx_sw_agg_prod; 1496 unsigned int offset = 0; 1497 struct rx_bd *rxbd; 1498 dma_addr_t mapping; 1499 netmem_ref netmem; 1500 1501 rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)]; 1502 netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp); 1503 if (!netmem) 1504 return -ENOMEM; 1505 1506 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1507 sw_prod = bnge_find_next_agg_idx(rxr, sw_prod); 1508 1509 __set_bit(sw_prod, rxr->rx_agg_bmap); 1510 rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod]; 1511 rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod)); 1512 1513 rx_agg_buf->netmem = netmem; 1514 rx_agg_buf->offset = offset; 1515 rx_agg_buf->mapping = mapping; 1516 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1517 rxbd->rx_bd_opaque = sw_prod; 1518 return 0; 1519 } 1520 1521 static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn, 1522 struct bnge_rx_ring_info *rxr, 1523 int ring_nr) 1524 { 1525 u32 prod = rxr->rx_agg_prod; 1526 int i, rc = 0; 1527 1528 for (i = 0; i < bn->rx_agg_ring_size; i++) { 1529 rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL); 1530 if (rc) 1531 break; 1532 prod = NEXT_RX_AGG(prod); 1533 } 1534 1535 if (rc && i < MAX_SKB_FRAGS) { 1536 netdev_err(bn->netdev, 1537 "Agg ring %d: allocated %d/%d buffers (min %d), abort\n", 1538 ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS); 1539 goto err_free_one_agg_ring_bufs; 1540 } 1541 1542 rxr->rx_agg_prod = prod; 1543 1544 if (i < bn->rx_agg_ring_size) 1545 netdev_warn(bn->netdev, 1546 "Agg ring %d: allocated %d/%d buffers, continuing\n", 1547 ring_nr, i, bn->rx_agg_ring_size); 1548 return 0; 1549 1550 err_free_one_agg_ring_bufs: 1551 bnge_free_one_agg_ring_bufs(bn, rxr); 1552 return -ENOMEM; 1553 } 1554 1555 static int bnge_alloc_one_tpa_info_data(struct bnge_net *bn, 1556 struct bnge_rx_ring_info *rxr) 1557 { 1558 dma_addr_t mapping; 1559 u8 *data; 1560 int i; 1561 1562 for (i = 0; i < bn->max_tpa; i++) { 1563 data = __bnge_alloc_rx_frag(bn, &mapping, rxr, 1564 GFP_KERNEL); 1565 if (!data) 1566 goto err_free_tpa_info_data; 1567 1568 rxr->rx_tpa[i].data = data; 1569 rxr->rx_tpa[i].data_ptr = data + bn->rx_offset; 1570 rxr->rx_tpa[i].mapping = mapping; 1571 } 1572 return 0; 1573 1574 err_free_tpa_info_data: 1575 bnge_free_one_tpa_info_data(bn, rxr); 1576 return -ENOMEM; 1577 } 1578 1579 static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr) 1580 { 1581 struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr]; 1582 int rc; 1583 1584 rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr); 1585 if (rc) 1586 return rc; 1587 1588 if (bnge_is_agg_reqd(bn->bd)) { 1589 rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr); 1590 if (rc) 1591 goto err_free_one_rx_ring_bufs; 1592 } 1593 1594 if (rxr->rx_tpa) { 1595 rc = bnge_alloc_one_tpa_info_data(bn, rxr); 1596 if (rc) 1597 goto err_free_one_agg_ring_bufs; 1598 } 1599 1600 return 0; 1601 1602 err_free_one_agg_ring_bufs: 1603 bnge_free_one_agg_ring_bufs(bn, rxr); 1604 err_free_one_rx_ring_bufs: 1605 bnge_free_one_rx_ring_bufs(bn, rxr); 1606 return rc; 1607 } 1608 1609 static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type) 1610 { 1611 struct rx_bd **rx_desc_ring; 1612 u32 prod; 1613 int i; 1614 1615 rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 1616 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 1617 struct rx_bd *rxbd = rx_desc_ring[i]; 1618 int j; 1619 1620 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 1621 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 1622 rxbd->rx_bd_opaque = prod; 1623 } 1624 } 1625 } 1626 1627 static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn, 1628 struct bnge_rx_ring_info *rxr) 1629 { 1630 struct bnge_ring_struct *ring; 1631 u32 type; 1632 1633 type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) | 1634 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 1635 1636 if (NET_IP_ALIGN == 2) 1637 type |= RX_BD_FLAGS_SOP; 1638 1639 ring = &rxr->rx_ring_struct; 1640 bnge_init_rxbd_pages(ring, type); 1641 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1642 } 1643 1644 static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn, 1645 struct bnge_rx_ring_info *rxr) 1646 { 1647 struct bnge_ring_struct *ring; 1648 u32 type; 1649 1650 ring = &rxr->rx_agg_ring_struct; 1651 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1652 if (bnge_is_agg_reqd(bn->bd)) { 1653 type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 1654 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; 1655 1656 bnge_init_rxbd_pages(ring, type); 1657 } 1658 } 1659 1660 static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr) 1661 { 1662 struct bnge_rx_ring_info *rxr; 1663 1664 rxr = &bn->rx_ring[ring_nr]; 1665 bnge_init_one_rx_ring_rxbd(bn, rxr); 1666 1667 netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX, 1668 &rxr->bnapi->napi); 1669 1670 bnge_init_one_agg_ring_rxbd(bn, rxr); 1671 } 1672 1673 static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn) 1674 { 1675 int i, rc; 1676 1677 for (i = 0; i < bn->bd->rx_nr_rings; i++) { 1678 rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i); 1679 if (rc) 1680 goto err_free_rx_ring_pair_bufs; 1681 } 1682 return 0; 1683 1684 err_free_rx_ring_pair_bufs: 1685 bnge_free_rx_ring_pair_bufs(bn); 1686 return rc; 1687 } 1688 1689 static void bnge_init_rx_rings(struct bnge_net *bn) 1690 { 1691 int i; 1692 1693 #define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 1694 #define BNGE_RX_DMA_OFFSET NET_SKB_PAD 1695 bn->rx_offset = BNGE_RX_OFFSET; 1696 bn->rx_dma_offset = BNGE_RX_DMA_OFFSET; 1697 1698 for (i = 0; i < bn->bd->rx_nr_rings; i++) 1699 bnge_init_one_rx_ring_pair(bn, i); 1700 } 1701 1702 static void bnge_init_tx_rings(struct bnge_net *bn) 1703 { 1704 int i; 1705 1706 bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT); 1707 1708 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 1709 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1710 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1711 1712 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 1713 1714 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, 1715 &txr->bnapi->napi); 1716 } 1717 } 1718 1719 static void bnge_init_vnics(struct bnge_net *bn) 1720 { 1721 struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 1722 int i; 1723 1724 for (i = 0; i < bn->nr_vnics; i++) { 1725 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 1726 int j; 1727 1728 vnic->fw_vnic_id = INVALID_HW_RING_ID; 1729 vnic->vnic_id = i; 1730 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) 1731 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 1732 1733 if (bn->vnic_info[i].rss_hash_key) { 1734 if (i == BNGE_VNIC_DEFAULT) { 1735 u8 *key = (void *)vnic->rss_hash_key; 1736 int k; 1737 1738 if (!bn->rss_hash_key_valid && 1739 !bn->rss_hash_key_updated) { 1740 get_random_bytes(bn->rss_hash_key, 1741 HW_HASH_KEY_SIZE); 1742 bn->rss_hash_key_updated = true; 1743 } 1744 1745 memcpy(vnic->rss_hash_key, bn->rss_hash_key, 1746 HW_HASH_KEY_SIZE); 1747 1748 if (!bn->rss_hash_key_updated) 1749 continue; 1750 1751 bn->rss_hash_key_updated = false; 1752 bn->rss_hash_key_valid = true; 1753 1754 bn->toeplitz_prefix = 0; 1755 for (k = 0; k < 8; k++) { 1756 bn->toeplitz_prefix <<= 8; 1757 bn->toeplitz_prefix |= key[k]; 1758 } 1759 } else { 1760 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 1761 HW_HASH_KEY_SIZE); 1762 } 1763 } 1764 } 1765 } 1766 1767 static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db, 1768 u32 ring_type) 1769 { 1770 switch (ring_type) { 1771 case HWRM_RING_ALLOC_TX: 1772 db->db_ring_mask = bn->tx_ring_mask; 1773 break; 1774 case HWRM_RING_ALLOC_RX: 1775 db->db_ring_mask = bn->rx_ring_mask; 1776 break; 1777 case HWRM_RING_ALLOC_AGG: 1778 db->db_ring_mask = bn->rx_agg_ring_mask; 1779 break; 1780 case HWRM_RING_ALLOC_CMPL: 1781 case HWRM_RING_ALLOC_NQ: 1782 db->db_ring_mask = bn->cp_ring_mask; 1783 break; 1784 } 1785 db->db_epoch_mask = db->db_ring_mask + 1; 1786 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 1787 } 1788 1789 static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db, 1790 u32 ring_type, u32 map_idx, u32 xid) 1791 { 1792 struct bnge_dev *bd = bn->bd; 1793 1794 switch (ring_type) { 1795 case HWRM_RING_ALLOC_TX: 1796 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 1797 break; 1798 case HWRM_RING_ALLOC_RX: 1799 case HWRM_RING_ALLOC_AGG: 1800 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 1801 break; 1802 case HWRM_RING_ALLOC_CMPL: 1803 db->db_key64 = DBR_PATH_L2; 1804 break; 1805 case HWRM_RING_ALLOC_NQ: 1806 db->db_key64 = DBR_PATH_L2; 1807 break; 1808 } 1809 db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID; 1810 1811 db->doorbell = bd->bar1 + bd->db_offset; 1812 bnge_set_db_mask(bn, db, ring_type); 1813 } 1814 1815 static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn, 1816 struct bnge_cp_ring_info *cpr) 1817 { 1818 const u32 type = HWRM_RING_ALLOC_CMPL; 1819 struct bnge_napi *bnapi = cpr->bnapi; 1820 struct bnge_ring_struct *ring; 1821 u32 map_idx = bnapi->index; 1822 int rc; 1823 1824 ring = &cpr->ring_struct; 1825 ring->handle = BNGE_SET_NQ_HDL(cpr); 1826 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1827 if (rc) 1828 return rc; 1829 1830 bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 1831 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1832 1833 return 0; 1834 } 1835 1836 static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn, 1837 struct bnge_tx_ring_info *txr, u32 tx_idx) 1838 { 1839 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 1840 const u32 type = HWRM_RING_ALLOC_TX; 1841 int rc; 1842 1843 rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx); 1844 if (rc) 1845 return rc; 1846 1847 bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 1848 1849 return 0; 1850 } 1851 1852 static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn, 1853 struct bnge_rx_ring_info *rxr) 1854 { 1855 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 1856 u32 type = HWRM_RING_ALLOC_AGG; 1857 struct bnge_dev *bd = bn->bd; 1858 u32 grp_idx = ring->grp_idx; 1859 u32 map_idx; 1860 int rc; 1861 1862 map_idx = grp_idx + bd->rx_nr_rings; 1863 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1864 if (rc) 1865 return rc; 1866 1867 bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx, 1868 ring->fw_ring_id); 1869 bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod); 1870 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1871 bn->grp_info[grp_idx].agg_fw_ring_id = (u16)ring->fw_ring_id; 1872 1873 return 0; 1874 } 1875 1876 static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn, 1877 struct bnge_rx_ring_info *rxr) 1878 { 1879 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 1880 struct bnge_napi *bnapi = rxr->bnapi; 1881 u32 type = HWRM_RING_ALLOC_RX; 1882 u32 map_idx = bnapi->index; 1883 int rc; 1884 1885 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1886 if (rc) 1887 return rc; 1888 1889 bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 1890 bn->grp_info[map_idx].rx_fw_ring_id = (u16)ring->fw_ring_id; 1891 1892 return 0; 1893 } 1894 1895 static int bnge_hwrm_ring_alloc(struct bnge_net *bn) 1896 { 1897 struct bnge_dev *bd = bn->bd; 1898 bool agg_rings; 1899 int i, rc = 0; 1900 1901 agg_rings = !!(bnge_is_agg_reqd(bd)); 1902 for (i = 0; i < bd->nq_nr_rings; i++) { 1903 struct bnge_napi *bnapi = bn->bnapi[i]; 1904 struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; 1905 struct bnge_ring_struct *ring = &nqr->ring_struct; 1906 u32 type = HWRM_RING_ALLOC_NQ; 1907 u32 map_idx = ring->map_idx; 1908 unsigned int vector; 1909 1910 vector = bd->irq_tbl[map_idx].vector; 1911 disable_irq_nosync(vector); 1912 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1913 if (rc) { 1914 enable_irq(vector); 1915 goto err_out; 1916 } 1917 bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id); 1918 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 1919 enable_irq(vector); 1920 bn->grp_info[i].nq_fw_ring_id = (u16)ring->fw_ring_id; 1921 1922 if (!i) { 1923 rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id); 1924 if (rc) 1925 netdev_warn(bn->netdev, "Failed to set async event completion ring.\n"); 1926 } 1927 } 1928 1929 for (i = 0; i < bd->tx_nr_rings; i++) { 1930 struct bnge_tx_ring_info *txr = &bn->tx_ring[i]; 1931 1932 rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr); 1933 if (rc) 1934 goto err_out; 1935 rc = bnge_hwrm_tx_ring_alloc(bn, txr, i); 1936 if (rc) 1937 goto err_out; 1938 } 1939 1940 for (i = 0; i < bd->rx_nr_rings; i++) { 1941 struct bnge_rx_ring_info *rxr = &bn->rx_ring[i]; 1942 struct bnge_cp_ring_info *cpr; 1943 struct bnge_ring_struct *ring; 1944 struct bnge_napi *bnapi; 1945 u32 map_idx, type; 1946 1947 rc = bnge_hwrm_rx_ring_alloc(bn, rxr); 1948 if (rc) 1949 goto err_out; 1950 /* If we have agg rings, post agg buffers first. */ 1951 if (!agg_rings) 1952 bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod); 1953 1954 cpr = rxr->rx_cpr; 1955 bnapi = rxr->bnapi; 1956 type = HWRM_RING_ALLOC_CMPL; 1957 map_idx = bnapi->index; 1958 1959 ring = &cpr->ring_struct; 1960 ring->handle = BNGE_SET_NQ_HDL(cpr); 1961 rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx); 1962 if (rc) 1963 goto err_out; 1964 bnge_set_db(bn, &cpr->cp_db, type, map_idx, 1965 ring->fw_ring_id); 1966 bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons); 1967 } 1968 1969 if (agg_rings) { 1970 for (i = 0; i < bd->rx_nr_rings; i++) { 1971 rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]); 1972 if (rc) 1973 goto err_out; 1974 } 1975 } 1976 err_out: 1977 return rc; 1978 } 1979 1980 void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic) 1981 { 1982 __le16 *ring_tbl = vnic->rss_table; 1983 struct bnge_rx_ring_info *rxr; 1984 struct bnge_dev *bd = bn->bd; 1985 u16 tbl_size, i; 1986 1987 tbl_size = bnge_get_rxfh_indir_size(bd); 1988 1989 for (i = 0; i < tbl_size; i++) { 1990 u32 j; 1991 1992 j = bd->rss_indir_tbl[i]; 1993 rxr = &bn->rx_ring[j]; 1994 1995 *ring_tbl++ = cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 1996 *ring_tbl++ = cpu_to_le16(bnge_cp_ring_for_rx(rxr)); 1997 } 1998 } 1999 2000 static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn, 2001 struct bnge_vnic_info *vnic) 2002 { 2003 int rc; 2004 2005 rc = bnge_hwrm_vnic_set_rss(bn, vnic, true); 2006 if (rc) { 2007 netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n", 2008 vnic->vnic_id, rc); 2009 return rc; 2010 } 2011 rc = bnge_hwrm_vnic_cfg(bn, vnic); 2012 if (rc) 2013 netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n", 2014 vnic->vnic_id, rc); 2015 return rc; 2016 } 2017 2018 static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic) 2019 { 2020 struct bnge_dev *bd = bn->bd; 2021 int rc, i, nr_ctxs; 2022 2023 nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings); 2024 for (i = 0; i < nr_ctxs; i++) { 2025 rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i); 2026 if (rc) { 2027 netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n", 2028 vnic->vnic_id, i, rc); 2029 return -ENOMEM; 2030 } 2031 bn->rsscos_nr_ctxs++; 2032 } 2033 2034 rc = bnge_hwrm_vnic_rss_cfg(bn, vnic); 2035 if (rc) 2036 return rc; 2037 2038 if (bnge_is_agg_reqd(bd)) { 2039 rc = bnge_hwrm_vnic_set_hds(bn, vnic); 2040 if (rc) 2041 netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n", 2042 vnic->vnic_id, rc); 2043 } 2044 return rc; 2045 } 2046 2047 static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr) 2048 { 2049 if (!refcount_dec_and_test(&fltr->refcnt)) 2050 return; 2051 hlist_del_rcu(&fltr->base.hash); 2052 kfree_rcu(fltr, base.rcu); 2053 } 2054 2055 static void bnge_init_l2_filter(struct bnge_net *bn, 2056 struct bnge_l2_filter *fltr, 2057 struct bnge_l2_key *key, u32 idx) 2058 { 2059 struct hlist_head *head; 2060 2061 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 2062 fltr->l2_key.vlan = key->vlan; 2063 fltr->base.type = BNGE_FLTR_TYPE_L2; 2064 2065 head = &bn->l2_fltr_hash_tbl[idx]; 2066 hlist_add_head_rcu(&fltr->base.hash, head); 2067 refcount_set(&fltr->refcnt, 1); 2068 } 2069 2070 static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn, 2071 struct bnge_l2_key *key, 2072 u32 idx) 2073 { 2074 struct bnge_l2_filter *fltr; 2075 struct hlist_head *head; 2076 2077 head = &bn->l2_fltr_hash_tbl[idx]; 2078 hlist_for_each_entry_rcu(fltr, head, base.hash) { 2079 struct bnge_l2_key *l2_key = &fltr->l2_key; 2080 2081 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 2082 l2_key->vlan == key->vlan) 2083 return fltr; 2084 } 2085 return NULL; 2086 } 2087 2088 static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn, 2089 struct bnge_l2_key *key, 2090 u32 idx) 2091 { 2092 struct bnge_l2_filter *fltr; 2093 2094 rcu_read_lock(); 2095 fltr = __bnge_lookup_l2_filter(bn, key, idx); 2096 if (fltr) 2097 refcount_inc(&fltr->refcnt); 2098 rcu_read_unlock(); 2099 return fltr; 2100 } 2101 2102 static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn, 2103 struct bnge_l2_key *key, 2104 gfp_t gfp) 2105 { 2106 struct bnge_l2_filter *fltr; 2107 u32 idx; 2108 2109 idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) & 2110 BNGE_L2_FLTR_HASH_MASK; 2111 fltr = bnge_lookup_l2_filter(bn, key, idx); 2112 if (fltr) 2113 return fltr; 2114 2115 fltr = kzalloc_obj(*fltr, gfp); 2116 if (!fltr) 2117 return ERR_PTR(-ENOMEM); 2118 2119 bnge_init_l2_filter(bn, fltr, key, idx); 2120 return fltr; 2121 } 2122 2123 static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx, 2124 const u8 *mac_addr) 2125 { 2126 struct bnge_l2_filter *fltr; 2127 struct bnge_l2_key key; 2128 int rc; 2129 2130 ether_addr_copy(key.dst_mac_addr, mac_addr); 2131 key.vlan = 0; 2132 fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL); 2133 if (IS_ERR(fltr)) 2134 return PTR_ERR(fltr); 2135 2136 fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id; 2137 rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr); 2138 if (rc) 2139 goto err_del_l2_filter; 2140 bn->vnic_info[vnic_id].l2_filters[idx] = fltr; 2141 return rc; 2142 2143 err_del_l2_filter: 2144 bnge_del_l2_filter(bn, fltr); 2145 return rc; 2146 } 2147 2148 static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask, 2149 const struct netdev_hw_addr_list *mc) 2150 { 2151 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2152 struct netdev_hw_addr *ha; 2153 int mc_count = 0, off = 0; 2154 bool update = false; 2155 u8 *haddr; 2156 2157 netdev_hw_addr_list_for_each(ha, mc) { 2158 if (mc_count >= BNGE_MAX_MC_ADDRS) { 2159 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2160 vnic->mc_list_count = 0; 2161 return false; 2162 } 2163 haddr = ha->addr; 2164 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 2165 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 2166 update = true; 2167 } 2168 off += ETH_ALEN; 2169 mc_count++; 2170 } 2171 if (mc_count) 2172 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2173 2174 if (mc_count != vnic->mc_list_count) { 2175 vnic->mc_list_count = mc_count; 2176 update = true; 2177 } 2178 return update; 2179 } 2180 2181 static bool bnge_uc_list_updated(struct bnge_net *bn, 2182 const struct netdev_hw_addr_list *uc) 2183 { 2184 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2185 struct netdev_hw_addr *ha; 2186 int off = 0; 2187 2188 if (netdev_hw_addr_list_count(uc) != (vnic->uc_filter_count - 1)) 2189 return true; 2190 2191 netdev_hw_addr_list_for_each(ha, uc) { 2192 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 2193 return true; 2194 2195 off += ETH_ALEN; 2196 } 2197 return false; 2198 } 2199 2200 static bool bnge_promisc_ok(struct bnge_net *bn) 2201 { 2202 return true; 2203 } 2204 2205 static int bnge_cfg_rx_mode(struct bnge_net *bn, struct netdev_hw_addr_list *uc, 2206 bool uc_update, bool snapshot) 2207 { 2208 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2209 struct net_device *dev = bn->netdev; 2210 struct bnge_dev *bd = bn->bd; 2211 struct netdev_hw_addr *ha; 2212 int i, off = 0, rc; 2213 2214 if (!uc_update) 2215 goto skip_uc; 2216 2217 for (i = 1; i < vnic->uc_filter_count; i++) { 2218 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2219 2220 bnge_hwrm_l2_filter_free(bd, fltr); 2221 bnge_del_l2_filter(bn, fltr); 2222 } 2223 2224 vnic->uc_filter_count = 1; 2225 2226 if (!snapshot) 2227 netif_addr_lock_bh(dev); 2228 if (netdev_hw_addr_list_count(uc) > (BNGE_MAX_UC_ADDRS - 1)) { 2229 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2230 } else { 2231 netdev_hw_addr_list_for_each(ha, uc) { 2232 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 2233 off += ETH_ALEN; 2234 vnic->uc_filter_count++; 2235 } 2236 } 2237 if (!snapshot) 2238 netif_addr_unlock_bh(dev); 2239 2240 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 2241 rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off); 2242 if (rc) { 2243 if (rc == -EAGAIN) 2244 netdev_warn(dev, "FW busy while setting vnic filter, will retry\n"); 2245 else 2246 netdev_err(dev, "HWRM vnic filter failure rc: %d\n", 2247 rc); 2248 vnic->uc_filter_count = i; 2249 return rc; 2250 } 2251 } 2252 2253 skip_uc: 2254 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 2255 !bnge_promisc_ok(bn)) 2256 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2257 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2258 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 2259 netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 2260 rc); 2261 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 2262 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2263 vnic->mc_list_count = 0; 2264 rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic); 2265 } 2266 if (rc) { 2267 if (rc == -EAGAIN) { 2268 netdev_warn(dev, "FW busy while setting l2 rx mask in CFA, will retry\n"); 2269 vnic->rx_mask &= ~BNGE_RX_MASK_CFG_FLAGS; 2270 } else { 2271 netdev_err(dev, "HWRM CFA L2 rx mask failure rc: %d\n", 2272 rc); 2273 } 2274 } 2275 2276 return rc; 2277 } 2278 2279 static int bnge_set_rx_mode(struct net_device *dev, 2280 struct netdev_hw_addr_list *uc, 2281 struct netdev_hw_addr_list *mc) 2282 { 2283 struct bnge_net *bn = netdev_priv(dev); 2284 struct bnge_vnic_info *vnic; 2285 bool mc_update = false; 2286 bool uc_update; 2287 u32 mask; 2288 2289 if (!test_bit(BNGE_STATE_OPEN, &bn->bd->state)) 2290 return 0; 2291 2292 vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2293 mask = vnic->rx_mask; 2294 mask &= ~BNGE_RX_MASK_CFG_FLAGS; 2295 2296 if (dev->flags & IFF_PROMISC) 2297 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2298 2299 uc_update = bnge_uc_list_updated(bn, uc); 2300 2301 if (dev->flags & IFF_BROADCAST) 2302 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2303 if (dev->flags & IFF_ALLMULTI) { 2304 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2305 vnic->mc_list_count = 0; 2306 } else if (dev->flags & IFF_MULTICAST) { 2307 mc_update = bnge_mc_list_updated(bn, &mask, mc); 2308 } 2309 2310 if (mask != vnic->rx_mask || uc_update || mc_update) { 2311 vnic->rx_mask = mask; 2312 return bnge_cfg_rx_mode(bn, uc, uc_update, true); 2313 } 2314 2315 return 0; 2316 } 2317 2318 static void bnge_disable_int(struct bnge_net *bn) 2319 { 2320 struct bnge_dev *bd = bn->bd; 2321 int i; 2322 2323 if (!bn->bnapi) 2324 return; 2325 2326 for (i = 0; i < bd->nq_nr_rings; i++) { 2327 struct bnge_napi *bnapi = bn->bnapi[i]; 2328 struct bnge_nq_ring_info *nqr; 2329 struct bnge_ring_struct *ring; 2330 2331 nqr = &bnapi->nq_ring; 2332 ring = &nqr->ring_struct; 2333 2334 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) 2335 bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons); 2336 } 2337 } 2338 2339 static void bnge_disable_int_sync(struct bnge_net *bn) 2340 { 2341 struct bnge_dev *bd = bn->bd; 2342 int i; 2343 2344 bnge_disable_int(bn); 2345 for (i = 0; i < bd->nq_nr_rings; i++) { 2346 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2347 2348 synchronize_irq(bd->irq_tbl[map_idx].vector); 2349 } 2350 } 2351 2352 static void bnge_enable_int(struct bnge_net *bn) 2353 { 2354 struct bnge_dev *bd = bn->bd; 2355 int i; 2356 2357 for (i = 0; i < bd->nq_nr_rings; i++) { 2358 struct bnge_napi *bnapi = bn->bnapi[i]; 2359 struct bnge_nq_ring_info *nqr; 2360 2361 nqr = &bnapi->nq_ring; 2362 bnge_db_nq_arm(bn, &nqr->nq_db, nqr->nq_raw_cons); 2363 } 2364 } 2365 2366 static void bnge_disable_napi(struct bnge_net *bn) 2367 { 2368 struct bnge_dev *bd = bn->bd; 2369 int i; 2370 2371 if (test_and_set_bit(BNGE_STATE_NAPI_DISABLED, &bn->state)) 2372 return; 2373 2374 for (i = 0; i < bd->nq_nr_rings; i++) { 2375 struct bnge_napi *bnapi = bn->bnapi[i]; 2376 2377 napi_disable_locked(&bnapi->napi); 2378 } 2379 } 2380 2381 static void bnge_enable_napi(struct bnge_net *bn) 2382 { 2383 struct bnge_dev *bd = bn->bd; 2384 int i; 2385 2386 clear_bit(BNGE_STATE_NAPI_DISABLED, &bn->state); 2387 for (i = 0; i < bd->nq_nr_rings; i++) { 2388 struct bnge_napi *bnapi = bn->bnapi[i]; 2389 2390 bnapi->in_reset = false; 2391 bnapi->tx_fault = 0; 2392 2393 napi_enable_locked(&bnapi->napi); 2394 } 2395 } 2396 2397 static void bnge_hwrm_vnic_free(struct bnge_net *bn) 2398 { 2399 int i; 2400 2401 for (i = 0; i < bn->nr_vnics; i++) 2402 bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]); 2403 } 2404 2405 static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn) 2406 { 2407 int i, j; 2408 2409 for (i = 0; i < bn->nr_vnics; i++) { 2410 struct bnge_vnic_info *vnic = &bn->vnic_info[i]; 2411 2412 for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) { 2413 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 2414 bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j); 2415 } 2416 } 2417 bn->rsscos_nr_ctxs = 0; 2418 } 2419 2420 static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn) 2421 { 2422 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2423 int i; 2424 2425 for (i = 0; i < vnic->uc_filter_count; i++) { 2426 struct bnge_l2_filter *fltr = vnic->l2_filters[i]; 2427 2428 bnge_hwrm_l2_filter_free(bn->bd, fltr); 2429 bnge_del_l2_filter(bn, fltr); 2430 } 2431 2432 vnic->uc_filter_count = 0; 2433 } 2434 2435 static void bnge_clear_vnic(struct bnge_net *bn) 2436 { 2437 bnge_hwrm_clear_vnic_filter(bn); 2438 bnge_hwrm_vnic_free(bn); 2439 bnge_hwrm_vnic_ctx_free(bn); 2440 } 2441 2442 static void bnge_hwrm_rx_ring_free(struct bnge_net *bn, 2443 struct bnge_rx_ring_info *rxr, 2444 bool close_path) 2445 { 2446 struct bnge_ring_struct *ring = &rxr->rx_ring_struct; 2447 u32 grp_idx = rxr->bnapi->index; 2448 u32 cmpl_ring_id; 2449 2450 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2451 return; 2452 2453 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2454 hwrm_ring_free_send_msg(bn, ring, 2455 RING_FREE_REQ_RING_TYPE_RX, 2456 close_path ? cmpl_ring_id : 2457 INVALID_HW_RING_ID); 2458 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2459 bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 2460 } 2461 2462 static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn, 2463 struct bnge_rx_ring_info *rxr, 2464 bool close_path) 2465 { 2466 struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct; 2467 u32 grp_idx = rxr->bnapi->index; 2468 u32 cmpl_ring_id; 2469 2470 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2471 return; 2472 2473 cmpl_ring_id = bnge_cp_ring_for_rx(rxr); 2474 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG, 2475 close_path ? cmpl_ring_id : 2476 INVALID_HW_RING_ID); 2477 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2478 bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 2479 } 2480 2481 static void bnge_hwrm_tx_ring_free(struct bnge_net *bn, 2482 struct bnge_tx_ring_info *txr, 2483 bool close_path) 2484 { 2485 struct bnge_ring_struct *ring = &txr->tx_ring_struct; 2486 u32 cmpl_ring_id; 2487 2488 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2489 return; 2490 2491 cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) : 2492 INVALID_HW_RING_ID; 2493 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX, 2494 cmpl_ring_id); 2495 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2496 } 2497 2498 static void bnge_hwrm_cp_ring_free(struct bnge_net *bn, 2499 struct bnge_cp_ring_info *cpr) 2500 { 2501 struct bnge_ring_struct *ring; 2502 2503 ring = &cpr->ring_struct; 2504 if (ring->fw_ring_id == INVALID_HW_RING_ID_32BIT) 2505 return; 2506 2507 hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 2508 INVALID_HW_RING_ID); 2509 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2510 } 2511 2512 static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path) 2513 { 2514 struct bnge_dev *bd = bn->bd; 2515 int i; 2516 2517 if (!bn->bnapi) 2518 return; 2519 2520 for (i = 0; i < bd->tx_nr_rings; i++) 2521 bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path); 2522 2523 for (i = 0; i < bd->rx_nr_rings; i++) { 2524 bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path); 2525 bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path); 2526 } 2527 2528 /* The completion rings are about to be freed. After that the 2529 * IRQ doorbell will not work anymore. So we need to disable 2530 * IRQ here. 2531 */ 2532 bnge_disable_int_sync(bn); 2533 2534 for (i = 0; i < bd->nq_nr_rings; i++) { 2535 struct bnge_napi *bnapi = bn->bnapi[i]; 2536 struct bnge_nq_ring_info *nqr; 2537 struct bnge_ring_struct *ring; 2538 int j; 2539 2540 nqr = &bnapi->nq_ring; 2541 for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++) 2542 bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]); 2543 2544 ring = &nqr->ring_struct; 2545 if (ring->fw_ring_id != INVALID_HW_RING_ID_32BIT) { 2546 hwrm_ring_free_send_msg(bn, ring, 2547 RING_FREE_REQ_RING_TYPE_NQ, 2548 INVALID_HW_RING_ID); 2549 ring->fw_ring_id = INVALID_HW_RING_ID_32BIT; 2550 bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID; 2551 } 2552 } 2553 } 2554 2555 static void bnge_setup_msix(struct bnge_net *bn) 2556 { 2557 struct net_device *dev = bn->netdev; 2558 struct bnge_dev *bd = bn->bd; 2559 int len, i; 2560 2561 len = sizeof(bd->irq_tbl[0].name); 2562 for (i = 0; i < bd->nq_nr_rings; i++) { 2563 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2564 char *attr; 2565 2566 if (bd->flags & BNGE_EN_SHARED_CHNL) 2567 attr = "TxRx"; 2568 else if (i < bd->rx_nr_rings) 2569 attr = "rx"; 2570 else 2571 attr = "tx"; 2572 2573 snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 2574 attr, i); 2575 bd->irq_tbl[map_idx].handler = bnge_msix; 2576 } 2577 } 2578 2579 static int bnge_setup_interrupts(struct bnge_net *bn) 2580 { 2581 struct net_device *dev = bn->netdev; 2582 struct bnge_dev *bd = bn->bd; 2583 2584 bnge_setup_msix(bn); 2585 2586 return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings); 2587 } 2588 2589 static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path) 2590 { 2591 bnge_clear_vnic(bn); 2592 bnge_hwrm_ring_free(bn, close_path); 2593 bnge_hwrm_stat_ctx_free(bn); 2594 } 2595 2596 static void bnge_free_irq(struct bnge_net *bn) 2597 { 2598 struct bnge_dev *bd = bn->bd; 2599 struct bnge_irq *irq; 2600 int i; 2601 2602 for (i = 0; i < bd->nq_nr_rings; i++) { 2603 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2604 2605 irq = &bd->irq_tbl[map_idx]; 2606 if (irq->requested) { 2607 if (irq->have_cpumask) { 2608 irq_set_affinity_hint(irq->vector, NULL); 2609 free_cpumask_var(irq->cpu_mask); 2610 irq->have_cpumask = 0; 2611 } 2612 free_irq(irq->vector, bn->bnapi[i]); 2613 } 2614 2615 irq->requested = 0; 2616 } 2617 } 2618 2619 static int bnge_request_irq(struct bnge_net *bn) 2620 { 2621 struct bnge_dev *bd = bn->bd; 2622 int i, rc; 2623 2624 rc = bnge_setup_interrupts(bn); 2625 if (rc) { 2626 netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc); 2627 return rc; 2628 } 2629 for (i = 0; i < bd->nq_nr_rings; i++) { 2630 int map_idx = bnge_cp_num_to_irq_num(bn, i); 2631 struct bnge_irq *irq = &bd->irq_tbl[map_idx]; 2632 2633 rc = request_irq(irq->vector, irq->handler, 0, irq->name, 2634 bn->bnapi[i]); 2635 if (rc) 2636 goto err_free_irq; 2637 2638 netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector); 2639 irq->requested = 1; 2640 2641 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 2642 int numa_node = dev_to_node(&bd->pdev->dev); 2643 2644 irq->have_cpumask = 1; 2645 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2646 irq->cpu_mask); 2647 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); 2648 if (rc) { 2649 netdev_warn(bn->netdev, 2650 "Set affinity failed, IRQ = %d\n", 2651 irq->vector); 2652 goto err_free_irq; 2653 } 2654 } 2655 } 2656 return 0; 2657 2658 err_free_irq: 2659 bnge_free_irq(bn); 2660 return rc; 2661 } 2662 2663 static int bnge_set_tpa(struct bnge_net *bn, bool set_tpa) 2664 { 2665 u32 tpa_flags = 0; 2666 int rc, i; 2667 2668 if (set_tpa) 2669 tpa_flags = bn->priv_flags & BNGE_NET_EN_TPA; 2670 else if (BNGE_NO_FW_ACCESS(bn->bd)) 2671 return 0; 2672 for (i = 0; i < bn->nr_vnics; i++) { 2673 rc = bnge_hwrm_vnic_set_tpa(bn->bd, &bn->vnic_info[i], 2674 tpa_flags); 2675 if (rc) { 2676 netdev_err(bn->netdev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 2677 i, rc); 2678 return rc; 2679 } 2680 } 2681 return 0; 2682 } 2683 2684 static int bnge_init_chip(struct bnge_net *bn) 2685 { 2686 struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT]; 2687 struct bnge_dev *bd = bn->bd; 2688 int rc; 2689 2690 #define BNGE_DEF_STATS_COAL_TICKS 1000000 2691 bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS; 2692 2693 rc = bnge_hwrm_stat_ctx_alloc(bn); 2694 if (rc) { 2695 netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc); 2696 goto err_out; 2697 } 2698 2699 rc = bnge_hwrm_ring_alloc(bn); 2700 if (rc) { 2701 netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc); 2702 goto err_out; 2703 } 2704 2705 rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings); 2706 if (rc) { 2707 netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc); 2708 goto err_out; 2709 } 2710 2711 rc = bnge_setup_vnic(bn, vnic); 2712 if (rc) 2713 goto err_out; 2714 2715 if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA) 2716 bnge_hwrm_update_rss_hash_cfg(bn); 2717 2718 if (bn->priv_flags & BNGE_NET_EN_TPA) { 2719 rc = bnge_set_tpa(bn, true); 2720 if (rc) 2721 goto err_out; 2722 } 2723 2724 /* Filter for default vnic 0 */ 2725 rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr); 2726 if (rc) { 2727 netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc); 2728 goto err_out; 2729 } 2730 vnic->uc_filter_count = 1; 2731 2732 vnic->rx_mask = 0; 2733 2734 if (bn->netdev->flags & IFF_BROADCAST) 2735 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 2736 2737 if (bn->netdev->flags & IFF_PROMISC) 2738 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 2739 2740 if (bn->netdev->flags & IFF_ALLMULTI) { 2741 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 2742 vnic->mc_list_count = 0; 2743 } else if (bn->netdev->flags & IFF_MULTICAST) { 2744 u32 mask = 0; 2745 2746 bnge_mc_list_updated(bn, &mask, &bn->netdev->mc); 2747 vnic->rx_mask |= mask; 2748 } 2749 2750 rc = bnge_cfg_rx_mode(bn, &bn->netdev->uc, true, false); 2751 if (rc == -EAGAIN) { 2752 netif_rx_mode_schedule_retry(bn->netdev); 2753 rc = 0; 2754 } else if (rc) { 2755 goto err_out; 2756 } 2757 return 0; 2758 2759 err_out: 2760 bnge_hwrm_resource_free(bn, 0); 2761 return rc; 2762 } 2763 2764 static void bnge_init_napi(struct bnge_net *bn) 2765 { 2766 struct bnge_dev *bd = bn->bd; 2767 struct bnge_napi *bnapi; 2768 int i; 2769 2770 for (i = 0; i < bd->nq_nr_rings; i++) { 2771 bnapi = bn->bnapi[i]; 2772 netif_napi_add_config_locked(bn->netdev, &bnapi->napi, 2773 bnge_napi_poll, bnapi->index); 2774 } 2775 } 2776 2777 static void bnge_del_napi(struct bnge_net *bn) 2778 { 2779 struct bnge_dev *bd = bn->bd; 2780 int i; 2781 2782 for (i = 0; i < bd->rx_nr_rings; i++) 2783 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL); 2784 for (i = 0; i < bd->tx_nr_rings; i++) 2785 netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL); 2786 2787 for (i = 0; i < bd->nq_nr_rings; i++) { 2788 struct bnge_napi *bnapi = bn->bnapi[i]; 2789 2790 __netif_napi_del_locked(&bnapi->napi); 2791 } 2792 2793 /* Wait for RCU grace period after removing NAPI instances */ 2794 synchronize_net(); 2795 } 2796 2797 static int bnge_init_nic(struct bnge_net *bn) 2798 { 2799 int rc; 2800 2801 bnge_init_nq_tree(bn); 2802 2803 bnge_init_rx_rings(bn); 2804 rc = bnge_alloc_rx_ring_pair_bufs(bn); 2805 if (rc) 2806 return rc; 2807 2808 bnge_init_tx_rings(bn); 2809 2810 rc = bnge_init_ring_grps(bn); 2811 if (rc) 2812 goto err_free_rx_ring_pair_bufs; 2813 2814 bnge_init_vnics(bn); 2815 2816 rc = bnge_init_chip(bn); 2817 if (rc) 2818 goto err_free_ring_grps; 2819 return rc; 2820 2821 err_free_ring_grps: 2822 bnge_free_ring_grps(bn); 2823 err_free_rx_ring_pair_bufs: 2824 bnge_free_rx_ring_pair_bufs(bn); 2825 return rc; 2826 } 2827 2828 static void bnge_tx_disable(struct bnge_net *bn) 2829 { 2830 struct bnge_tx_ring_info *txr; 2831 int i; 2832 2833 if (bn->tx_ring) { 2834 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2835 txr = &bn->tx_ring[i]; 2836 WRITE_ONCE(txr->dev_state, BNGE_DEV_STATE_CLOSING); 2837 } 2838 } 2839 /* Make sure napi polls see @dev_state change */ 2840 synchronize_net(); 2841 2842 if (!bn->netdev) 2843 return; 2844 /* Drop carrier first to prevent TX timeout */ 2845 netif_carrier_off(bn->netdev); 2846 /* Stop all TX queues */ 2847 netif_tx_disable(bn->netdev); 2848 } 2849 2850 static void bnge_tx_enable(struct bnge_net *bn) 2851 { 2852 struct bnge_tx_ring_info *txr; 2853 int i; 2854 2855 for (i = 0; i < bn->bd->tx_nr_rings; i++) { 2856 txr = &bn->tx_ring[i]; 2857 WRITE_ONCE(txr->dev_state, 0); 2858 } 2859 /* Make sure napi polls see @dev_state change */ 2860 synchronize_net(); 2861 netif_tx_wake_all_queues(bn->netdev); 2862 if (BNGE_LINK_IS_UP(bn->bd)) 2863 netif_carrier_on(bn->netdev); 2864 } 2865 2866 static int bnge_hwrm_if_change(struct bnge_dev *bd, bool up) 2867 { 2868 struct hwrm_func_drv_if_change_input *req; 2869 int rc; 2870 2871 if (!(bd->fw_cap & BNGE_FW_CAP_IF_CHANGE)) 2872 return 0; 2873 2874 rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_DRV_IF_CHANGE); 2875 if (rc) 2876 return rc; 2877 2878 if (up) 2879 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 2880 2881 return bnge_hwrm_req_send(bd, req); 2882 } 2883 2884 static int bnge_open_core(struct bnge_net *bn) 2885 { 2886 struct bnge_dev *bd = bn->bd; 2887 int rc; 2888 2889 netif_carrier_off(bn->netdev); 2890 2891 rc = bnge_hwrm_if_change(bd, true); 2892 if (rc) { 2893 netdev_err(bn->netdev, "bnge_hwrm_if_change err: %d\n", rc); 2894 return rc; 2895 } 2896 2897 rc = bnge_reserve_rings(bd); 2898 if (rc) { 2899 netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc); 2900 goto err_if_change; 2901 } 2902 2903 rc = bnge_alloc_core(bn); 2904 if (rc) { 2905 netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc); 2906 goto err_if_change; 2907 } 2908 2909 bnge_init_napi(bn); 2910 rc = bnge_request_irq(bn); 2911 if (rc) { 2912 netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc); 2913 goto err_del_napi; 2914 } 2915 2916 rc = bnge_init_nic(bn); 2917 if (rc) { 2918 netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc); 2919 goto err_free_irq; 2920 } 2921 2922 bnge_enable_napi(bn); 2923 2924 rc = bnge_update_phy_setting(bn); 2925 if (rc) { 2926 netdev_warn(bn->netdev, "failed to update PHY settings (rc: %d)\n", 2927 rc); 2928 WRITE_ONCE(bd->link_info.phy_retry_expires, jiffies + 5 * HZ); 2929 WRITE_ONCE(bd->link_info.phy_retry, true); 2930 } else { 2931 WRITE_ONCE(bd->link_info.phy_retry, false); 2932 } 2933 2934 set_bit(BNGE_STATE_OPEN, &bd->state); 2935 set_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 2936 2937 bnge_enable_int(bn); 2938 2939 bnge_tx_enable(bn); 2940 2941 mod_timer(&bn->timer, jiffies + bn->current_interval); 2942 2943 /* Poll link status and check for SFP+ module status */ 2944 bnge_get_port_module_status(bn); 2945 2946 return 0; 2947 2948 err_free_irq: 2949 bnge_free_irq(bn); 2950 err_del_napi: 2951 bnge_del_napi(bn); 2952 bnge_free_core(bn); 2953 err_if_change: 2954 bnge_hwrm_if_change(bd, false); 2955 return rc; 2956 } 2957 2958 static int bnge_open(struct net_device *dev) 2959 { 2960 struct bnge_net *bn = netdev_priv(dev); 2961 int rc; 2962 2963 rc = bnge_open_core(bn); 2964 if (rc) 2965 netdev_err(dev, "bnge_open_core err: %d\n", rc); 2966 2967 return rc; 2968 } 2969 2970 static int bnge_shutdown_nic(struct bnge_net *bn) 2971 { 2972 bnge_hwrm_resource_free(bn, 1); 2973 return 0; 2974 } 2975 2976 static void bnge_add_prev_ring_stats64(struct bnge_net *bn, 2977 struct rtnl_link_stats64 *stats) 2978 { 2979 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 2980 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 2981 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 2982 2983 stats->rx_packets += rx_save->packets; 2984 stats->tx_packets += tx_save->packets; 2985 stats->rx_bytes += rx_save->bytes; 2986 stats->tx_bytes += tx_save->bytes; 2987 stats->rx_missed_errors += rx_save->hw_drop_overruns; 2988 stats->tx_dropped += tx_save->hw_drop_errors; 2989 2990 stats->multicast += stats64_save->multicast; 2991 } 2992 2993 static void bnge_get_ring_stats64(struct bnge_dev *bd, 2994 struct rtnl_link_stats64 *stats) 2995 { 2996 struct bnge_net *bn = netdev_priv(bd->netdev); 2997 int i; 2998 2999 for (i = 0; i < bd->nq_nr_rings; i++) { 3000 struct bnge_napi *bnapi = bn->bnapi[i]; 3001 u64 tx_bytes, tx_packets, tx_dropped; 3002 u64 multicast, rx_missed_errors; 3003 struct bnge_nq_ring_info *nqr; 3004 u64 rx_bytes, rx_packets; 3005 unsigned int start; 3006 u64 *sw; 3007 3008 nqr = &bnapi->nq_ring; 3009 sw = nqr->stats.sw_stats; 3010 3011 do { 3012 start = u64_stats_fetch_begin(&nqr->stats.syncp); 3013 3014 rx_packets = BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3015 rx_packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3016 rx_packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3017 3018 tx_packets = BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3019 tx_packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3020 tx_packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3021 3022 rx_bytes = BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3023 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3024 rx_bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3025 3026 tx_bytes = BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3027 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3028 tx_bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3029 3030 multicast = BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3031 rx_missed_errors = 3032 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3033 tx_dropped = 3034 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3035 } while (u64_stats_fetch_retry(&nqr->stats.syncp, start)); 3036 3037 stats->rx_packets += rx_packets; 3038 stats->tx_packets += tx_packets; 3039 stats->rx_bytes += rx_bytes; 3040 stats->tx_bytes += tx_bytes; 3041 stats->multicast += multicast; 3042 stats->rx_missed_errors += rx_missed_errors; 3043 stats->tx_dropped += tx_dropped; 3044 } 3045 } 3046 3047 static void bnge_get_port_stats64(struct bnge_net *bn, 3048 struct rtnl_link_stats64 *stats) 3049 { 3050 unsigned int start; 3051 u64 *tx, *rx; 3052 3053 rx = bn->port_stats.sw_stats; 3054 tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; 3055 3056 do { 3057 start = u64_stats_fetch_begin(&bn->port_stats.syncp); 3058 3059 stats->rx_crc_errors = 3060 BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 3061 stats->rx_frame_errors = 3062 BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 3063 stats->rx_length_errors = 3064 BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 3065 BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 3066 BNGE_GET_RX_PORT_STATS64(rx, rx_runt_frames); 3067 stats->rx_errors = 3068 BNGE_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 3069 BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 3070 stats->collisions = 3071 BNGE_GET_TX_PORT_STATS64(tx, tx_total_collisions); 3072 stats->tx_fifo_errors = 3073 BNGE_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 3074 stats->tx_errors = BNGE_GET_TX_PORT_STATS64(tx, tx_err); 3075 } while (u64_stats_fetch_retry(&bn->port_stats.syncp, start)); 3076 } 3077 3078 static void bnge_fill_prev_stats64(struct bnge_net *bn, 3079 struct rtnl_link_stats64 *stats) 3080 { 3081 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3082 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3083 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3084 3085 stats->rx_packets = rx_save->packets; 3086 stats->tx_packets = tx_save->packets; 3087 stats->rx_bytes = rx_save->bytes; 3088 stats->tx_bytes = tx_save->bytes; 3089 stats->rx_missed_errors = rx_save->hw_drop_overruns; 3090 stats->tx_dropped = tx_save->hw_drop_errors; 3091 stats->multicast = stats64_save->multicast; 3092 } 3093 3094 static void bnge_get_stats64(struct net_device *dev, 3095 struct rtnl_link_stats64 *stats) 3096 { 3097 struct bnge_net *bn = netdev_priv(dev); 3098 3099 if (bn->flags & BNGE_FLAG_PORT_STATS) 3100 bnge_get_port_stats64(bn, stats); 3101 3102 spin_lock_bh(&bn->stats_lock); 3103 if (!test_bit(BNGE_STATE_STATS_ENABLE, &bn->state)) { 3104 bnge_fill_prev_stats64(bn, stats); 3105 spin_unlock_bh(&bn->stats_lock); 3106 return; 3107 } 3108 3109 bnge_get_ring_stats64(bn->bd, stats); 3110 bnge_add_prev_ring_stats64(bn, stats); 3111 spin_unlock_bh(&bn->stats_lock); 3112 } 3113 3114 static void bnge_save_ring_stats(struct bnge_net *bn) 3115 { 3116 struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; 3117 struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; 3118 struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; 3119 int i; 3120 3121 for (i = 0; i < bn->bd->nq_nr_rings; i++) { 3122 struct bnge_napi *bnapi = bn->bnapi[i]; 3123 struct bnge_nq_ring_info *nqr; 3124 u64 *sw; 3125 3126 nqr = &bnapi->nq_ring; 3127 sw = nqr->stats.sw_stats; 3128 3129 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3130 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3131 rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3132 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3133 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3134 rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3135 rx_save->hw_drop_overruns += BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3136 rx_save->hw_drops += BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3137 BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3138 3139 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3140 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3141 tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3142 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3143 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3144 tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3145 tx_save->hw_drop_errors += BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3146 tx_save->hw_drops += BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3147 BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3148 3149 stats64_save->multicast += 3150 BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3151 } 3152 } 3153 3154 static void bnge_close_core(struct bnge_net *bn) 3155 { 3156 struct bnge_dev *bd = bn->bd; 3157 3158 bnge_tx_disable(bn); 3159 3160 clear_bit(BNGE_STATE_OPEN, &bd->state); 3161 3162 timer_delete_sync(&bn->timer); 3163 bnge_shutdown_nic(bn); 3164 bnge_disable_napi(bn); 3165 3166 /* Save ring stats before shutdown */ 3167 spin_lock_bh(&bn->stats_lock); 3168 bnge_save_ring_stats(bn); 3169 clear_bit(BNGE_STATE_STATS_ENABLE, &bn->state); 3170 spin_unlock_bh(&bn->stats_lock); 3171 3172 bnge_free_all_rings_bufs(bn); 3173 bnge_free_irq(bn); 3174 bnge_del_napi(bn); 3175 3176 bnge_free_core(bn); 3177 } 3178 3179 static int bnge_close(struct net_device *dev) 3180 { 3181 struct bnge_net *bn = netdev_priv(dev); 3182 3183 bnge_close_core(bn); 3184 bnge_hwrm_shutdown_link(bn->bd); 3185 bnge_hwrm_if_change(bn->bd, false); 3186 bn->sp_event = 0; 3187 3188 return 0; 3189 } 3190 3191 static void bnge_get_queue_stats_rx(struct net_device *dev, int i, 3192 struct netdev_queue_stats_rx *stats) 3193 { 3194 struct bnge_net *bn = netdev_priv(dev); 3195 struct bnge_nq_ring_info *nqr; 3196 u64 *sw; 3197 3198 if (!bn->bnapi) 3199 return; 3200 3201 nqr = &bn->bnapi[i]->nq_ring; 3202 sw = nqr->stats.sw_stats; 3203 3204 stats->packets = 0; 3205 stats->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); 3206 stats->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); 3207 stats->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); 3208 3209 stats->bytes = 0; 3210 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); 3211 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); 3212 stats->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); 3213 3214 stats->hw_drop_overruns = BNGE_GET_RING_STATS64(sw, rx_discard_pkts); 3215 stats->hw_drops = BNGE_GET_RING_STATS64(sw, rx_error_pkts) + 3216 stats->hw_drop_overruns; 3217 } 3218 3219 static void bnge_get_queue_stats_tx(struct net_device *dev, int i, 3220 struct netdev_queue_stats_tx *stats) 3221 { 3222 struct bnge_net *bn = netdev_priv(dev); 3223 struct bnge_napi *bnapi; 3224 u64 *sw; 3225 3226 if (!bn->tx_ring) 3227 return; 3228 3229 bnapi = bn->tx_ring[bn->tx_ring_map[i]].bnapi; 3230 sw = bnapi->nq_ring.stats.sw_stats; 3231 3232 stats->packets = 0; 3233 stats->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); 3234 stats->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); 3235 stats->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); 3236 3237 stats->bytes = 0; 3238 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); 3239 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); 3240 stats->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); 3241 3242 stats->hw_drop_errors = BNGE_GET_RING_STATS64(sw, tx_error_pkts); 3243 stats->hw_drops = BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + 3244 stats->hw_drop_errors; 3245 } 3246 3247 static void bnge_get_base_stats(struct net_device *dev, 3248 struct netdev_queue_stats_rx *rx, 3249 struct netdev_queue_stats_tx *tx) 3250 { 3251 struct bnge_net *bn = netdev_priv(dev); 3252 3253 rx->packets = bn->rxq_prv_stats.packets; 3254 rx->bytes = bn->rxq_prv_stats.bytes; 3255 rx->hw_drops = bn->rxq_prv_stats.hw_drops; 3256 rx->hw_drop_overruns = bn->rxq_prv_stats.hw_drop_overruns; 3257 3258 tx->packets = bn->txq_prv_stats.packets; 3259 tx->bytes = bn->txq_prv_stats.bytes; 3260 tx->hw_drops = bn->txq_prv_stats.hw_drops; 3261 tx->hw_drop_errors = bn->txq_prv_stats.hw_drop_errors; 3262 } 3263 3264 static const struct netdev_stat_ops bnge_stat_ops = { 3265 .get_queue_stats_rx = bnge_get_queue_stats_rx, 3266 .get_queue_stats_tx = bnge_get_queue_stats_tx, 3267 .get_base_stats = bnge_get_base_stats, 3268 }; 3269 3270 static const struct net_device_ops bnge_netdev_ops = { 3271 .ndo_open = bnge_open, 3272 .ndo_stop = bnge_close, 3273 .ndo_start_xmit = bnge_start_xmit, 3274 .ndo_get_stats64 = bnge_get_stats64, 3275 .ndo_set_rx_mode_async = bnge_set_rx_mode, 3276 .ndo_features_check = bnge_features_check, 3277 }; 3278 3279 static void bnge_init_mac_addr(struct bnge_dev *bd) 3280 { 3281 eth_hw_addr_set(bd->netdev, bd->pf.mac_addr); 3282 } 3283 3284 static void bnge_set_tpa_flags(struct bnge_dev *bd) 3285 { 3286 struct bnge_net *bn = netdev_priv(bd->netdev); 3287 3288 bn->priv_flags &= ~BNGE_NET_EN_TPA; 3289 3290 if (bd->netdev->features & NETIF_F_LRO) 3291 bn->priv_flags |= BNGE_NET_EN_LRO; 3292 else if (bd->netdev->features & NETIF_F_GRO_HW) 3293 bn->priv_flags |= BNGE_NET_EN_GRO; 3294 } 3295 3296 static void bnge_init_l2_fltr_tbl(struct bnge_net *bn) 3297 { 3298 int i; 3299 3300 for (i = 0; i < BNGE_L2_FLTR_HASH_SIZE; i++) 3301 INIT_HLIST_HEAD(&bn->l2_fltr_hash_tbl[i]); 3302 get_random_bytes(&bn->hash_seed, sizeof(bn->hash_seed)); 3303 } 3304 3305 void bnge_set_ring_params(struct bnge_dev *bd) 3306 { 3307 struct bnge_net *bn = netdev_priv(bd->netdev); 3308 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 3309 u32 agg_factor = 0, agg_ring_size = 0; 3310 3311 /* 8 for CRC and VLAN */ 3312 rx_size = SKB_DATA_ALIGN(bn->netdev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 3313 3314 rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) + 3315 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3316 3317 ring_size = bn->rx_ring_size; 3318 bn->rx_agg_ring_size = 0; 3319 bn->rx_agg_nr_pages = 0; 3320 3321 if (bn->priv_flags & BNGE_NET_EN_TPA) 3322 agg_factor = min_t(u32, 4, 65536 / BNGE_RX_PAGE_SIZE); 3323 3324 bn->priv_flags &= ~BNGE_NET_EN_JUMBO; 3325 if (rx_space > PAGE_SIZE) { 3326 u32 jumbo_factor; 3327 3328 bn->priv_flags |= BNGE_NET_EN_JUMBO; 3329 jumbo_factor = PAGE_ALIGN(bn->netdev->mtu - 40) >> PAGE_SHIFT; 3330 if (jumbo_factor > agg_factor) 3331 agg_factor = jumbo_factor; 3332 } 3333 if (agg_factor) { 3334 if (ring_size > BNGE_MAX_RX_DESC_CNT_JUM_ENA) { 3335 ring_size = BNGE_MAX_RX_DESC_CNT_JUM_ENA; 3336 netdev_warn(bn->netdev, "RX ring size reduced from %d to %d due to jumbo ring\n", 3337 bn->rx_ring_size, ring_size); 3338 bn->rx_ring_size = ring_size; 3339 } 3340 agg_ring_size = ring_size * agg_factor; 3341 3342 bn->rx_agg_nr_pages = bnge_adjust_pow_two(agg_ring_size, 3343 RX_DESC_CNT); 3344 if (bn->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 3345 u32 tmp = agg_ring_size; 3346 3347 bn->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 3348 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 3349 netdev_warn(bn->netdev, "RX agg ring size %d reduced to %d.\n", 3350 tmp, agg_ring_size); 3351 } 3352 bn->rx_agg_ring_size = agg_ring_size; 3353 bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1; 3354 3355 rx_size = max3(BNGE_DEFAULT_RX_COPYBREAK, 3356 bn->rx_copybreak, 3357 bn->netdev->cfg_pending->hds_thresh); 3358 rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); 3359 rx_space = rx_size + NET_SKB_PAD + 3360 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3361 } 3362 3363 bn->rx_buf_use_size = rx_size; 3364 bn->rx_buf_size = rx_space; 3365 3366 bn->rx_nr_pages = bnge_adjust_pow_two(ring_size, RX_DESC_CNT); 3367 bn->rx_ring_mask = (bn->rx_nr_pages * RX_DESC_CNT) - 1; 3368 3369 ring_size = bn->tx_ring_size; 3370 bn->tx_nr_pages = bnge_adjust_pow_two(ring_size, TX_DESC_CNT); 3371 bn->tx_ring_mask = (bn->tx_nr_pages * TX_DESC_CNT) - 1; 3372 3373 max_rx_cmpl = bn->rx_ring_size; 3374 3375 if (bn->priv_flags & BNGE_NET_EN_TPA) 3376 max_rx_cmpl += bd->max_tpa_v2; 3377 ring_size = max_rx_cmpl * 2 + agg_ring_size + bn->tx_ring_size; 3378 bn->cp_ring_size = ring_size; 3379 3380 bn->cp_nr_pages = bnge_adjust_pow_two(ring_size, CP_DESC_CNT); 3381 if (bn->cp_nr_pages > MAX_CP_PAGES) { 3382 bn->cp_nr_pages = MAX_CP_PAGES; 3383 bn->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 3384 netdev_warn(bn->netdev, "completion ring size %d reduced to %d.\n", 3385 ring_size, bn->cp_ring_size); 3386 } 3387 bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT; 3388 bn->cp_ring_mask = bn->cp_bit - 1; 3389 } 3390 3391 static void bnge_init_ring_params(struct bnge_net *bn) 3392 { 3393 u32 rx_size; 3394 3395 bn->rx_copybreak = BNGE_DEFAULT_RX_COPYBREAK; 3396 /* Try to fit 4 chunks into a 4k page */ 3397 rx_size = SZ_1K - 3398 NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3399 bn->netdev->cfg->hds_thresh = max(BNGE_DEFAULT_RX_COPYBREAK, rx_size); 3400 } 3401 3402 int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) 3403 { 3404 struct net_device *netdev; 3405 struct bnge_net *bn; 3406 int rc; 3407 3408 netdev = alloc_etherdev_mqs(sizeof(*bn), max_irqs * BNGE_MAX_QUEUE, 3409 max_irqs); 3410 if (!netdev) 3411 return -ENOMEM; 3412 3413 SET_NETDEV_DEV(netdev, bd->dev); 3414 bd->netdev = netdev; 3415 3416 netdev->netdev_ops = &bnge_netdev_ops; 3417 netdev->stat_ops = &bnge_stat_ops; 3418 3419 bnge_set_ethtool_ops(netdev); 3420 3421 bn = netdev_priv(netdev); 3422 bn->netdev = netdev; 3423 bn->bd = bd; 3424 3425 netdev->min_mtu = ETH_ZLEN; 3426 netdev->max_mtu = bd->max_mtu; 3427 3428 netdev->hw_features = NETIF_F_IP_CSUM | 3429 NETIF_F_IPV6_CSUM | 3430 NETIF_F_SG | 3431 NETIF_F_TSO | 3432 NETIF_F_TSO6 | 3433 NETIF_F_GSO_UDP_TUNNEL | 3434 NETIF_F_GSO_GRE | 3435 NETIF_F_GSO_IPXIP4 | 3436 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3437 NETIF_F_GSO_GRE_CSUM | 3438 NETIF_F_GSO_PARTIAL | 3439 NETIF_F_RXHASH | 3440 NETIF_F_RXCSUM | 3441 NETIF_F_GRO; 3442 3443 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3444 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 3445 3446 if (BNGE_SUPPORTS_TPA(bd)) 3447 netdev->hw_features |= NETIF_F_LRO; 3448 3449 netdev->hw_enc_features = NETIF_F_IP_CSUM | 3450 NETIF_F_IPV6_CSUM | 3451 NETIF_F_SG | 3452 NETIF_F_TSO | 3453 NETIF_F_TSO6 | 3454 NETIF_F_GSO_UDP_TUNNEL | 3455 NETIF_F_GSO_GRE | 3456 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3457 NETIF_F_GSO_GRE_CSUM | 3458 NETIF_F_GSO_IPXIP4 | 3459 NETIF_F_GSO_PARTIAL; 3460 3461 if (bd->flags & BNGE_EN_UDP_GSO_SUPP) 3462 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 3463 3464 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 3465 NETIF_F_GSO_GRE_CSUM; 3466 3467 netdev->vlan_features = netdev->hw_features | NETIF_F_HIGHDMA; 3468 if (bd->fw_cap & BNGE_FW_CAP_VLAN_RX_STRIP) 3469 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_RX; 3470 if (bd->fw_cap & BNGE_FW_CAP_VLAN_TX_INSERT) 3471 netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_TX; 3472 3473 if (BNGE_SUPPORTS_TPA(bd)) 3474 netdev->hw_features |= NETIF_F_GRO_HW; 3475 3476 netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA; 3477 3478 if (netdev->features & NETIF_F_GRO_HW) 3479 netdev->features &= ~NETIF_F_LRO; 3480 3481 netdev->priv_flags |= IFF_UNICAST_FLT; 3482 3483 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 3484 if (bd->tso_max_segs) 3485 netif_set_tso_max_segs(netdev, bd->tso_max_segs); 3486 3487 INIT_WORK(&bn->sp_task, bnge_sp_task); 3488 timer_setup(&bn->timer, bnge_timer, 0); 3489 bn->current_interval = BNGE_TIMER_INTERVAL; 3490 3491 bn->bnge_pf_wq = alloc_ordered_workqueue("bnge_pf_wq-%s", 0, 3492 dev_name(bd->dev)); 3493 if (!bn->bnge_pf_wq) { 3494 netdev_err(netdev, "Unable to create workqueue.\n"); 3495 rc = -ENOMEM; 3496 goto err_netdev; 3497 } 3498 3499 bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE; 3500 bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE; 3501 bn->rx_dir = DMA_FROM_DEVICE; 3502 3503 bnge_set_tpa_flags(bd); 3504 bnge_init_ring_params(bn); 3505 bnge_set_ring_params(bd); 3506 3507 bnge_init_l2_fltr_tbl(bn); 3508 bnge_init_mac_addr(bd); 3509 3510 rc = bnge_probe_phy(bn, true); 3511 if (rc) 3512 goto err_free_workq; 3513 3514 rc = bnge_alloc_port_stats(bn); 3515 if (rc) 3516 goto err_free_workq; 3517 spin_lock_init(&bn->stats_lock); 3518 3519 netdev->request_ops_lock = true; 3520 rc = register_netdev(netdev); 3521 if (rc) { 3522 dev_err(bd->dev, "Register netdev failed rc: %d\n", rc); 3523 goto err_free_port_stats; 3524 } 3525 3526 return 0; 3527 3528 err_free_port_stats: 3529 bnge_free_port_stats(bn); 3530 err_free_workq: 3531 destroy_workqueue(bn->bnge_pf_wq); 3532 err_netdev: 3533 free_netdev(netdev); 3534 return rc; 3535 } 3536 3537 void bnge_netdev_free(struct bnge_dev *bd) 3538 { 3539 struct net_device *netdev = bd->netdev; 3540 struct bnge_net *bn; 3541 3542 bn = netdev_priv(netdev); 3543 3544 unregister_netdev(netdev); 3545 3546 timer_shutdown_sync(&bn->timer); 3547 cancel_work_sync(&bn->sp_task); 3548 bn->sp_event = 0; 3549 destroy_workqueue(bn->bnge_pf_wq); 3550 3551 bnge_free_port_stats(bn); 3552 3553 free_netdev(netdev); 3554 bd->netdev = NULL; 3555 } 3556