1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <linux/etherdevice.h> 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/pci.h> 8 #include <linux/mm.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/vmalloc.h> 11 #include <linux/crash_dump.h> 12 #include <linux/bnxt/hsi.h> 13 14 #include "bnge.h" 15 #include "bnge_hwrm_lib.h" 16 #include "bnge_rmem.h" 17 18 static void bnge_init_ctx_mem(struct bnge_ctx_mem_type *ctxm, 19 void *p, int len) 20 { 21 u8 init_val = ctxm->init_value; 22 u16 offset = ctxm->init_offset; 23 u8 *p2 = p; 24 int i; 25 26 if (!init_val) 27 return; 28 if (offset == BNGE_CTX_INIT_INVALID_OFFSET) { 29 memset(p, init_val, len); 30 return; 31 } 32 for (i = 0; i < len; i += ctxm->entry_size) 33 *(p2 + i + offset) = init_val; 34 } 35 36 void bnge_free_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem) 37 { 38 struct pci_dev *pdev = bd->pdev; 39 int i; 40 41 if (!rmem->pg_arr) 42 goto skip_pages; 43 44 for (i = 0; i < rmem->nr_pages; i++) { 45 if (!rmem->pg_arr[i]) 46 continue; 47 48 dma_free_coherent(&pdev->dev, rmem->page_size, 49 rmem->pg_arr[i], rmem->dma_arr[i]); 50 51 rmem->pg_arr[i] = NULL; 52 } 53 skip_pages: 54 if (rmem->pg_tbl) { 55 size_t pg_tbl_size = rmem->nr_pages * 8; 56 57 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG) 58 pg_tbl_size = rmem->page_size; 59 dma_free_coherent(&pdev->dev, pg_tbl_size, 60 rmem->pg_tbl, rmem->dma_pg_tbl); 61 rmem->pg_tbl = NULL; 62 } 63 if (rmem->vmem_size && *rmem->vmem) { 64 vfree(*rmem->vmem); 65 *rmem->vmem = NULL; 66 } 67 } 68 69 int bnge_alloc_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem) 70 { 71 struct pci_dev *pdev = bd->pdev; 72 u64 valid_bit = 0; 73 int i; 74 75 if (rmem->flags & (BNGE_RMEM_VALID_PTE_FLAG | BNGE_RMEM_RING_PTE_FLAG)) 76 valid_bit = PTU_PTE_VALID; 77 78 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 79 size_t pg_tbl_size = rmem->nr_pages * 8; 80 81 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG) 82 pg_tbl_size = rmem->page_size; 83 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 84 &rmem->dma_pg_tbl, 85 GFP_KERNEL); 86 if (!rmem->pg_tbl) 87 return -ENOMEM; 88 } 89 90 for (i = 0; i < rmem->nr_pages; i++) { 91 u64 extra_bits = valid_bit; 92 93 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 94 rmem->page_size, 95 &rmem->dma_arr[i], 96 GFP_KERNEL); 97 if (!rmem->pg_arr[i]) 98 goto err_free_ring; 99 100 if (rmem->ctx_mem) 101 bnge_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i], 102 rmem->page_size); 103 104 if (rmem->nr_pages > 1 || rmem->depth > 0) { 105 if (i == rmem->nr_pages - 2 && 106 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG)) 107 extra_bits |= PTU_PTE_NEXT_TO_LAST; 108 else if (i == rmem->nr_pages - 1 && 109 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG)) 110 extra_bits |= PTU_PTE_LAST; 111 rmem->pg_tbl[i] = 112 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 113 } 114 } 115 116 if (rmem->vmem_size) { 117 *rmem->vmem = vzalloc(rmem->vmem_size); 118 if (!(*rmem->vmem)) 119 goto err_free_ring; 120 } 121 return 0; 122 123 err_free_ring: 124 bnge_free_ring(bd, rmem); 125 return -ENOMEM; 126 } 127 128 static int bnge_alloc_ctx_one_lvl(struct bnge_dev *bd, 129 struct bnge_ctx_pg_info *ctx_pg) 130 { 131 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 132 133 rmem->page_size = BNGE_PAGE_SIZE; 134 rmem->pg_arr = ctx_pg->ctx_pg_arr; 135 rmem->dma_arr = ctx_pg->ctx_dma_arr; 136 rmem->flags = BNGE_RMEM_VALID_PTE_FLAG; 137 if (rmem->depth >= 1) 138 rmem->flags |= BNGE_RMEM_USE_FULL_PAGE_FLAG; 139 return bnge_alloc_ring(bd, rmem); 140 } 141 142 static int bnge_alloc_ctx_pg_tbls(struct bnge_dev *bd, 143 struct bnge_ctx_pg_info *ctx_pg, u32 mem_size, 144 u8 depth, struct bnge_ctx_mem_type *ctxm) 145 { 146 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 147 int rc; 148 149 if (!mem_size) 150 return -EINVAL; 151 152 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE); 153 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 154 ctx_pg->nr_pages = 0; 155 return -EINVAL; 156 } 157 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 158 int nr_tbls, i; 159 160 rmem->depth = 2; 161 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 162 GFP_KERNEL); 163 if (!ctx_pg->ctx_pg_tbl) 164 return -ENOMEM; 165 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 166 rmem->nr_pages = nr_tbls; 167 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg); 168 if (rc) 169 return rc; 170 for (i = 0; i < nr_tbls; i++) { 171 struct bnge_ctx_pg_info *pg_tbl; 172 173 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 174 if (!pg_tbl) 175 return -ENOMEM; 176 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 177 rmem = &pg_tbl->ring_mem; 178 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 179 rmem->dma_pg_tbl = ctx_pg->ctx_dma_arr[i]; 180 rmem->depth = 1; 181 rmem->nr_pages = MAX_CTX_PAGES; 182 rmem->ctx_mem = ctxm; 183 if (i == (nr_tbls - 1)) { 184 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 185 186 if (rem) 187 rmem->nr_pages = rem; 188 } 189 rc = bnge_alloc_ctx_one_lvl(bd, pg_tbl); 190 if (rc) 191 break; 192 } 193 } else { 194 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE); 195 if (rmem->nr_pages > 1 || depth) 196 rmem->depth = 1; 197 rmem->ctx_mem = ctxm; 198 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg); 199 } 200 201 return rc; 202 } 203 204 static void bnge_free_ctx_pg_tbls(struct bnge_dev *bd, 205 struct bnge_ctx_pg_info *ctx_pg) 206 { 207 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 208 209 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 210 ctx_pg->ctx_pg_tbl) { 211 int i, nr_tbls = rmem->nr_pages; 212 213 for (i = 0; i < nr_tbls; i++) { 214 struct bnge_ctx_pg_info *pg_tbl; 215 struct bnge_ring_mem_info *rmem2; 216 217 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 218 if (!pg_tbl) 219 continue; 220 rmem2 = &pg_tbl->ring_mem; 221 bnge_free_ring(bd, rmem2); 222 ctx_pg->ctx_pg_arr[i] = NULL; 223 kfree(pg_tbl); 224 ctx_pg->ctx_pg_tbl[i] = NULL; 225 } 226 kfree(ctx_pg->ctx_pg_tbl); 227 ctx_pg->ctx_pg_tbl = NULL; 228 } 229 bnge_free_ring(bd, rmem); 230 ctx_pg->nr_pages = 0; 231 } 232 233 static int bnge_setup_ctxm_pg_tbls(struct bnge_dev *bd, 234 struct bnge_ctx_mem_type *ctxm, u32 entries, 235 u8 pg_lvl) 236 { 237 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info; 238 int i, rc = 0, n = 1; 239 u32 mem_size; 240 241 if (!ctxm->entry_size || !ctx_pg) 242 return -EINVAL; 243 if (ctxm->instance_bmap) 244 n = hweight32(ctxm->instance_bmap); 245 if (ctxm->entry_multiple) 246 entries = roundup(entries, ctxm->entry_multiple); 247 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries); 248 mem_size = entries * ctxm->entry_size; 249 for (i = 0; i < n && !rc; i++) { 250 ctx_pg[i].entries = entries; 251 rc = bnge_alloc_ctx_pg_tbls(bd, &ctx_pg[i], mem_size, pg_lvl, 252 ctxm->init_value ? ctxm : NULL); 253 } 254 255 return rc; 256 } 257 258 static int bnge_backing_store_cfg(struct bnge_dev *bd, u32 ena) 259 { 260 struct bnge_ctx_mem_info *ctx = bd->ctx; 261 struct bnge_ctx_mem_type *ctxm; 262 u16 last_type; 263 int rc = 0; 264 u16 type; 265 266 if (!ena) 267 return 0; 268 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) 269 last_type = BNGE_CTX_MAX - 1; 270 else 271 last_type = BNGE_CTX_L2_MAX - 1; 272 ctx->ctx_arr[last_type].last = 1; 273 274 for (type = 0 ; type < BNGE_CTX_V2_MAX; type++) { 275 ctxm = &ctx->ctx_arr[type]; 276 277 rc = bnge_hwrm_func_backing_store(bd, ctxm, ctxm->last); 278 if (rc) 279 return rc; 280 } 281 282 return 0; 283 } 284 285 void bnge_free_ctx_mem(struct bnge_dev *bd) 286 { 287 struct bnge_ctx_mem_info *ctx = bd->ctx; 288 u16 type; 289 290 if (!ctx) 291 return; 292 293 for (type = 0; type < BNGE_CTX_V2_MAX; type++) { 294 struct bnge_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 295 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info; 296 int i, n = 1; 297 298 if (!ctx_pg) 299 continue; 300 if (ctxm->instance_bmap) 301 n = hweight32(ctxm->instance_bmap); 302 for (i = 0; i < n; i++) 303 bnge_free_ctx_pg_tbls(bd, &ctx_pg[i]); 304 305 kfree(ctx_pg); 306 ctxm->pg_info = NULL; 307 } 308 309 ctx->flags &= ~BNGE_CTX_FLAG_INITED; 310 kfree(ctx); 311 bd->ctx = NULL; 312 } 313 314 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 315 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 316 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 317 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 318 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 319 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 320 321 int bnge_alloc_ctx_mem(struct bnge_dev *bd) 322 { 323 struct bnge_ctx_mem_type *ctxm; 324 struct bnge_ctx_mem_info *ctx; 325 u32 l2_qps, qp1_qps, max_qps; 326 u32 ena, entries_sp, entries; 327 u32 srqs, max_srqs, min; 328 u32 num_mr, num_ah; 329 u32 extra_srqs = 0; 330 u32 extra_qps = 0; 331 u32 fast_qpmd_qps; 332 u8 pg_lvl = 1; 333 int i, rc; 334 335 rc = bnge_hwrm_func_backing_store_qcaps(bd); 336 if (rc) { 337 dev_err(bd->dev, "Failed querying ctx mem caps, rc: %d\n", rc); 338 return rc; 339 } 340 341 ctx = bd->ctx; 342 if (!ctx || (ctx->flags & BNGE_CTX_FLAG_INITED)) 343 return 0; 344 345 ctxm = &ctx->ctx_arr[BNGE_CTX_QP]; 346 l2_qps = ctxm->qp_l2_entries; 347 qp1_qps = ctxm->qp_qp1_entries; 348 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries; 349 max_qps = ctxm->max_entries; 350 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ]; 351 srqs = ctxm->srq_l2_entries; 352 max_srqs = ctxm->max_entries; 353 ena = 0; 354 if (bnge_is_roce_en(bd) && !is_kdump_kernel()) { 355 pg_lvl = 2; 356 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps); 357 /* allocate extra qps if fast qp destroy feature enabled */ 358 extra_qps += fast_qpmd_qps; 359 extra_srqs = min_t(u32, 8192, max_srqs - srqs); 360 if (fast_qpmd_qps) 361 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD; 362 } 363 364 ctxm = &ctx->ctx_arr[BNGE_CTX_QP]; 365 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 366 pg_lvl); 367 if (rc) 368 return rc; 369 370 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ]; 371 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, srqs + extra_srqs, pg_lvl); 372 if (rc) 373 return rc; 374 375 ctxm = &ctx->ctx_arr[BNGE_CTX_CQ]; 376 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->cq_l2_entries + 377 extra_qps * 2, pg_lvl); 378 if (rc) 379 return rc; 380 381 ctxm = &ctx->ctx_arr[BNGE_CTX_VNIC]; 382 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1); 383 if (rc) 384 return rc; 385 386 ctxm = &ctx->ctx_arr[BNGE_CTX_STAT]; 387 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1); 388 if (rc) 389 return rc; 390 391 if (!bnge_is_roce_en(bd)) 392 goto skip_rdma; 393 394 ctxm = &ctx->ctx_arr[BNGE_CTX_MRAV]; 395 /* 128K extra is needed to accommodate static AH context 396 * allocation by f/w. 397 */ 398 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); 399 num_ah = min_t(u32, num_mr, 1024 * 128); 400 ctxm->split_entry_cnt = BNGE_CTX_MRAV_AV_SPLIT_ENTRY + 1; 401 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah) 402 ctxm->mrav_av_entries = num_ah; 403 404 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, num_mr + num_ah, 2); 405 if (rc) 406 return rc; 407 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 408 409 ctxm = &ctx->ctx_arr[BNGE_CTX_TIM]; 410 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 1); 411 if (rc) 412 return rc; 413 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 414 415 skip_rdma: 416 ctxm = &ctx->ctx_arr[BNGE_CTX_STQM]; 417 min = ctxm->min_entries; 418 entries_sp = ctx->ctx_arr[BNGE_CTX_VNIC].vnic_entries + l2_qps + 419 2 * (extra_qps + qp1_qps) + min; 420 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries_sp, 2); 421 if (rc) 422 return rc; 423 424 ctxm = &ctx->ctx_arr[BNGE_CTX_FTQM]; 425 entries = l2_qps + 2 * (extra_qps + qp1_qps); 426 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries, 2); 427 if (rc) 428 return rc; 429 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 430 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 431 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 432 433 rc = bnge_backing_store_cfg(bd, ena); 434 if (rc) { 435 dev_err(bd->dev, "Failed configuring ctx mem, rc: %d\n", rc); 436 return rc; 437 } 438 ctx->flags |= BNGE_CTX_FLAG_INITED; 439 440 return 0; 441 } 442 443 void bnge_init_ring_struct(struct bnge_net *bn) 444 { 445 struct bnge_dev *bd = bn->bd; 446 int i, j; 447 448 for (i = 0; i < bd->nq_nr_rings; i++) { 449 struct bnge_napi *bnapi = bn->bnapi[i]; 450 struct bnge_ring_mem_info *rmem; 451 struct bnge_nq_ring_info *nqr; 452 struct bnge_rx_ring_info *rxr; 453 struct bnge_tx_ring_info *txr; 454 struct bnge_ring_struct *ring; 455 456 nqr = &bnapi->nq_ring; 457 ring = &nqr->ring_struct; 458 rmem = &ring->ring_mem; 459 rmem->nr_pages = bn->cp_nr_pages; 460 rmem->page_size = HW_CMPD_RING_SIZE; 461 rmem->pg_arr = (void **)nqr->desc_ring; 462 rmem->dma_arr = nqr->desc_mapping; 463 rmem->vmem_size = 0; 464 465 rxr = bnapi->rx_ring; 466 if (!rxr) 467 goto skip_rx; 468 469 ring = &rxr->rx_ring_struct; 470 rmem = &ring->ring_mem; 471 rmem->nr_pages = bn->rx_nr_pages; 472 rmem->page_size = HW_RXBD_RING_SIZE; 473 rmem->pg_arr = (void **)rxr->rx_desc_ring; 474 rmem->dma_arr = rxr->rx_desc_mapping; 475 rmem->vmem_size = SW_RXBD_RING_SIZE * bn->rx_nr_pages; 476 rmem->vmem = (void **)&rxr->rx_buf_ring; 477 478 ring = &rxr->rx_agg_ring_struct; 479 rmem = &ring->ring_mem; 480 rmem->nr_pages = bn->rx_agg_nr_pages; 481 rmem->page_size = HW_RXBD_RING_SIZE; 482 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 483 rmem->dma_arr = rxr->rx_agg_desc_mapping; 484 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bn->rx_agg_nr_pages; 485 rmem->vmem = (void **)&rxr->rx_agg_buf_ring; 486 487 skip_rx: 488 bnge_for_each_napi_tx(j, bnapi, txr) { 489 ring = &txr->tx_ring_struct; 490 rmem = &ring->ring_mem; 491 rmem->nr_pages = bn->tx_nr_pages; 492 rmem->page_size = HW_TXBD_RING_SIZE; 493 rmem->pg_arr = (void **)txr->tx_desc_ring; 494 rmem->dma_arr = txr->tx_desc_mapping; 495 rmem->vmem_size = SW_TXBD_RING_SIZE * bn->tx_nr_pages; 496 rmem->vmem = (void **)&txr->tx_buf_ring; 497 } 498 } 499 } 500