1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <linux/etherdevice.h> 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/pci.h> 8 #include <linux/mm.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/vmalloc.h> 11 #include <linux/crash_dump.h> 12 #include <linux/bnxt/hsi.h> 13 14 #include "bnge.h" 15 #include "bnge_hwrm_lib.h" 16 #include "bnge_rmem.h" 17 18 static void bnge_init_ctx_mem(struct bnge_ctx_mem_type *ctxm, 19 void *p, int len) 20 { 21 u8 init_val = ctxm->init_value; 22 u16 offset = ctxm->init_offset; 23 u8 *p2 = p; 24 int i; 25 26 if (!init_val) 27 return; 28 if (offset == BNGE_CTX_INIT_INVALID_OFFSET) { 29 memset(p, init_val, len); 30 return; 31 } 32 for (i = 0; i < len; i += ctxm->entry_size) 33 *(p2 + i + offset) = init_val; 34 } 35 36 void bnge_free_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem) 37 { 38 struct pci_dev *pdev = bd->pdev; 39 int i; 40 41 if (!rmem->pg_arr) 42 goto skip_pages; 43 44 for (i = 0; i < rmem->nr_pages; i++) { 45 if (!rmem->pg_arr[i]) 46 continue; 47 48 dma_free_coherent(&pdev->dev, rmem->page_size, 49 rmem->pg_arr[i], rmem->dma_arr[i]); 50 51 rmem->pg_arr[i] = NULL; 52 } 53 skip_pages: 54 if (rmem->pg_tbl) { 55 size_t pg_tbl_size = rmem->nr_pages * 8; 56 57 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG) 58 pg_tbl_size = rmem->page_size; 59 dma_free_coherent(&pdev->dev, pg_tbl_size, 60 rmem->pg_tbl, rmem->dma_pg_tbl); 61 rmem->pg_tbl = NULL; 62 } 63 if (rmem->vmem_size && *rmem->vmem) { 64 vfree(*rmem->vmem); 65 *rmem->vmem = NULL; 66 } 67 } 68 69 int bnge_alloc_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem) 70 { 71 struct pci_dev *pdev = bd->pdev; 72 u64 valid_bit = 0; 73 int i; 74 75 if (rmem->flags & (BNGE_RMEM_VALID_PTE_FLAG | BNGE_RMEM_RING_PTE_FLAG)) 76 valid_bit = PTU_PTE_VALID; 77 78 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 79 size_t pg_tbl_size = rmem->nr_pages * 8; 80 81 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG) 82 pg_tbl_size = rmem->page_size; 83 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 84 &rmem->dma_pg_tbl, 85 GFP_KERNEL); 86 if (!rmem->pg_tbl) 87 return -ENOMEM; 88 } 89 90 for (i = 0; i < rmem->nr_pages; i++) { 91 u64 extra_bits = valid_bit; 92 93 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 94 rmem->page_size, 95 &rmem->dma_arr[i], 96 GFP_KERNEL); 97 if (!rmem->pg_arr[i]) 98 return -ENOMEM; 99 100 if (rmem->ctx_mem) 101 bnge_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i], 102 rmem->page_size); 103 104 if (rmem->nr_pages > 1 || rmem->depth > 0) { 105 if (i == rmem->nr_pages - 2 && 106 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG)) 107 extra_bits |= PTU_PTE_NEXT_TO_LAST; 108 else if (i == rmem->nr_pages - 1 && 109 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG)) 110 extra_bits |= PTU_PTE_LAST; 111 rmem->pg_tbl[i] = 112 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 113 } 114 } 115 116 if (rmem->vmem_size) { 117 *rmem->vmem = vzalloc(rmem->vmem_size); 118 if (!(*rmem->vmem)) 119 return -ENOMEM; 120 } 121 122 return 0; 123 } 124 125 static int bnge_alloc_ctx_one_lvl(struct bnge_dev *bd, 126 struct bnge_ctx_pg_info *ctx_pg) 127 { 128 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 129 130 rmem->page_size = BNGE_PAGE_SIZE; 131 rmem->pg_arr = ctx_pg->ctx_pg_arr; 132 rmem->dma_arr = ctx_pg->ctx_dma_arr; 133 rmem->flags = BNGE_RMEM_VALID_PTE_FLAG; 134 if (rmem->depth >= 1) 135 rmem->flags |= BNGE_RMEM_USE_FULL_PAGE_FLAG; 136 return bnge_alloc_ring(bd, rmem); 137 } 138 139 static int bnge_alloc_ctx_pg_tbls(struct bnge_dev *bd, 140 struct bnge_ctx_pg_info *ctx_pg, u32 mem_size, 141 u8 depth, struct bnge_ctx_mem_type *ctxm) 142 { 143 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 144 int rc; 145 146 if (!mem_size) 147 return -EINVAL; 148 149 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE); 150 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 151 ctx_pg->nr_pages = 0; 152 return -EINVAL; 153 } 154 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 155 int nr_tbls, i; 156 157 rmem->depth = 2; 158 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 159 GFP_KERNEL); 160 if (!ctx_pg->ctx_pg_tbl) 161 return -ENOMEM; 162 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 163 rmem->nr_pages = nr_tbls; 164 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg); 165 if (rc) 166 return rc; 167 for (i = 0; i < nr_tbls; i++) { 168 struct bnge_ctx_pg_info *pg_tbl; 169 170 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 171 if (!pg_tbl) 172 return -ENOMEM; 173 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 174 rmem = &pg_tbl->ring_mem; 175 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 176 rmem->dma_pg_tbl = ctx_pg->ctx_dma_arr[i]; 177 rmem->depth = 1; 178 rmem->nr_pages = MAX_CTX_PAGES; 179 rmem->ctx_mem = ctxm; 180 if (i == (nr_tbls - 1)) { 181 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 182 183 if (rem) 184 rmem->nr_pages = rem; 185 } 186 rc = bnge_alloc_ctx_one_lvl(bd, pg_tbl); 187 if (rc) 188 break; 189 } 190 } else { 191 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE); 192 if (rmem->nr_pages > 1 || depth) 193 rmem->depth = 1; 194 rmem->ctx_mem = ctxm; 195 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg); 196 } 197 198 return rc; 199 } 200 201 static void bnge_free_ctx_pg_tbls(struct bnge_dev *bd, 202 struct bnge_ctx_pg_info *ctx_pg) 203 { 204 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem; 205 206 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 207 ctx_pg->ctx_pg_tbl) { 208 int i, nr_tbls = rmem->nr_pages; 209 210 for (i = 0; i < nr_tbls; i++) { 211 struct bnge_ctx_pg_info *pg_tbl; 212 struct bnge_ring_mem_info *rmem2; 213 214 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 215 if (!pg_tbl) 216 continue; 217 rmem2 = &pg_tbl->ring_mem; 218 bnge_free_ring(bd, rmem2); 219 ctx_pg->ctx_pg_arr[i] = NULL; 220 kfree(pg_tbl); 221 ctx_pg->ctx_pg_tbl[i] = NULL; 222 } 223 kfree(ctx_pg->ctx_pg_tbl); 224 ctx_pg->ctx_pg_tbl = NULL; 225 } 226 bnge_free_ring(bd, rmem); 227 ctx_pg->nr_pages = 0; 228 } 229 230 static int bnge_setup_ctxm_pg_tbls(struct bnge_dev *bd, 231 struct bnge_ctx_mem_type *ctxm, u32 entries, 232 u8 pg_lvl) 233 { 234 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info; 235 int i, rc = 0, n = 1; 236 u32 mem_size; 237 238 if (!ctxm->entry_size || !ctx_pg) 239 return -EINVAL; 240 if (ctxm->instance_bmap) 241 n = hweight32(ctxm->instance_bmap); 242 if (ctxm->entry_multiple) 243 entries = roundup(entries, ctxm->entry_multiple); 244 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries); 245 mem_size = entries * ctxm->entry_size; 246 for (i = 0; i < n && !rc; i++) { 247 ctx_pg[i].entries = entries; 248 rc = bnge_alloc_ctx_pg_tbls(bd, &ctx_pg[i], mem_size, pg_lvl, 249 ctxm->init_value ? ctxm : NULL); 250 } 251 252 return rc; 253 } 254 255 static int bnge_backing_store_cfg(struct bnge_dev *bd, u32 ena) 256 { 257 struct bnge_ctx_mem_info *ctx = bd->ctx; 258 struct bnge_ctx_mem_type *ctxm; 259 u16 last_type; 260 int rc = 0; 261 u16 type; 262 263 if (!ena) 264 return 0; 265 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) 266 last_type = BNGE_CTX_MAX - 1; 267 else 268 last_type = BNGE_CTX_L2_MAX - 1; 269 ctx->ctx_arr[last_type].last = 1; 270 271 for (type = 0 ; type < BNGE_CTX_V2_MAX; type++) { 272 ctxm = &ctx->ctx_arr[type]; 273 274 rc = bnge_hwrm_func_backing_store(bd, ctxm, ctxm->last); 275 if (rc) 276 return rc; 277 } 278 279 return 0; 280 } 281 282 void bnge_free_ctx_mem(struct bnge_dev *bd) 283 { 284 struct bnge_ctx_mem_info *ctx = bd->ctx; 285 u16 type; 286 287 if (!ctx) 288 return; 289 290 for (type = 0; type < BNGE_CTX_V2_MAX; type++) { 291 struct bnge_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 292 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info; 293 int i, n = 1; 294 295 if (!ctx_pg) 296 continue; 297 if (ctxm->instance_bmap) 298 n = hweight32(ctxm->instance_bmap); 299 for (i = 0; i < n; i++) 300 bnge_free_ctx_pg_tbls(bd, &ctx_pg[i]); 301 302 kfree(ctx_pg); 303 ctxm->pg_info = NULL; 304 } 305 306 ctx->flags &= ~BNGE_CTX_FLAG_INITED; 307 kfree(ctx); 308 bd->ctx = NULL; 309 } 310 311 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 312 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 313 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 314 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 315 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 316 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 317 318 int bnge_alloc_ctx_mem(struct bnge_dev *bd) 319 { 320 struct bnge_ctx_mem_type *ctxm; 321 struct bnge_ctx_mem_info *ctx; 322 u32 l2_qps, qp1_qps, max_qps; 323 u32 ena, entries_sp, entries; 324 u32 srqs, max_srqs, min; 325 u32 num_mr, num_ah; 326 u32 extra_srqs = 0; 327 u32 extra_qps = 0; 328 u32 fast_qpmd_qps; 329 u8 pg_lvl = 1; 330 int i, rc; 331 332 rc = bnge_hwrm_func_backing_store_qcaps(bd); 333 if (rc) { 334 dev_err(bd->dev, "Failed querying ctx mem caps, rc: %d\n", rc); 335 return rc; 336 } 337 338 ctx = bd->ctx; 339 if (!ctx || (ctx->flags & BNGE_CTX_FLAG_INITED)) 340 return 0; 341 342 ctxm = &ctx->ctx_arr[BNGE_CTX_QP]; 343 l2_qps = ctxm->qp_l2_entries; 344 qp1_qps = ctxm->qp_qp1_entries; 345 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries; 346 max_qps = ctxm->max_entries; 347 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ]; 348 srqs = ctxm->srq_l2_entries; 349 max_srqs = ctxm->max_entries; 350 ena = 0; 351 if (bnge_is_roce_en(bd) && !is_kdump_kernel()) { 352 pg_lvl = 2; 353 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps); 354 /* allocate extra qps if fast qp destroy feature enabled */ 355 extra_qps += fast_qpmd_qps; 356 extra_srqs = min_t(u32, 8192, max_srqs - srqs); 357 if (fast_qpmd_qps) 358 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD; 359 } 360 361 ctxm = &ctx->ctx_arr[BNGE_CTX_QP]; 362 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 363 pg_lvl); 364 if (rc) 365 return rc; 366 367 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ]; 368 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, srqs + extra_srqs, pg_lvl); 369 if (rc) 370 return rc; 371 372 ctxm = &ctx->ctx_arr[BNGE_CTX_CQ]; 373 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->cq_l2_entries + 374 extra_qps * 2, pg_lvl); 375 if (rc) 376 return rc; 377 378 ctxm = &ctx->ctx_arr[BNGE_CTX_VNIC]; 379 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1); 380 if (rc) 381 return rc; 382 383 ctxm = &ctx->ctx_arr[BNGE_CTX_STAT]; 384 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1); 385 if (rc) 386 return rc; 387 388 if (!bnge_is_roce_en(bd)) 389 goto skip_rdma; 390 391 ctxm = &ctx->ctx_arr[BNGE_CTX_MRAV]; 392 /* 128K extra is needed to accommodate static AH context 393 * allocation by f/w. 394 */ 395 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); 396 num_ah = min_t(u32, num_mr, 1024 * 128); 397 ctxm->split_entry_cnt = BNGE_CTX_MRAV_AV_SPLIT_ENTRY + 1; 398 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah) 399 ctxm->mrav_av_entries = num_ah; 400 401 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, num_mr + num_ah, 2); 402 if (rc) 403 return rc; 404 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 405 406 ctxm = &ctx->ctx_arr[BNGE_CTX_TIM]; 407 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 1); 408 if (rc) 409 return rc; 410 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 411 412 skip_rdma: 413 ctxm = &ctx->ctx_arr[BNGE_CTX_STQM]; 414 min = ctxm->min_entries; 415 entries_sp = ctx->ctx_arr[BNGE_CTX_VNIC].vnic_entries + l2_qps + 416 2 * (extra_qps + qp1_qps) + min; 417 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries_sp, 2); 418 if (rc) 419 return rc; 420 421 ctxm = &ctx->ctx_arr[BNGE_CTX_FTQM]; 422 entries = l2_qps + 2 * (extra_qps + qp1_qps); 423 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries, 2); 424 if (rc) 425 return rc; 426 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 427 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 428 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 429 430 rc = bnge_backing_store_cfg(bd, ena); 431 if (rc) { 432 dev_err(bd->dev, "Failed configuring ctx mem, rc: %d\n", rc); 433 return rc; 434 } 435 ctx->flags |= BNGE_CTX_FLAG_INITED; 436 437 return 0; 438 } 439