1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2025 Broadcom.
3
4 #include <linux/etherdevice.h>
5 #include <linux/init.h>
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/mm.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/vmalloc.h>
11 #include <linux/crash_dump.h>
12 #include <linux/bnxt/hsi.h>
13
14 #include "bnge.h"
15 #include "bnge_hwrm_lib.h"
16 #include "bnge_rmem.h"
17
bnge_init_ctx_mem(struct bnge_ctx_mem_type * ctxm,void * p,int len)18 static void bnge_init_ctx_mem(struct bnge_ctx_mem_type *ctxm,
19 void *p, int len)
20 {
21 u8 init_val = ctxm->init_value;
22 u16 offset = ctxm->init_offset;
23 u8 *p2 = p;
24 int i;
25
26 if (!init_val)
27 return;
28 if (offset == BNGE_CTX_INIT_INVALID_OFFSET) {
29 memset(p, init_val, len);
30 return;
31 }
32 for (i = 0; i < len; i += ctxm->entry_size)
33 *(p2 + i + offset) = init_val;
34 }
35
bnge_free_ring(struct bnge_dev * bd,struct bnge_ring_mem_info * rmem)36 void bnge_free_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem)
37 {
38 struct pci_dev *pdev = bd->pdev;
39 int i;
40
41 if (!rmem->pg_arr)
42 goto skip_pages;
43
44 for (i = 0; i < rmem->nr_pages; i++) {
45 if (!rmem->pg_arr[i])
46 continue;
47
48 dma_free_coherent(&pdev->dev, rmem->page_size,
49 rmem->pg_arr[i], rmem->dma_arr[i]);
50
51 rmem->pg_arr[i] = NULL;
52 }
53 skip_pages:
54 if (rmem->pg_tbl) {
55 size_t pg_tbl_size = rmem->nr_pages * 8;
56
57 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG)
58 pg_tbl_size = rmem->page_size;
59 dma_free_coherent(&pdev->dev, pg_tbl_size,
60 rmem->pg_tbl, rmem->dma_pg_tbl);
61 rmem->pg_tbl = NULL;
62 }
63 if (rmem->vmem_size && *rmem->vmem) {
64 vfree(*rmem->vmem);
65 *rmem->vmem = NULL;
66 }
67 }
68
bnge_alloc_ring(struct bnge_dev * bd,struct bnge_ring_mem_info * rmem)69 int bnge_alloc_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem)
70 {
71 struct pci_dev *pdev = bd->pdev;
72 u64 valid_bit = 0;
73 int i;
74
75 if (rmem->flags & (BNGE_RMEM_VALID_PTE_FLAG | BNGE_RMEM_RING_PTE_FLAG))
76 valid_bit = PTU_PTE_VALID;
77
78 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
79 size_t pg_tbl_size = rmem->nr_pages * 8;
80
81 if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG)
82 pg_tbl_size = rmem->page_size;
83 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
84 &rmem->dma_pg_tbl,
85 GFP_KERNEL);
86 if (!rmem->pg_tbl)
87 return -ENOMEM;
88 }
89
90 for (i = 0; i < rmem->nr_pages; i++) {
91 u64 extra_bits = valid_bit;
92
93 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
94 rmem->page_size,
95 &rmem->dma_arr[i],
96 GFP_KERNEL);
97 if (!rmem->pg_arr[i])
98 return -ENOMEM;
99
100 if (rmem->ctx_mem)
101 bnge_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
102 rmem->page_size);
103
104 if (rmem->nr_pages > 1 || rmem->depth > 0) {
105 if (i == rmem->nr_pages - 2 &&
106 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG))
107 extra_bits |= PTU_PTE_NEXT_TO_LAST;
108 else if (i == rmem->nr_pages - 1 &&
109 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG))
110 extra_bits |= PTU_PTE_LAST;
111 rmem->pg_tbl[i] =
112 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
113 }
114 }
115
116 if (rmem->vmem_size) {
117 *rmem->vmem = vzalloc(rmem->vmem_size);
118 if (!(*rmem->vmem))
119 return -ENOMEM;
120 }
121
122 return 0;
123 }
124
bnge_alloc_ctx_one_lvl(struct bnge_dev * bd,struct bnge_ctx_pg_info * ctx_pg)125 static int bnge_alloc_ctx_one_lvl(struct bnge_dev *bd,
126 struct bnge_ctx_pg_info *ctx_pg)
127 {
128 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
129
130 rmem->page_size = BNGE_PAGE_SIZE;
131 rmem->pg_arr = ctx_pg->ctx_pg_arr;
132 rmem->dma_arr = ctx_pg->ctx_dma_arr;
133 rmem->flags = BNGE_RMEM_VALID_PTE_FLAG;
134 if (rmem->depth >= 1)
135 rmem->flags |= BNGE_RMEM_USE_FULL_PAGE_FLAG;
136 return bnge_alloc_ring(bd, rmem);
137 }
138
bnge_alloc_ctx_pg_tbls(struct bnge_dev * bd,struct bnge_ctx_pg_info * ctx_pg,u32 mem_size,u8 depth,struct bnge_ctx_mem_type * ctxm)139 static int bnge_alloc_ctx_pg_tbls(struct bnge_dev *bd,
140 struct bnge_ctx_pg_info *ctx_pg, u32 mem_size,
141 u8 depth, struct bnge_ctx_mem_type *ctxm)
142 {
143 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
144 int rc;
145
146 if (!mem_size)
147 return -EINVAL;
148
149 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE);
150 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
151 ctx_pg->nr_pages = 0;
152 return -EINVAL;
153 }
154 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
155 int nr_tbls, i;
156
157 rmem->depth = 2;
158 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
159 GFP_KERNEL);
160 if (!ctx_pg->ctx_pg_tbl)
161 return -ENOMEM;
162 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
163 rmem->nr_pages = nr_tbls;
164 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg);
165 if (rc)
166 return rc;
167 for (i = 0; i < nr_tbls; i++) {
168 struct bnge_ctx_pg_info *pg_tbl;
169
170 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
171 if (!pg_tbl)
172 return -ENOMEM;
173 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
174 rmem = &pg_tbl->ring_mem;
175 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
176 rmem->dma_pg_tbl = ctx_pg->ctx_dma_arr[i];
177 rmem->depth = 1;
178 rmem->nr_pages = MAX_CTX_PAGES;
179 rmem->ctx_mem = ctxm;
180 if (i == (nr_tbls - 1)) {
181 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
182
183 if (rem)
184 rmem->nr_pages = rem;
185 }
186 rc = bnge_alloc_ctx_one_lvl(bd, pg_tbl);
187 if (rc)
188 break;
189 }
190 } else {
191 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE);
192 if (rmem->nr_pages > 1 || depth)
193 rmem->depth = 1;
194 rmem->ctx_mem = ctxm;
195 rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg);
196 }
197
198 return rc;
199 }
200
bnge_free_ctx_pg_tbls(struct bnge_dev * bd,struct bnge_ctx_pg_info * ctx_pg)201 static void bnge_free_ctx_pg_tbls(struct bnge_dev *bd,
202 struct bnge_ctx_pg_info *ctx_pg)
203 {
204 struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
205
206 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
207 ctx_pg->ctx_pg_tbl) {
208 int i, nr_tbls = rmem->nr_pages;
209
210 for (i = 0; i < nr_tbls; i++) {
211 struct bnge_ctx_pg_info *pg_tbl;
212 struct bnge_ring_mem_info *rmem2;
213
214 pg_tbl = ctx_pg->ctx_pg_tbl[i];
215 if (!pg_tbl)
216 continue;
217 rmem2 = &pg_tbl->ring_mem;
218 bnge_free_ring(bd, rmem2);
219 ctx_pg->ctx_pg_arr[i] = NULL;
220 kfree(pg_tbl);
221 ctx_pg->ctx_pg_tbl[i] = NULL;
222 }
223 kfree(ctx_pg->ctx_pg_tbl);
224 ctx_pg->ctx_pg_tbl = NULL;
225 }
226 bnge_free_ring(bd, rmem);
227 ctx_pg->nr_pages = 0;
228 }
229
bnge_setup_ctxm_pg_tbls(struct bnge_dev * bd,struct bnge_ctx_mem_type * ctxm,u32 entries,u8 pg_lvl)230 static int bnge_setup_ctxm_pg_tbls(struct bnge_dev *bd,
231 struct bnge_ctx_mem_type *ctxm, u32 entries,
232 u8 pg_lvl)
233 {
234 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info;
235 int i, rc = 0, n = 1;
236 u32 mem_size;
237
238 if (!ctxm->entry_size || !ctx_pg)
239 return -EINVAL;
240 if (ctxm->instance_bmap)
241 n = hweight32(ctxm->instance_bmap);
242 if (ctxm->entry_multiple)
243 entries = roundup(entries, ctxm->entry_multiple);
244 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
245 mem_size = entries * ctxm->entry_size;
246 for (i = 0; i < n && !rc; i++) {
247 ctx_pg[i].entries = entries;
248 rc = bnge_alloc_ctx_pg_tbls(bd, &ctx_pg[i], mem_size, pg_lvl,
249 ctxm->init_value ? ctxm : NULL);
250 }
251
252 return rc;
253 }
254
bnge_backing_store_cfg(struct bnge_dev * bd,u32 ena)255 static int bnge_backing_store_cfg(struct bnge_dev *bd, u32 ena)
256 {
257 struct bnge_ctx_mem_info *ctx = bd->ctx;
258 struct bnge_ctx_mem_type *ctxm;
259 u16 last_type;
260 int rc = 0;
261 u16 type;
262
263 if (!ena)
264 return 0;
265 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
266 last_type = BNGE_CTX_MAX - 1;
267 else
268 last_type = BNGE_CTX_L2_MAX - 1;
269 ctx->ctx_arr[last_type].last = 1;
270
271 for (type = 0 ; type < BNGE_CTX_V2_MAX; type++) {
272 ctxm = &ctx->ctx_arr[type];
273
274 rc = bnge_hwrm_func_backing_store(bd, ctxm, ctxm->last);
275 if (rc)
276 return rc;
277 }
278
279 return 0;
280 }
281
bnge_free_ctx_mem(struct bnge_dev * bd)282 void bnge_free_ctx_mem(struct bnge_dev *bd)
283 {
284 struct bnge_ctx_mem_info *ctx = bd->ctx;
285 u16 type;
286
287 if (!ctx)
288 return;
289
290 for (type = 0; type < BNGE_CTX_V2_MAX; type++) {
291 struct bnge_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
292 struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info;
293 int i, n = 1;
294
295 if (!ctx_pg)
296 continue;
297 if (ctxm->instance_bmap)
298 n = hweight32(ctxm->instance_bmap);
299 for (i = 0; i < n; i++)
300 bnge_free_ctx_pg_tbls(bd, &ctx_pg[i]);
301
302 kfree(ctx_pg);
303 ctxm->pg_info = NULL;
304 }
305
306 ctx->flags &= ~BNGE_CTX_FLAG_INITED;
307 kfree(ctx);
308 bd->ctx = NULL;
309 }
310
311 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
312 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
313 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
314 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
315 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
316 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
317
bnge_alloc_ctx_mem(struct bnge_dev * bd)318 int bnge_alloc_ctx_mem(struct bnge_dev *bd)
319 {
320 struct bnge_ctx_mem_type *ctxm;
321 struct bnge_ctx_mem_info *ctx;
322 u32 l2_qps, qp1_qps, max_qps;
323 u32 ena, entries_sp, entries;
324 u32 srqs, max_srqs, min;
325 u32 num_mr, num_ah;
326 u32 extra_srqs = 0;
327 u32 extra_qps = 0;
328 u32 fast_qpmd_qps;
329 u8 pg_lvl = 1;
330 int i, rc;
331
332 rc = bnge_hwrm_func_backing_store_qcaps(bd);
333 if (rc) {
334 dev_err(bd->dev, "Failed querying ctx mem caps, rc: %d\n", rc);
335 return rc;
336 }
337
338 ctx = bd->ctx;
339 if (!ctx || (ctx->flags & BNGE_CTX_FLAG_INITED))
340 return 0;
341
342 ctxm = &ctx->ctx_arr[BNGE_CTX_QP];
343 l2_qps = ctxm->qp_l2_entries;
344 qp1_qps = ctxm->qp_qp1_entries;
345 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
346 max_qps = ctxm->max_entries;
347 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ];
348 srqs = ctxm->srq_l2_entries;
349 max_srqs = ctxm->max_entries;
350 ena = 0;
351 if (bnge_is_roce_en(bd) && !is_kdump_kernel()) {
352 pg_lvl = 2;
353 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
354 /* allocate extra qps if fast qp destroy feature enabled */
355 extra_qps += fast_qpmd_qps;
356 extra_srqs = min_t(u32, 8192, max_srqs - srqs);
357 if (fast_qpmd_qps)
358 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
359 }
360
361 ctxm = &ctx->ctx_arr[BNGE_CTX_QP];
362 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps,
363 pg_lvl);
364 if (rc)
365 return rc;
366
367 ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ];
368 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, srqs + extra_srqs, pg_lvl);
369 if (rc)
370 return rc;
371
372 ctxm = &ctx->ctx_arr[BNGE_CTX_CQ];
373 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->cq_l2_entries +
374 extra_qps * 2, pg_lvl);
375 if (rc)
376 return rc;
377
378 ctxm = &ctx->ctx_arr[BNGE_CTX_VNIC];
379 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1);
380 if (rc)
381 return rc;
382
383 ctxm = &ctx->ctx_arr[BNGE_CTX_STAT];
384 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1);
385 if (rc)
386 return rc;
387
388 if (!bnge_is_roce_en(bd))
389 goto skip_rdma;
390
391 ctxm = &ctx->ctx_arr[BNGE_CTX_MRAV];
392 /* 128K extra is needed to accommodate static AH context
393 * allocation by f/w.
394 */
395 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
396 num_ah = min_t(u32, num_mr, 1024 * 128);
397 ctxm->split_entry_cnt = BNGE_CTX_MRAV_AV_SPLIT_ENTRY + 1;
398 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
399 ctxm->mrav_av_entries = num_ah;
400
401 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, num_mr + num_ah, 2);
402 if (rc)
403 return rc;
404 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
405
406 ctxm = &ctx->ctx_arr[BNGE_CTX_TIM];
407 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 1);
408 if (rc)
409 return rc;
410 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
411
412 skip_rdma:
413 ctxm = &ctx->ctx_arr[BNGE_CTX_STQM];
414 min = ctxm->min_entries;
415 entries_sp = ctx->ctx_arr[BNGE_CTX_VNIC].vnic_entries + l2_qps +
416 2 * (extra_qps + qp1_qps) + min;
417 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries_sp, 2);
418 if (rc)
419 return rc;
420
421 ctxm = &ctx->ctx_arr[BNGE_CTX_FTQM];
422 entries = l2_qps + 2 * (extra_qps + qp1_qps);
423 rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries, 2);
424 if (rc)
425 return rc;
426 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
427 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
428 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
429
430 rc = bnge_backing_store_cfg(bd, ena);
431 if (rc) {
432 dev_err(bd->dev, "Failed configuring ctx mem, rc: %d\n", rc);
433 return rc;
434 }
435 ctx->flags |= BNGE_CTX_FLAG_INITED;
436
437 return 0;
438 }
439