1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2025 Broadcom.
3
4 #include <linux/pci.h>
5 #include <linux/vmalloc.h>
6 #include <rdma/ib_umem.h>
7
8 #include <linux/bnxt/hsi.h>
9 #include "bng_res.h"
10 #include "roce_hsi.h"
11
12 /* Stats */
bng_re_free_stats_ctx_mem(struct pci_dev * pdev,struct bng_re_stats * stats)13 void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,
14 struct bng_re_stats *stats)
15 {
16 if (stats->dma) {
17 dma_free_coherent(&pdev->dev, stats->size,
18 stats->dma, stats->dma_map);
19 }
20 memset(stats, 0, sizeof(*stats));
21 stats->fw_id = -1;
22 }
23
bng_re_alloc_stats_ctx_mem(struct pci_dev * pdev,struct bng_re_chip_ctx * cctx,struct bng_re_stats * stats)24 int bng_re_alloc_stats_ctx_mem(struct pci_dev *pdev,
25 struct bng_re_chip_ctx *cctx,
26 struct bng_re_stats *stats)
27 {
28 memset(stats, 0, sizeof(*stats));
29 stats->fw_id = -1;
30 stats->size = cctx->hw_stats_size;
31 stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
32 &stats->dma_map, GFP_KERNEL);
33 if (!stats->dma)
34 return -ENOMEM;
35
36 return 0;
37 }
38
bng_free_pbl(struct bng_re_res * res,struct bng_re_pbl * pbl)39 static void bng_free_pbl(struct bng_re_res *res, struct bng_re_pbl *pbl)
40 {
41 struct pci_dev *pdev = res->pdev;
42 int i;
43
44 for (i = 0; i < pbl->pg_count; i++) {
45 if (pbl->pg_arr[i])
46 dma_free_coherent(&pdev->dev, pbl->pg_size,
47 (void *)((unsigned long)
48 pbl->pg_arr[i] &
49 PAGE_MASK),
50 pbl->pg_map_arr[i]);
51 else
52 dev_warn(&pdev->dev,
53 "PBL free pg_arr[%d] empty?!\n", i);
54 pbl->pg_arr[i] = NULL;
55 }
56
57 vfree(pbl->pg_arr);
58 pbl->pg_arr = NULL;
59 vfree(pbl->pg_map_arr);
60 pbl->pg_map_arr = NULL;
61 pbl->pg_count = 0;
62 pbl->pg_size = 0;
63 }
64
bng_alloc_pbl(struct bng_re_res * res,struct bng_re_pbl * pbl,struct bng_re_sg_info * sginfo)65 static int bng_alloc_pbl(struct bng_re_res *res,
66 struct bng_re_pbl *pbl,
67 struct bng_re_sg_info *sginfo)
68 {
69 struct pci_dev *pdev = res->pdev;
70 u32 pages;
71 int i;
72
73 if (sginfo->nopte)
74 return 0;
75 pages = sginfo->npages;
76
77 /* page ptr arrays */
78 pbl->pg_arr = vmalloc_array(pages, sizeof(void *));
79 if (!pbl->pg_arr)
80 return -ENOMEM;
81
82 pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));
83 if (!pbl->pg_map_arr) {
84 vfree(pbl->pg_arr);
85 pbl->pg_arr = NULL;
86 return -ENOMEM;
87 }
88 pbl->pg_count = 0;
89 pbl->pg_size = sginfo->pgsize;
90
91 for (i = 0; i < pages; i++) {
92 pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
93 pbl->pg_size,
94 &pbl->pg_map_arr[i],
95 GFP_KERNEL);
96 if (!pbl->pg_arr[i])
97 goto fail;
98 pbl->pg_count++;
99 }
100
101 return 0;
102 fail:
103 bng_free_pbl(res, pbl);
104 return -ENOMEM;
105 }
106
bng_re_free_hwq(struct bng_re_res * res,struct bng_re_hwq * hwq)107 void bng_re_free_hwq(struct bng_re_res *res,
108 struct bng_re_hwq *hwq)
109 {
110 int i;
111
112 if (!hwq->max_elements)
113 return;
114 if (hwq->level >= BNG_PBL_LVL_MAX)
115 return;
116
117 for (i = 0; i < hwq->level + 1; i++)
118 bng_free_pbl(res, &hwq->pbl[i]);
119
120 hwq->level = BNG_PBL_LVL_MAX;
121 hwq->max_elements = 0;
122 hwq->element_size = 0;
123 hwq->prod = 0;
124 hwq->cons = 0;
125 }
126
127 /* All HWQs are power of 2 in size */
bng_re_alloc_init_hwq(struct bng_re_hwq * hwq,struct bng_re_hwq_attr * hwq_attr)128 int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
129 struct bng_re_hwq_attr *hwq_attr)
130 {
131 u32 npages, pg_size;
132 struct bng_re_sg_info sginfo = {};
133 u32 depth, stride, npbl, npde;
134 dma_addr_t *src_phys_ptr, **dst_virt_ptr;
135 struct bng_re_res *res;
136 struct pci_dev *pdev;
137 int i, rc, lvl;
138
139 res = hwq_attr->res;
140 pdev = res->pdev;
141 pg_size = hwq_attr->sginfo->pgsize;
142 hwq->level = BNG_PBL_LVL_MAX;
143
144 depth = roundup_pow_of_two(hwq_attr->depth);
145 stride = roundup_pow_of_two(hwq_attr->stride);
146
147 npages = (depth * stride) / pg_size;
148 if ((depth * stride) % pg_size)
149 npages++;
150 if (!npages)
151 return -EINVAL;
152 hwq_attr->sginfo->npages = npages;
153
154 if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
155 /* This request is Level 0, map PTE */
156 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], hwq_attr->sginfo);
157 if (rc)
158 goto fail;
159 hwq->level = BNG_PBL_LVL_0;
160 goto done;
161 }
162
163 if (npages >= MAX_PBL_LVL_0_PGS) {
164 if (npages > MAX_PBL_LVL_1_PGS) {
165 u32 flag = PTU_PTE_VALID;
166 /* 2 levels of indirection */
167 npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
168 if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
169 npbl++;
170 npde = npbl >> MAX_PDL_LVL_SHIFT;
171 if (npbl % BIT(MAX_PDL_LVL_SHIFT))
172 npde++;
173 /* Alloc PDE pages */
174 sginfo.pgsize = npde * pg_size;
175 sginfo.npages = 1;
176 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], &sginfo);
177 if (rc)
178 goto fail;
179
180 /* Alloc PBL pages */
181 sginfo.npages = npbl;
182 sginfo.pgsize = PAGE_SIZE;
183 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_1], &sginfo);
184 if (rc)
185 goto fail;
186 /* Fill PDL with PBL page pointers */
187 dst_virt_ptr =
188 (dma_addr_t **)hwq->pbl[BNG_PBL_LVL_0].pg_arr;
189 src_phys_ptr = hwq->pbl[BNG_PBL_LVL_1].pg_map_arr;
190 for (i = 0; i < hwq->pbl[BNG_PBL_LVL_1].pg_count; i++)
191 dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;
192
193 /* Alloc or init PTEs */
194 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_2],
195 hwq_attr->sginfo);
196 if (rc)
197 goto fail;
198 hwq->level = BNG_PBL_LVL_2;
199 if (hwq_attr->sginfo->nopte)
200 goto done;
201 /* Fill PBLs with PTE pointers */
202 dst_virt_ptr =
203 (dma_addr_t **)hwq->pbl[BNG_PBL_LVL_1].pg_arr;
204 src_phys_ptr = hwq->pbl[BNG_PBL_LVL_2].pg_map_arr;
205 for (i = 0; i < hwq->pbl[BNG_PBL_LVL_2].pg_count; i++) {
206 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
207 src_phys_ptr[i] | PTU_PTE_VALID;
208 }
209 if (hwq_attr->type == BNG_HWQ_TYPE_QUEUE) {
210 /* Find the last pg of the size */
211 i = hwq->pbl[BNG_PBL_LVL_2].pg_count;
212 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
213 PTU_PTE_LAST;
214 if (i > 1)
215 dst_virt_ptr[PTR_PG(i - 2)]
216 [PTR_IDX(i - 2)] |=
217 PTU_PTE_NEXT_TO_LAST;
218 }
219 } else { /* pages < 512 npbl = 1, npde = 0 */
220 u32 flag = PTU_PTE_VALID;
221
222 /* 1 level of indirection */
223 npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
224 if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
225 npbl++;
226 sginfo.npages = npbl;
227 sginfo.pgsize = PAGE_SIZE;
228 /* Alloc PBL page */
229 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], &sginfo);
230 if (rc)
231 goto fail;
232 /* Alloc or init PTEs */
233 rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_1],
234 hwq_attr->sginfo);
235 if (rc)
236 goto fail;
237 hwq->level = BNG_PBL_LVL_1;
238 if (hwq_attr->sginfo->nopte)
239 goto done;
240 /* Fill PBL with PTE pointers */
241 dst_virt_ptr =
242 (dma_addr_t **)hwq->pbl[BNG_PBL_LVL_0].pg_arr;
243 src_phys_ptr = hwq->pbl[BNG_PBL_LVL_1].pg_map_arr;
244 for (i = 0; i < hwq->pbl[BNG_PBL_LVL_1].pg_count; i++)
245 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
246 src_phys_ptr[i] | flag;
247 if (hwq_attr->type == BNG_HWQ_TYPE_QUEUE) {
248 /* Find the last pg of the size */
249 i = hwq->pbl[BNG_PBL_LVL_1].pg_count;
250 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
251 PTU_PTE_LAST;
252 if (i > 1)
253 dst_virt_ptr[PTR_PG(i - 2)]
254 [PTR_IDX(i - 2)] |=
255 PTU_PTE_NEXT_TO_LAST;
256 }
257 }
258 }
259 done:
260 hwq->prod = 0;
261 hwq->cons = 0;
262 hwq->pdev = pdev;
263 hwq->depth = hwq_attr->depth;
264 hwq->max_elements = hwq->depth;
265 hwq->element_size = stride;
266 hwq->qe_ppg = pg_size / stride;
267 /* For direct access to the elements */
268 lvl = hwq->level;
269 if (hwq_attr->sginfo->nopte && hwq->level)
270 lvl = hwq->level - 1;
271 hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
272 hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
273 spin_lock_init(&hwq->lock);
274
275 return 0;
276 fail:
277 bng_re_free_hwq(res, hwq);
278 return -ENOMEM;
279 }
280