1 /*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: QPLib resource manager
37 */
38
39 #define dev_fmt(fmt) "QPLIB: " fmt
40
41 #include <linux/spinlock.h>
42 #include <linux/pci.h>
43 #include <linux/interrupt.h>
44 #include <linux/inetdevice.h>
45 #include <linux/dma-mapping.h>
46 #include <linux/if_vlan.h>
47 #include <linux/vmalloc.h>
48 #include <rdma/ib_verbs.h>
49 #include <rdma/ib_umem.h>
50
51 #include "roce_hsi.h"
52 #include "qplib_res.h"
53 #include "qplib_sp.h"
54 #include "qplib_rcfw.h"
55
56 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
57 struct bnxt_qplib_stats *stats);
58 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
59 struct bnxt_qplib_chip_ctx *cctx,
60 struct bnxt_qplib_stats *stats);
61
62 /* PBL */
__free_pbl(struct bnxt_qplib_res * res,struct bnxt_qplib_pbl * pbl,bool is_umem)63 static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
64 bool is_umem)
65 {
66 struct pci_dev *pdev = res->pdev;
67 int i;
68
69 if (!is_umem) {
70 for (i = 0; i < pbl->pg_count; i++) {
71 if (pbl->pg_arr[i])
72 dma_free_coherent(&pdev->dev, pbl->pg_size,
73 (void *)((unsigned long)
74 pbl->pg_arr[i] &
75 PAGE_MASK),
76 pbl->pg_map_arr[i]);
77 else
78 dev_warn(&pdev->dev,
79 "PBL free pg_arr[%d] empty?!\n", i);
80 pbl->pg_arr[i] = NULL;
81 }
82 }
83 vfree(pbl->pg_arr);
84 pbl->pg_arr = NULL;
85 vfree(pbl->pg_map_arr);
86 pbl->pg_map_arr = NULL;
87 pbl->pg_count = 0;
88 pbl->pg_size = 0;
89 }
90
bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl * pbl,struct bnxt_qplib_sg_info * sginfo)91 static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
92 struct bnxt_qplib_sg_info *sginfo)
93 {
94 struct ib_block_iter biter;
95 int i = 0;
96
97 rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) {
98 pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter);
99 pbl->pg_arr[i] = NULL;
100 pbl->pg_count++;
101 i++;
102 }
103 }
104
__alloc_pbl(struct bnxt_qplib_res * res,struct bnxt_qplib_pbl * pbl,struct bnxt_qplib_sg_info * sginfo)105 static int __alloc_pbl(struct bnxt_qplib_res *res,
106 struct bnxt_qplib_pbl *pbl,
107 struct bnxt_qplib_sg_info *sginfo)
108 {
109 struct pci_dev *pdev = res->pdev;
110 bool is_umem = false;
111 u32 pages;
112 int i;
113
114 if (sginfo->nopte)
115 return 0;
116 if (sginfo->umem)
117 pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize);
118 else
119 pages = sginfo->npages;
120 /* page ptr arrays */
121 pbl->pg_arr = vmalloc_array(pages, sizeof(void *));
122 if (!pbl->pg_arr)
123 return -ENOMEM;
124 memset(pbl->pg_arr, 0, pages * sizeof(void *));
125
126 pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));
127 if (!pbl->pg_map_arr) {
128 vfree(pbl->pg_arr);
129 pbl->pg_arr = NULL;
130 return -ENOMEM;
131 }
132 memset(pbl->pg_map_arr, 0, pages * sizeof(dma_addr_t));
133 pbl->pg_count = 0;
134 pbl->pg_size = sginfo->pgsize;
135
136 if (!sginfo->umem) {
137 for (i = 0; i < pages; i++) {
138 pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
139 pbl->pg_size,
140 &pbl->pg_map_arr[i],
141 GFP_KERNEL);
142 if (!pbl->pg_arr[i])
143 goto fail;
144 pbl->pg_count++;
145 }
146 } else {
147 is_umem = true;
148 bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
149 }
150
151 return 0;
152 fail:
153 __free_pbl(res, pbl, is_umem);
154 return -ENOMEM;
155 }
156
157 /* HWQ */
bnxt_qplib_free_hwq(struct bnxt_qplib_res * res,struct bnxt_qplib_hwq * hwq)158 void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
159 struct bnxt_qplib_hwq *hwq)
160 {
161 int i;
162
163 if (!hwq->max_elements)
164 return;
165 if (hwq->level >= PBL_LVL_MAX)
166 return;
167
168 for (i = 0; i < hwq->level + 1; i++) {
169 if (i == hwq->level)
170 __free_pbl(res, &hwq->pbl[i], hwq->is_user);
171 else
172 __free_pbl(res, &hwq->pbl[i], false);
173 }
174
175 hwq->level = PBL_LVL_MAX;
176 hwq->max_elements = 0;
177 hwq->element_size = 0;
178 hwq->prod = 0;
179 hwq->cons = 0;
180 hwq->cp_bit = 0;
181 }
182
183 /* All HWQs are power of 2 in size */
184
bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq * hwq,struct bnxt_qplib_hwq_attr * hwq_attr)185 int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
186 struct bnxt_qplib_hwq_attr *hwq_attr)
187 {
188 u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
189 struct bnxt_qplib_sg_info sginfo = {};
190 u32 depth, stride, npbl, npde;
191 dma_addr_t *src_phys_ptr, **dst_virt_ptr;
192 struct bnxt_qplib_res *res;
193 struct pci_dev *pdev;
194 int i, rc, lvl;
195
196 res = hwq_attr->res;
197 pdev = res->pdev;
198 pg_size = hwq_attr->sginfo->pgsize;
199 hwq->level = PBL_LVL_MAX;
200
201 depth = roundup_pow_of_two(hwq_attr->depth);
202 stride = roundup_pow_of_two(hwq_attr->stride);
203 if (hwq_attr->aux_depth) {
204 aux_slots = hwq_attr->aux_depth;
205 aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
206 aux_pages = (aux_slots * aux_size) / pg_size;
207 if ((aux_slots * aux_size) % pg_size)
208 aux_pages++;
209 }
210
211 if (!hwq_attr->sginfo->umem) {
212 hwq->is_user = false;
213 npages = (depth * stride) / pg_size + aux_pages;
214 if ((depth * stride) % pg_size)
215 npages++;
216 if (!npages)
217 return -EINVAL;
218 hwq_attr->sginfo->npages = npages;
219 } else {
220 npages = ib_umem_num_dma_blocks(hwq_attr->sginfo->umem,
221 hwq_attr->sginfo->pgsize);
222 hwq->is_user = true;
223 }
224
225 if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
226 /* This request is Level 0, map PTE */
227 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
228 if (rc)
229 goto fail;
230 hwq->level = PBL_LVL_0;
231 goto done;
232 }
233
234 if (npages >= MAX_PBL_LVL_0_PGS) {
235 if (npages > MAX_PBL_LVL_1_PGS) {
236 u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
237 0 : PTU_PTE_VALID;
238 /* 2 levels of indirection */
239 npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
240 if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
241 npbl++;
242 npde = npbl >> MAX_PDL_LVL_SHIFT;
243 if (npbl % BIT(MAX_PDL_LVL_SHIFT))
244 npde++;
245 /* Alloc PDE pages */
246 sginfo.pgsize = npde * pg_size;
247 sginfo.npages = 1;
248 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
249 if (rc)
250 goto fail;
251
252 /* Alloc PBL pages */
253 sginfo.npages = npbl;
254 sginfo.pgsize = PAGE_SIZE;
255 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
256 if (rc)
257 goto fail;
258 /* Fill PDL with PBL page pointers */
259 dst_virt_ptr =
260 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
261 src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
262 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
263 dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;
264
265 /* Alloc or init PTEs */
266 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
267 hwq_attr->sginfo);
268 if (rc)
269 goto fail;
270 hwq->level = PBL_LVL_2;
271 if (hwq_attr->sginfo->nopte)
272 goto done;
273 /* Fill PBLs with PTE pointers */
274 dst_virt_ptr =
275 (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
276 src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
277 for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
278 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
279 src_phys_ptr[i] | PTU_PTE_VALID;
280 }
281 if (hwq_attr->type == HWQ_TYPE_QUEUE) {
282 /* Find the last pg of the size */
283 i = hwq->pbl[PBL_LVL_2].pg_count;
284 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
285 PTU_PTE_LAST;
286 if (i > 1)
287 dst_virt_ptr[PTR_PG(i - 2)]
288 [PTR_IDX(i - 2)] |=
289 PTU_PTE_NEXT_TO_LAST;
290 }
291 } else { /* pages < 512 npbl = 1, npde = 0 */
292 u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
293 0 : PTU_PTE_VALID;
294
295 /* 1 level of indirection */
296 npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
297 if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
298 npbl++;
299 sginfo.npages = npbl;
300 sginfo.pgsize = PAGE_SIZE;
301 /* Alloc PBL page */
302 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
303 if (rc)
304 goto fail;
305 /* Alloc or init PTEs */
306 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
307 hwq_attr->sginfo);
308 if (rc)
309 goto fail;
310 hwq->level = PBL_LVL_1;
311 if (hwq_attr->sginfo->nopte)
312 goto done;
313 /* Fill PBL with PTE pointers */
314 dst_virt_ptr =
315 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
316 src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
317 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
318 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
319 src_phys_ptr[i] | flag;
320 if (hwq_attr->type == HWQ_TYPE_QUEUE) {
321 /* Find the last pg of the size */
322 i = hwq->pbl[PBL_LVL_1].pg_count;
323 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
324 PTU_PTE_LAST;
325 if (i > 1)
326 dst_virt_ptr[PTR_PG(i - 2)]
327 [PTR_IDX(i - 2)] |=
328 PTU_PTE_NEXT_TO_LAST;
329 }
330 }
331 }
332 done:
333 hwq->prod = 0;
334 hwq->cons = 0;
335 hwq->pdev = pdev;
336 hwq->depth = hwq_attr->depth;
337 hwq->max_elements = hwq->depth;
338 hwq->element_size = stride;
339 hwq->qe_ppg = pg_size / stride;
340 /* For direct access to the elements */
341 lvl = hwq->level;
342 if (hwq_attr->sginfo->nopte && hwq->level)
343 lvl = hwq->level - 1;
344 hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
345 hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
346 spin_lock_init(&hwq->lock);
347
348 return 0;
349 fail:
350 bnxt_qplib_free_hwq(res, hwq);
351 return -ENOMEM;
352 }
353
354 /* Context Tables */
bnxt_qplib_free_ctx(struct bnxt_qplib_res * res,struct bnxt_qplib_ctx * ctx)355 void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
356 struct bnxt_qplib_ctx *ctx)
357 {
358 int i;
359
360 bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
361 bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
362 bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
363 bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
364 bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
365 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
366 bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
367 /* restore original pde level before destroy */
368 ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
369 bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
370 bnxt_qplib_free_stats_ctx(res->pdev, &ctx->stats);
371 }
372
bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res * res,struct bnxt_qplib_ctx * ctx)373 static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
374 struct bnxt_qplib_ctx *ctx)
375 {
376 struct bnxt_qplib_hwq_attr hwq_attr = {};
377 struct bnxt_qplib_sg_info sginfo = {};
378 struct bnxt_qplib_tqm_ctx *tqmctx;
379 int rc;
380 int i;
381
382 tqmctx = &ctx->tqm_ctx;
383
384 sginfo.pgsize = PAGE_SIZE;
385 sginfo.pgshft = PAGE_SHIFT;
386 hwq_attr.sginfo = &sginfo;
387 hwq_attr.res = res;
388 hwq_attr.type = HWQ_TYPE_CTX;
389 hwq_attr.depth = 512;
390 hwq_attr.stride = sizeof(u64);
391 /* Alloc pdl buffer */
392 rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
393 if (rc)
394 goto out;
395 /* Save original pdl level */
396 tqmctx->pde_level = tqmctx->pde.level;
397
398 hwq_attr.stride = 1;
399 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
400 if (!tqmctx->qcount[i])
401 continue;
402 hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
403 rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
404 if (rc)
405 goto out;
406 }
407 out:
408 return rc;
409 }
410
bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx * ctx)411 static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
412 {
413 struct bnxt_qplib_hwq *tbl;
414 dma_addr_t *dma_ptr;
415 __le64 **pbl_ptr, *ptr;
416 int i, j, k;
417 int fnz_idx = -1;
418 int pg_count;
419
420 pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
421
422 for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
423 i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
424 tbl = &ctx->qtbl[i];
425 if (!tbl->max_elements)
426 continue;
427 if (fnz_idx == -1)
428 fnz_idx = i; /* first non-zero index */
429 switch (tbl->level) {
430 case PBL_LVL_2:
431 pg_count = tbl->pbl[PBL_LVL_1].pg_count;
432 for (k = 0; k < pg_count; k++) {
433 ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
434 dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
435 *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
436 }
437 break;
438 case PBL_LVL_1:
439 case PBL_LVL_0:
440 default:
441 ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
442 *ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
443 PTU_PTE_VALID);
444 break;
445 }
446 }
447 if (fnz_idx == -1)
448 fnz_idx = 0;
449 /* update pde level as per page table programming */
450 ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
451 ctx->qtbl[fnz_idx].level + 1;
452 }
453
bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res * res,struct bnxt_qplib_ctx * ctx)454 static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
455 struct bnxt_qplib_ctx *ctx)
456 {
457 int rc;
458
459 rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
460 if (rc)
461 goto fail;
462
463 bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
464 fail:
465 return rc;
466 }
467
468 /*
469 * Routine: bnxt_qplib_alloc_ctx
470 * Description:
471 * Context tables are memories which are used by the chip fw.
472 * The 6 tables defined are:
473 * QPC ctx - holds QP states
474 * MRW ctx - holds memory region and window
475 * SRQ ctx - holds shared RQ states
476 * CQ ctx - holds completion queue states
477 * TQM ctx - holds Tx Queue Manager context
478 * TIM ctx - holds timer context
479 * Depending on the size of the tbl requested, either a 1 Page Buffer List
480 * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
481 * instead.
482 * Table might be employed as follows:
483 * For 0 < ctx size <= 1 PAGE, 0 level of ind is used
484 * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
485 * For 512 < ctx size <= MAX, 2 levels of ind is used
486 * Returns:
487 * 0 if success, else -ERRORS
488 */
bnxt_qplib_alloc_ctx(struct bnxt_qplib_res * res,struct bnxt_qplib_ctx * ctx,bool virt_fn,bool is_p5)489 int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
490 struct bnxt_qplib_ctx *ctx,
491 bool virt_fn, bool is_p5)
492 {
493 struct bnxt_qplib_hwq_attr hwq_attr = {};
494 struct bnxt_qplib_sg_info sginfo = {};
495 int rc;
496
497 if (virt_fn || is_p5)
498 goto stats_alloc;
499
500 /* QPC Tables */
501 sginfo.pgsize = PAGE_SIZE;
502 sginfo.pgshft = PAGE_SHIFT;
503 hwq_attr.sginfo = &sginfo;
504
505 hwq_attr.res = res;
506 hwq_attr.depth = ctx->qpc_count;
507 hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
508 hwq_attr.type = HWQ_TYPE_CTX;
509 rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
510 if (rc)
511 goto fail;
512
513 /* MRW Tables */
514 hwq_attr.depth = ctx->mrw_count;
515 hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
516 rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
517 if (rc)
518 goto fail;
519
520 /* SRQ Tables */
521 hwq_attr.depth = ctx->srqc_count;
522 hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
523 rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
524 if (rc)
525 goto fail;
526
527 /* CQ Tables */
528 hwq_attr.depth = ctx->cq_count;
529 hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
530 rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
531 if (rc)
532 goto fail;
533
534 /* TQM Buffer */
535 rc = bnxt_qplib_setup_tqm_rings(res, ctx);
536 if (rc)
537 goto fail;
538 /* TIM Buffer */
539 ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
540 hwq_attr.depth = ctx->qpc_count * 16;
541 hwq_attr.stride = 1;
542 rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
543 if (rc)
544 goto fail;
545 stats_alloc:
546 /* Stats */
547 rc = bnxt_qplib_alloc_stats_ctx(res->pdev, res->cctx, &ctx->stats);
548 if (rc)
549 goto fail;
550
551 return 0;
552
553 fail:
554 bnxt_qplib_free_ctx(res, ctx);
555 return rc;
556 }
557
bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_sgid_tbl * sgid_tbl)558 static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
559 struct bnxt_qplib_sgid_tbl *sgid_tbl)
560 {
561 kfree(sgid_tbl->tbl);
562 kfree(sgid_tbl->hw_id);
563 kfree(sgid_tbl->ctx);
564 kfree(sgid_tbl->vlan);
565 sgid_tbl->tbl = NULL;
566 sgid_tbl->hw_id = NULL;
567 sgid_tbl->ctx = NULL;
568 sgid_tbl->vlan = NULL;
569 sgid_tbl->max = 0;
570 sgid_tbl->active = 0;
571 }
572
bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_sgid_tbl * sgid_tbl,u16 max)573 static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
574 struct bnxt_qplib_sgid_tbl *sgid_tbl,
575 u16 max)
576 {
577 sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
578 if (!sgid_tbl->tbl)
579 return -ENOMEM;
580
581 sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
582 if (!sgid_tbl->hw_id)
583 goto out_free1;
584
585 sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
586 if (!sgid_tbl->ctx)
587 goto out_free2;
588
589 sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
590 if (!sgid_tbl->vlan)
591 goto out_free3;
592
593 sgid_tbl->max = max;
594 return 0;
595 out_free3:
596 kfree(sgid_tbl->ctx);
597 sgid_tbl->ctx = NULL;
598 out_free2:
599 kfree(sgid_tbl->hw_id);
600 sgid_tbl->hw_id = NULL;
601 out_free1:
602 kfree(sgid_tbl->tbl);
603 sgid_tbl->tbl = NULL;
604 return -ENOMEM;
605 };
606
bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_sgid_tbl * sgid_tbl)607 static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
608 struct bnxt_qplib_sgid_tbl *sgid_tbl)
609 {
610 int i;
611
612 for (i = 0; i < sgid_tbl->max; i++) {
613 if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
614 sizeof(bnxt_qplib_gid_zero)))
615 bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
616 sgid_tbl->tbl[i].vlan_id, true);
617 }
618 memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
619 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
620 memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
621 sgid_tbl->active = 0;
622 }
623
bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl * sgid_tbl,struct net_device * netdev)624 static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
625 struct net_device *netdev)
626 {
627 u32 i;
628
629 for (i = 0; i < sgid_tbl->max; i++)
630 sgid_tbl->tbl[i].vlan_id = 0xffff;
631
632 memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
633 }
634
635 /* PDs */
bnxt_qplib_alloc_pd(struct bnxt_qplib_res * res,struct bnxt_qplib_pd * pd)636 int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd)
637 {
638 struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl;
639 u32 bit_num;
640 int rc = 0;
641
642 mutex_lock(&res->pd_tbl_lock);
643 bit_num = find_first_bit(pdt->tbl, pdt->max);
644 if (bit_num == pdt->max) {
645 rc = -ENOMEM;
646 goto exit;
647 }
648
649 /* Found unused PD */
650 clear_bit(bit_num, pdt->tbl);
651 pd->id = bit_num;
652 exit:
653 mutex_unlock(&res->pd_tbl_lock);
654 return rc;
655 }
656
bnxt_qplib_dealloc_pd(struct bnxt_qplib_res * res,struct bnxt_qplib_pd_tbl * pdt,struct bnxt_qplib_pd * pd)657 int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
658 struct bnxt_qplib_pd_tbl *pdt,
659 struct bnxt_qplib_pd *pd)
660 {
661 int rc = 0;
662
663 mutex_lock(&res->pd_tbl_lock);
664 if (test_and_set_bit(pd->id, pdt->tbl)) {
665 dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
666 pd->id);
667 rc = -EINVAL;
668 goto exit;
669 }
670 pd->id = 0;
671 exit:
672 mutex_unlock(&res->pd_tbl_lock);
673 return rc;
674 }
675
bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl * pdt)676 static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
677 {
678 kfree(pdt->tbl);
679 pdt->tbl = NULL;
680 pdt->max = 0;
681 }
682
bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_pd_tbl * pdt,u32 max)683 static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
684 struct bnxt_qplib_pd_tbl *pdt,
685 u32 max)
686 {
687 u32 bytes;
688
689 bytes = max >> 3;
690 if (!bytes)
691 bytes = 1;
692 pdt->tbl = kmalloc(bytes, GFP_KERNEL);
693 if (!pdt->tbl)
694 return -ENOMEM;
695
696 pdt->max = max;
697 memset((u8 *)pdt->tbl, 0xFF, bytes);
698 mutex_init(&res->pd_tbl_lock);
699
700 return 0;
701 }
702
703 /* DPIs */
bnxt_qplib_alloc_dpi(struct bnxt_qplib_res * res,struct bnxt_qplib_dpi * dpi,void * app,u8 type)704 int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
705 struct bnxt_qplib_dpi *dpi,
706 void *app, u8 type)
707 {
708 struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
709 struct bnxt_qplib_reg_desc *reg;
710 u32 bit_num;
711 u64 umaddr;
712
713 reg = &dpit->wcreg;
714 mutex_lock(&res->dpi_tbl_lock);
715
716 bit_num = find_first_bit(dpit->tbl, dpit->max);
717 if (bit_num == dpit->max) {
718 mutex_unlock(&res->dpi_tbl_lock);
719 return -ENOMEM;
720 }
721
722 /* Found unused DPI */
723 clear_bit(bit_num, dpit->tbl);
724 dpit->app_tbl[bit_num] = app;
725
726 dpi->bit = bit_num;
727 dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
728
729 umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
730 dpi->umdbr = umaddr;
731
732 switch (type) {
733 case BNXT_QPLIB_DPI_TYPE_KERNEL:
734 /* privileged dbr was already mapped just initialize it. */
735 dpi->umdbr = dpit->ucreg.bar_base +
736 dpit->ucreg.offset + bit_num * PAGE_SIZE;
737 dpi->dbr = dpit->priv_db;
738 dpi->dpi = dpi->bit;
739 break;
740 case BNXT_QPLIB_DPI_TYPE_WC:
741 dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
742 break;
743 default:
744 dpi->dbr = ioremap(umaddr, PAGE_SIZE);
745 break;
746 }
747
748 dpi->type = type;
749 mutex_unlock(&res->dpi_tbl_lock);
750 return 0;
751
752 }
753
bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res * res,struct bnxt_qplib_dpi * dpi)754 int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
755 struct bnxt_qplib_dpi *dpi)
756 {
757 struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
758
759 mutex_lock(&res->dpi_tbl_lock);
760 if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL)
761 pci_iounmap(res->pdev, dpi->dbr);
762
763 if (test_and_set_bit(dpi->bit, dpit->tbl)) {
764 dev_warn(&res->pdev->dev,
765 "Freeing an unused DPI? dpi = %d, bit = %d\n",
766 dpi->dpi, dpi->bit);
767 mutex_unlock(&res->dpi_tbl_lock);
768 return -EINVAL;
769 }
770 if (dpit->app_tbl)
771 dpit->app_tbl[dpi->bit] = NULL;
772 memset(dpi, 0, sizeof(*dpi));
773 mutex_unlock(&res->dpi_tbl_lock);
774 return 0;
775 }
776
bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_dpi_tbl * dpit)777 static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res *res,
778 struct bnxt_qplib_dpi_tbl *dpit)
779 {
780 kfree(dpit->tbl);
781 kfree(dpit->app_tbl);
782 dpit->tbl = NULL;
783 dpit->app_tbl = NULL;
784 dpit->max = 0;
785 }
786
bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res * res,struct bnxt_qplib_dev_attr * dev_attr)787 static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
788 struct bnxt_qplib_dev_attr *dev_attr)
789 {
790 struct bnxt_qplib_dpi_tbl *dpit;
791 struct bnxt_qplib_reg_desc *reg;
792 unsigned long bar_len;
793 u32 dbr_offset;
794 u32 bytes;
795
796 dpit = &res->dpi_tbl;
797 reg = &dpit->wcreg;
798
799 if (!bnxt_qplib_is_chip_gen_p5_p7(res->cctx)) {
800 /* Offest should come from L2 driver */
801 dbr_offset = dev_attr->l2_db_size;
802 dpit->ucreg.offset = dbr_offset;
803 dpit->wcreg.offset = dbr_offset;
804 }
805
806 bar_len = pci_resource_len(res->pdev, reg->bar_id);
807 dpit->max = (bar_len - reg->offset) / PAGE_SIZE;
808 if (dev_attr->max_dpi)
809 dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
810
811 dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
812 if (!dpit->app_tbl)
813 return -ENOMEM;
814
815 bytes = dpit->max >> 3;
816 if (!bytes)
817 bytes = 1;
818
819 dpit->tbl = kmalloc(bytes, GFP_KERNEL);
820 if (!dpit->tbl) {
821 kfree(dpit->app_tbl);
822 dpit->app_tbl = NULL;
823 return -ENOMEM;
824 }
825
826 memset((u8 *)dpit->tbl, 0xFF, bytes);
827 mutex_init(&res->dpi_tbl_lock);
828 dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
829
830 return 0;
831
832 }
833
834 /* Stats */
bnxt_qplib_free_stats_ctx(struct pci_dev * pdev,struct bnxt_qplib_stats * stats)835 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
836 struct bnxt_qplib_stats *stats)
837 {
838 if (stats->dma) {
839 dma_free_coherent(&pdev->dev, stats->size,
840 stats->dma, stats->dma_map);
841 }
842 memset(stats, 0, sizeof(*stats));
843 stats->fw_id = -1;
844 }
845
bnxt_qplib_alloc_stats_ctx(struct pci_dev * pdev,struct bnxt_qplib_chip_ctx * cctx,struct bnxt_qplib_stats * stats)846 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
847 struct bnxt_qplib_chip_ctx *cctx,
848 struct bnxt_qplib_stats *stats)
849 {
850 memset(stats, 0, sizeof(*stats));
851 stats->fw_id = -1;
852 stats->size = cctx->hw_stats_size;
853 stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
854 &stats->dma_map, GFP_KERNEL);
855 if (!stats->dma) {
856 dev_err(&pdev->dev, "Stats DMA allocation failed\n");
857 return -ENOMEM;
858 }
859 return 0;
860 }
861
bnxt_qplib_cleanup_res(struct bnxt_qplib_res * res)862 void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
863 {
864 bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
865 }
866
bnxt_qplib_init_res(struct bnxt_qplib_res * res)867 int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
868 {
869 bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
870
871 return 0;
872 }
873
bnxt_qplib_free_res(struct bnxt_qplib_res * res)874 void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
875 {
876 kfree(res->rcfw->qp_tbl);
877 bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
878 bnxt_qplib_free_pd_tbl(&res->pd_tbl);
879 bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
880 }
881
bnxt_qplib_alloc_res(struct bnxt_qplib_res * res,struct net_device * netdev)882 int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev)
883 {
884 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
885 struct bnxt_qplib_dev_attr *dev_attr;
886 int rc;
887
888 res->netdev = netdev;
889 dev_attr = res->dattr;
890
891 /* Allocate one extra to hold the QP1 entries */
892 rcfw->qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr->max_qp);
893 rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),
894 GFP_KERNEL);
895 if (!rcfw->qp_tbl)
896 return -ENOMEM;
897
898 rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
899 if (rc)
900 goto fail;
901
902 rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
903 if (rc)
904 goto fail;
905
906 rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr);
907 if (rc)
908 goto fail;
909
910 return 0;
911 fail:
912 bnxt_qplib_free_res(res);
913 return rc;
914 }
915
bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res * res)916 void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res)
917 {
918 struct bnxt_qplib_reg_desc *reg;
919
920 reg = &res->dpi_tbl.ucreg;
921 if (reg->bar_reg)
922 pci_iounmap(res->pdev, reg->bar_reg);
923 reg->bar_reg = NULL;
924 reg->bar_base = 0;
925 reg->len = 0;
926 reg->bar_id = 0;
927 }
928
bnxt_qplib_map_db_bar(struct bnxt_qplib_res * res)929 int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res)
930 {
931 struct bnxt_qplib_reg_desc *ucreg;
932 struct bnxt_qplib_reg_desc *wcreg;
933
934 wcreg = &res->dpi_tbl.wcreg;
935 wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
936 wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id);
937
938 ucreg = &res->dpi_tbl.ucreg;
939 ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
940 ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id);
941 ucreg->len = ucreg->offset + PAGE_SIZE;
942 if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) {
943 dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d",
944 (int)ucreg->len);
945 return -EINVAL;
946 }
947 ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len);
948 if (!ucreg->bar_reg) {
949 dev_err(&res->pdev->dev, "privileged dpi map failed!");
950 return -ENOMEM;
951 }
952
953 return 0;
954 }
955
bnxt_qplib_determine_atomics(struct pci_dev * dev)956 int bnxt_qplib_determine_atomics(struct pci_dev *dev)
957 {
958 int comp;
959 u16 ctl2;
960
961 comp = pci_enable_atomic_ops_to_root(dev,
962 PCI_EXP_DEVCAP2_ATOMIC_COMP32);
963 if (comp)
964 return -EOPNOTSUPP;
965 comp = pci_enable_atomic_ops_to_root(dev,
966 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
967 if (comp)
968 return -EOPNOTSUPP;
969 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
970 return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
971 }
972