1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Marvell RVU Admin Function driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/pci.h>
10 #include "rvu_struct.h"
11 #include "rvu_reg.h"
12 #include "mbox.h"
13 #include "rvu.h"
14
15 /* CPT PF device id */
16 #define PCI_DEVID_OTX2_CPT_PF 0xA0FD
17 #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2
18
19 /* Length of initial context fetch in 128 byte words */
20 #define CPT_CTX_ILEN 1ULL
21
22 /* Interrupt vector count of CPT RVU and RAS interrupts */
23 #define CPT_10K_AF_RVU_RAS_INT_VEC_CNT 2
24
25 /* Default CPT_AF_RXC_CFG1:max_rxc_icb_cnt */
26 #define CPT_DFLT_MAX_RXC_ICB_CNT 0xC0ULL
27
28 #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \
29 ({ \
30 u64 free_sts = 0, busy_sts = 0; \
31 typeof(rsp) _rsp = rsp; \
32 u32 e, i; \
33 \
34 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \
35 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \
36 if (reg & 0x1) \
37 busy_sts |= 1ULL << i; \
38 \
39 if (reg & 0x2) \
40 free_sts |= 1ULL << i; \
41 } \
42 (_rsp)->busy_sts_##etype = busy_sts; \
43 (_rsp)->free_sts_##etype = free_sts; \
44 })
45
46 #define MAX_AE GENMASK_ULL(47, 32)
47 #define MAX_IE GENMASK_ULL(31, 16)
48 #define MAX_SE GENMASK_ULL(15, 0)
49
cpt_max_engines_get(struct rvu * rvu)50 static u16 cpt_max_engines_get(struct rvu *rvu)
51 {
52 u16 max_ses, max_ies, max_aes;
53 u64 reg;
54
55 reg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS1);
56 max_ses = FIELD_GET(MAX_SE, reg);
57 max_ies = FIELD_GET(MAX_IE, reg);
58 max_aes = FIELD_GET(MAX_AE, reg);
59
60 return max_ses + max_ies + max_aes;
61 }
62
63 /* Number of flt interrupt vectors are depends on number of engines that the
64 * chip has. Each flt vector represents 64 engines.
65 */
cpt_10k_flt_nvecs_get(struct rvu * rvu,u16 max_engs)66 static int cpt_10k_flt_nvecs_get(struct rvu *rvu, u16 max_engs)
67 {
68 int flt_vecs;
69
70 flt_vecs = DIV_ROUND_UP(max_engs, 64);
71
72 if (flt_vecs > CPT_10K_AF_INT_VEC_FLT_MAX) {
73 dev_warn_once(rvu->dev, "flt_vecs:%d exceeds the max vectors:%d\n",
74 flt_vecs, CPT_10K_AF_INT_VEC_FLT_MAX);
75 flt_vecs = CPT_10K_AF_INT_VEC_FLT_MAX;
76 }
77
78 return flt_vecs;
79 }
80
cpt_af_flt_intr_handler(int vec,void * ptr)81 static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr)
82 {
83 struct rvu_block *block = ptr;
84 struct rvu *rvu = block->rvu;
85 int blkaddr = block->addr;
86 u64 reg, val;
87 int i, eng;
88 u8 grp;
89
90 reg = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(vec));
91 dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx", vec, reg);
92
93 i = -1;
94 while ((i = find_next_bit((unsigned long *)®, 64, i + 1)) < 64) {
95 switch (vec) {
96 case 0:
97 eng = i;
98 break;
99 case 1:
100 eng = i + 64;
101 break;
102 case 2:
103 eng = i + 128;
104 break;
105 }
106 grp = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF;
107 /* Disable and enable the engine which triggers fault */
108 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), 0x0);
109 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng));
110 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val & ~1ULL);
111
112 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), grp);
113 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val | 1ULL);
114
115 spin_lock(&rvu->cpt_intr_lock);
116 block->cpt_flt_eng_map[vec] |= BIT_ULL(i);
117 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(eng));
118 val = val & 0x3;
119 if (val == 0x1 || val == 0x2)
120 block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i);
121 spin_unlock(&rvu->cpt_intr_lock);
122 }
123 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(vec), reg);
124
125 return IRQ_HANDLED;
126 }
127
rvu_cpt_af_flt0_intr_handler(int irq,void * ptr)128 static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr)
129 {
130 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT0, ptr);
131 }
132
rvu_cpt_af_flt1_intr_handler(int irq,void * ptr)133 static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr)
134 {
135 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT1, ptr);
136 }
137
rvu_cpt_af_flt2_intr_handler(int irq,void * ptr)138 static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr)
139 {
140 return cpt_af_flt_intr_handler(CPT_10K_AF_INT_VEC_FLT2, ptr);
141 }
142
rvu_cpt_af_rvu_intr_handler(int irq,void * ptr)143 static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr)
144 {
145 struct rvu_block *block = ptr;
146 struct rvu *rvu = block->rvu;
147 int blkaddr = block->addr;
148 u64 reg;
149
150 reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT);
151 dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg);
152
153 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg);
154 return IRQ_HANDLED;
155 }
156
rvu_cpt_af_ras_intr_handler(int irq,void * ptr)157 static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr)
158 {
159 struct rvu_block *block = ptr;
160 struct rvu *rvu = block->rvu;
161 int blkaddr = block->addr;
162 u64 reg;
163
164 reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT);
165 dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg);
166
167 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg);
168 return IRQ_HANDLED;
169 }
170
rvu_cpt_do_register_interrupt(struct rvu_block * block,int irq_offs,irq_handler_t handler,const char * name)171 static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs,
172 irq_handler_t handler,
173 const char *name)
174 {
175 struct rvu *rvu = block->rvu;
176 int ret;
177
178 ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0,
179 name, block);
180 if (ret) {
181 dev_err(rvu->dev, "RVUAF: %s irq registration failed", name);
182 return ret;
183 }
184
185 WARN_ON(rvu->irq_allocated[irq_offs]);
186 rvu->irq_allocated[irq_offs] = true;
187 return 0;
188 }
189
cpt_10k_unregister_interrupts(struct rvu_block * block,int off)190 static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off)
191 {
192 struct rvu *rvu = block->rvu;
193 int blkaddr = block->addr;
194 int i, flt_vecs;
195 u16 max_engs;
196 u8 nr;
197
198 max_engs = cpt_max_engines_get(rvu);
199 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);
200
201 /* Disable all CPT AF interrupts */
202 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) {
203 nr = (max_engs > 64) ? 64 : max_engs;
204 max_engs -= nr;
205 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i),
206 INTR_MASK(nr));
207 }
208
209 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
210 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
211
212 /* CPT AF interrupt vectors are flt_int, rvu_int and ras_int. */
213 for (i = 0; i < flt_vecs + CPT_10K_AF_RVU_RAS_INT_VEC_CNT; i++)
214 if (rvu->irq_allocated[off + i]) {
215 free_irq(pci_irq_vector(rvu->pdev, off + i), block);
216 rvu->irq_allocated[off + i] = false;
217 }
218 }
219
cpt_unregister_interrupts(struct rvu * rvu,int blkaddr)220 static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr)
221 {
222 struct rvu_hwinfo *hw = rvu->hw;
223 struct rvu_block *block;
224 int i, offs;
225
226 if (!is_block_implemented(rvu->hw, blkaddr))
227 return;
228 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
229 if (!offs) {
230 dev_warn(rvu->dev,
231 "Failed to get CPT_AF_INT vector offsets\n");
232 return;
233 }
234 block = &hw->block[blkaddr];
235 if (!is_rvu_otx2(rvu))
236 return cpt_10k_unregister_interrupts(block, offs);
237
238 /* Disable all CPT AF interrupts */
239 for (i = 0; i < CPT_AF_INT_VEC_RVU; i++)
240 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), ~0ULL);
241 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
242 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
243
244 for (i = 0; i < CPT_AF_INT_VEC_CNT; i++)
245 if (rvu->irq_allocated[offs + i]) {
246 free_irq(pci_irq_vector(rvu->pdev, offs + i), block);
247 rvu->irq_allocated[offs + i] = false;
248 }
249 }
250
rvu_cpt_unregister_interrupts(struct rvu * rvu)251 void rvu_cpt_unregister_interrupts(struct rvu *rvu)
252 {
253 cpt_unregister_interrupts(rvu, BLKADDR_CPT0);
254 cpt_unregister_interrupts(rvu, BLKADDR_CPT1);
255 }
256
cpt_10k_register_interrupts(struct rvu_block * block,int off)257 static int cpt_10k_register_interrupts(struct rvu_block *block, int off)
258 {
259 int rvu_intr_vec, ras_intr_vec;
260 struct rvu *rvu = block->rvu;
261 int blkaddr = block->addr;
262 irq_handler_t flt_fn;
263 int i, ret, flt_vecs;
264 u16 max_engs;
265 u8 nr;
266
267 max_engs = cpt_max_engines_get(rvu);
268 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);
269
270 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) {
271 sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i);
272
273 switch (i) {
274 case CPT_10K_AF_INT_VEC_FLT0:
275 flt_fn = rvu_cpt_af_flt0_intr_handler;
276 break;
277 case CPT_10K_AF_INT_VEC_FLT1:
278 flt_fn = rvu_cpt_af_flt1_intr_handler;
279 break;
280 case CPT_10K_AF_INT_VEC_FLT2:
281 flt_fn = rvu_cpt_af_flt2_intr_handler;
282 break;
283 }
284 ret = rvu_cpt_do_register_interrupt(block, off + i,
285 flt_fn, &rvu->irq_name[(off + i) * NAME_SIZE]);
286 if (ret)
287 goto err;
288
289 nr = (max_engs > 64) ? 64 : max_engs;
290 max_engs -= nr;
291 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i),
292 INTR_MASK(nr));
293 }
294
295 rvu_intr_vec = flt_vecs;
296 ras_intr_vec = rvu_intr_vec + 1;
297
298 ret = rvu_cpt_do_register_interrupt(block, off + rvu_intr_vec,
299 rvu_cpt_af_rvu_intr_handler,
300 "CPTAF RVU");
301 if (ret)
302 goto err;
303 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
304
305 ret = rvu_cpt_do_register_interrupt(block, off + ras_intr_vec,
306 rvu_cpt_af_ras_intr_handler,
307 "CPTAF RAS");
308 if (ret)
309 goto err;
310 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
311
312 return 0;
313 err:
314 rvu_cpt_unregister_interrupts(rvu);
315 return ret;
316 }
317
cpt_register_interrupts(struct rvu * rvu,int blkaddr)318 static int cpt_register_interrupts(struct rvu *rvu, int blkaddr)
319 {
320 struct rvu_hwinfo *hw = rvu->hw;
321 struct rvu_block *block;
322 irq_handler_t flt_fn;
323 int i, offs, ret = 0;
324
325 if (!is_block_implemented(rvu->hw, blkaddr))
326 return 0;
327
328 block = &hw->block[blkaddr];
329 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
330 if (!offs) {
331 dev_warn(rvu->dev,
332 "Failed to get CPT_AF_INT vector offsets\n");
333 return 0;
334 }
335
336 if (!is_rvu_otx2(rvu))
337 return cpt_10k_register_interrupts(block, offs);
338
339 for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) {
340 sprintf(&rvu->irq_name[(offs + i) * NAME_SIZE], "CPTAF FLT%d", i);
341 switch (i) {
342 case CPT_AF_INT_VEC_FLT0:
343 flt_fn = rvu_cpt_af_flt0_intr_handler;
344 break;
345 case CPT_AF_INT_VEC_FLT1:
346 flt_fn = rvu_cpt_af_flt1_intr_handler;
347 break;
348 }
349 ret = rvu_cpt_do_register_interrupt(block, offs + i,
350 flt_fn, &rvu->irq_name[(offs + i) * NAME_SIZE]);
351 if (ret)
352 goto err;
353 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);
354 }
355
356 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU,
357 rvu_cpt_af_rvu_intr_handler,
358 "CPTAF RVU");
359 if (ret)
360 goto err;
361 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
362
363 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS,
364 rvu_cpt_af_ras_intr_handler,
365 "CPTAF RAS");
366 if (ret)
367 goto err;
368 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
369
370 return 0;
371 err:
372 rvu_cpt_unregister_interrupts(rvu);
373 return ret;
374 }
375
rvu_cpt_register_interrupts(struct rvu * rvu)376 int rvu_cpt_register_interrupts(struct rvu *rvu)
377 {
378 int ret;
379
380 ret = cpt_register_interrupts(rvu, BLKADDR_CPT0);
381 if (ret)
382 return ret;
383
384 return cpt_register_interrupts(rvu, BLKADDR_CPT1);
385 }
386
get_cpt_pf_num(struct rvu * rvu)387 static int get_cpt_pf_num(struct rvu *rvu)
388 {
389 int i, domain_nr, cpt_pf_num = -1;
390 struct pci_dev *pdev;
391
392 domain_nr = pci_domain_nr(rvu->pdev->bus);
393 for (i = 0; i < rvu->hw->total_pfs; i++) {
394 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0);
395 if (!pdev)
396 continue;
397
398 if (pdev->device == PCI_DEVID_OTX2_CPT_PF ||
399 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) {
400 cpt_pf_num = i;
401 put_device(&pdev->dev);
402 break;
403 }
404 put_device(&pdev->dev);
405 }
406 return cpt_pf_num;
407 }
408
is_cpt_pf(struct rvu * rvu,u16 pcifunc)409 static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc)
410 {
411 int cpt_pf_num = rvu->cpt_pf_num;
412
413 if (rvu_get_pf(pcifunc) != cpt_pf_num)
414 return false;
415 if (pcifunc & RVU_PFVF_FUNC_MASK)
416 return false;
417
418 return true;
419 }
420
is_cpt_vf(struct rvu * rvu,u16 pcifunc)421 static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc)
422 {
423 int cpt_pf_num = rvu->cpt_pf_num;
424
425 if (rvu_get_pf(pcifunc) != cpt_pf_num)
426 return false;
427 if (!(pcifunc & RVU_PFVF_FUNC_MASK))
428 return false;
429
430 return true;
431 }
432
validate_and_get_cpt_blkaddr(int req_blkaddr)433 static int validate_and_get_cpt_blkaddr(int req_blkaddr)
434 {
435 int blkaddr;
436
437 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0;
438 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)
439 return -EINVAL;
440
441 return blkaddr;
442 }
443
rvu_mbox_handler_cpt_lf_alloc(struct rvu * rvu,struct cpt_lf_alloc_req_msg * req,struct msg_rsp * rsp)444 int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu,
445 struct cpt_lf_alloc_req_msg *req,
446 struct msg_rsp *rsp)
447 {
448 u16 pcifunc = req->hdr.pcifunc;
449 struct rvu_block *block;
450 int cptlf, blkaddr;
451 int num_lfs, slot;
452 u64 val;
453
454 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
455 if (blkaddr < 0)
456 return blkaddr;
457
458 if (req->eng_grpmsk == 0x0)
459 return CPT_AF_ERR_GRP_INVALID;
460
461 block = &rvu->hw->block[blkaddr];
462 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
463 block->addr);
464 if (!num_lfs)
465 return CPT_AF_ERR_LF_INVALID;
466
467 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
468 if (req->nix_pf_func) {
469 /* If default, use 'this' CPTLF's PFFUNC */
470 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC)
471 req->nix_pf_func = pcifunc;
472 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX))
473 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
474 }
475
476 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
477 if (req->sso_pf_func) {
478 /* If default, use 'this' CPTLF's PFFUNC */
479 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC)
480 req->sso_pf_func = pcifunc;
481 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO))
482 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
483 }
484
485 for (slot = 0; slot < num_lfs; slot++) {
486 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
487 if (cptlf < 0)
488 return CPT_AF_ERR_LF_INVALID;
489
490 /* Set CPT LF group and priority */
491 val = (u64)req->eng_grpmsk << 48 | 1;
492 if (!is_rvu_otx2(rvu)) {
493 if (req->ctx_ilen_valid)
494 val |= (req->ctx_ilen << 17);
495 else
496 val |= (CPT_CTX_ILEN << 17);
497 }
498
499 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
500
501 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set
502 * on reset.
503 */
504 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
505 val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32));
506 val |= ((u64)req->nix_pf_func << 48 |
507 (u64)req->sso_pf_func << 32);
508 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
509 }
510
511 return 0;
512 }
513
cpt_lf_free(struct rvu * rvu,struct msg_req * req,int blkaddr)514 static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr)
515 {
516 u16 pcifunc = req->hdr.pcifunc;
517 int num_lfs, cptlf, slot, err;
518 struct rvu_block *block;
519
520 block = &rvu->hw->block[blkaddr];
521 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
522 block->addr);
523 if (!num_lfs)
524 return 0;
525
526 for (slot = 0; slot < num_lfs; slot++) {
527 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
528 if (cptlf < 0)
529 return CPT_AF_ERR_LF_INVALID;
530
531 /* Perform teardown */
532 rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot);
533
534 /* Reset LF */
535 err = rvu_lf_reset(rvu, block, cptlf);
536 if (err) {
537 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
538 block->addr, cptlf);
539 }
540 }
541
542 return 0;
543 }
544
rvu_mbox_handler_cpt_lf_free(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)545 int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req,
546 struct msg_rsp *rsp)
547 {
548 int ret;
549
550 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0);
551 if (ret)
552 return ret;
553
554 if (is_block_implemented(rvu->hw, BLKADDR_CPT1))
555 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1);
556
557 return ret;
558 }
559
cpt_inline_ipsec_cfg_inbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)560 static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf,
561 struct cpt_inline_ipsec_cfg_msg *req)
562 {
563 u16 sso_pf_func = req->sso_pf_func;
564 u8 nix_sel;
565 u64 val;
566
567 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
568 if (req->enable && (val & BIT_ULL(16))) {
569 /* IPSec inline outbound path is already enabled for a given
570 * CPT LF, HRM states that inline inbound & outbound paths
571 * must not be enabled at the same time for a given CPT LF
572 */
573 return CPT_AF_ERR_INLINE_IPSEC_INB_ENA;
574 }
575 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
576 if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO))
577 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
578
579 nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0;
580 /* Enable CPT LF for IPsec inline inbound operations */
581 if (req->enable)
582 val |= BIT_ULL(9);
583 else
584 val &= ~BIT_ULL(9);
585
586 val |= (u64)nix_sel << 8;
587 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
588
589 if (sso_pf_func) {
590 /* Set SSO_PF_FUNC */
591 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
592 val |= (u64)sso_pf_func << 32;
593 val |= (u64)req->nix_pf_func << 48;
594 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
595 }
596 if (req->sso_pf_func_ovrd)
597 /* Set SSO_PF_FUNC_OVRD for inline IPSec */
598 rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1);
599
600 /* Configure the X2P Link register with the cpt base channel number and
601 * range of channels it should propagate to X2P
602 */
603 if (!is_rvu_otx2(rvu)) {
604 val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16);
605 val |= (u64)rvu->hw->cpt_chan_base;
606
607 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val);
608 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val);
609 }
610
611 return 0;
612 }
613
cpt_inline_ipsec_cfg_outbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)614 static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf,
615 struct cpt_inline_ipsec_cfg_msg *req)
616 {
617 u16 nix_pf_func = req->nix_pf_func;
618 int nix_blkaddr;
619 u8 nix_sel;
620 u64 val;
621
622 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
623 if (req->enable && (val & BIT_ULL(9))) {
624 /* IPSec inline inbound path is already enabled for a given
625 * CPT LF, HRM states that inline inbound & outbound paths
626 * must not be enabled at the same time for a given CPT LF
627 */
628 return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA;
629 }
630
631 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
632 if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX))
633 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
634
635 /* Enable CPT LF for IPsec inline outbound operations */
636 if (req->enable)
637 val |= BIT_ULL(16);
638 else
639 val &= ~BIT_ULL(16);
640 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
641
642 if (nix_pf_func) {
643 /* Set NIX_PF_FUNC */
644 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
645 val |= (u64)nix_pf_func << 48;
646 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
647
648 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func);
649 nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1;
650
651 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
652 val |= (u64)nix_sel << 8;
653 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
654 }
655
656 return 0;
657 }
658
rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu * rvu,struct cpt_inline_ipsec_cfg_msg * req,struct msg_rsp * rsp)659 int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu,
660 struct cpt_inline_ipsec_cfg_msg *req,
661 struct msg_rsp *rsp)
662 {
663 u16 pcifunc = req->hdr.pcifunc;
664 struct rvu_block *block;
665 int cptlf, blkaddr, ret;
666 u16 actual_slot;
667
668 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
669 req->slot, &actual_slot);
670 if (blkaddr < 0)
671 return CPT_AF_ERR_LF_INVALID;
672
673 block = &rvu->hw->block[blkaddr];
674
675 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
676 if (cptlf < 0)
677 return CPT_AF_ERR_LF_INVALID;
678
679 switch (req->dir) {
680 case CPT_INLINE_INBOUND:
681 ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req);
682 break;
683
684 case CPT_INLINE_OUTBOUND:
685 ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req);
686 break;
687
688 default:
689 return CPT_AF_ERR_PARAM;
690 }
691
692 return ret;
693 }
694
validate_and_update_reg_offset(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req,u64 * reg_offset)695 static bool validate_and_update_reg_offset(struct rvu *rvu,
696 struct cpt_rd_wr_reg_msg *req,
697 u64 *reg_offset)
698 {
699 u64 offset = req->reg_offset;
700 int blkaddr, num_lfs, lf;
701 struct rvu_block *block;
702 struct rvu_pfvf *pfvf;
703
704 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
705 if (blkaddr < 0)
706 return false;
707
708 /* Registers that can be accessed from PF/VF */
709 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) ||
710 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) {
711 if (offset & 7)
712 return false;
713
714 lf = (offset & 0xFFF) >> 3;
715 block = &rvu->hw->block[blkaddr];
716 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
717 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
718 if (lf >= num_lfs)
719 /* Slot is not valid for that PF/VF */
720 return false;
721
722 /* Translate local LF used by VFs to global CPT LF */
723 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr],
724 req->hdr.pcifunc, lf);
725 if (lf < 0)
726 return false;
727
728 /* Translate local LF's offset to global CPT LF's offset to
729 * access LFX register.
730 */
731 *reg_offset = (req->reg_offset & 0xFF000) + (lf << 3);
732
733 return true;
734 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) {
735 /* Registers that can be accessed from PF */
736 switch (offset) {
737 case CPT_AF_DIAG:
738 case CPT_AF_CTL:
739 case CPT_AF_PF_FUNC:
740 case CPT_AF_BLK_RST:
741 case CPT_AF_CONSTANTS1:
742 case CPT_AF_CTX_FLUSH_TIMER:
743 case CPT_AF_RXC_CFG1:
744 return true;
745 }
746
747 switch (offset & 0xFF000) {
748 case CPT_AF_EXEX_STS(0):
749 case CPT_AF_EXEX_CTL(0):
750 case CPT_AF_EXEX_CTL2(0):
751 case CPT_AF_EXEX_UCODE_BASE(0):
752 if (offset & 7)
753 return false;
754 break;
755 default:
756 return false;
757 }
758 return true;
759 }
760 return false;
761 }
762
rvu_mbox_handler_cpt_rd_wr_register(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req,struct cpt_rd_wr_reg_msg * rsp)763 int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu,
764 struct cpt_rd_wr_reg_msg *req,
765 struct cpt_rd_wr_reg_msg *rsp)
766 {
767 u64 offset = req->reg_offset;
768 int blkaddr;
769
770 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
771 if (blkaddr < 0)
772 return blkaddr;
773
774 /* This message is accepted only if sent from CPT PF/VF */
775 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
776 !is_cpt_vf(rvu, req->hdr.pcifunc))
777 return CPT_AF_ERR_ACCESS_DENIED;
778
779 if (!validate_and_update_reg_offset(rvu, req, &offset))
780 return CPT_AF_ERR_ACCESS_DENIED;
781
782 rsp->reg_offset = req->reg_offset;
783 rsp->ret_val = req->ret_val;
784 rsp->is_write = req->is_write;
785
786 if (req->is_write)
787 rvu_write64(rvu, blkaddr, offset, req->val);
788 else
789 rsp->val = rvu_read64(rvu, blkaddr, offset);
790
791 return 0;
792 }
793
get_ctx_pc(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)794 static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
795 {
796 struct rvu_hwinfo *hw = rvu->hw;
797
798 if (is_rvu_otx2(rvu))
799 return;
800
801 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC);
802 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC);
803 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC);
804 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr,
805 CPT_AF_CTX_AOP_LATENCY_PC);
806 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC);
807 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr,
808 CPT_AF_CTX_IFETCH_LATENCY_PC);
809 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
810 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr,
811 CPT_AF_CTX_FFETCH_LATENCY_PC);
812 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
813 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr,
814 CPT_AF_CTX_FFETCH_LATENCY_PC);
815 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
816 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr,
817 CPT_AF_CTX_FFETCH_LATENCY_PC);
818 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR);
819 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID);
820 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER);
821 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0));
822 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1));
823
824 if (!hw->cap.cpt_rxc)
825 return;
826 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME);
827 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
828 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
829 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
830 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
831 }
832
get_eng_sts(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)833 static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
834 {
835 u16 max_ses, max_ies, max_aes;
836 u32 e_min = 0, e_max = 0;
837 u64 reg;
838
839 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1);
840 max_ses = reg & 0xffff;
841 max_ies = (reg >> 16) & 0xffff;
842 max_aes = (reg >> 32) & 0xffff;
843
844 /* Get AE status */
845 e_min = max_ses + max_ies;
846 e_max = max_ses + max_ies + max_aes;
847 cpt_get_eng_sts(e_min, e_max, rsp, ae);
848 /* Get SE status */
849 e_min = 0;
850 e_max = max_ses;
851 cpt_get_eng_sts(e_min, e_max, rsp, se);
852 /* Get IE status */
853 e_min = max_ses;
854 e_max = max_ses + max_ies;
855 cpt_get_eng_sts(e_min, e_max, rsp, ie);
856 }
857
rvu_mbox_handler_cpt_sts(struct rvu * rvu,struct cpt_sts_req * req,struct cpt_sts_rsp * rsp)858 int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req,
859 struct cpt_sts_rsp *rsp)
860 {
861 int blkaddr;
862
863 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
864 if (blkaddr < 0)
865 return blkaddr;
866
867 /* This message is accepted only if sent from CPT PF/VF */
868 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
869 !is_cpt_vf(rvu, req->hdr.pcifunc))
870 return CPT_AF_ERR_ACCESS_DENIED;
871
872 get_ctx_pc(rvu, rsp, blkaddr);
873
874 /* Get CPT engines status */
875 get_eng_sts(rvu, rsp, blkaddr);
876
877 /* Read CPT instruction PC registers */
878 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC);
879 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC);
880 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC);
881 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC);
882 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC);
883 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr,
884 CPT_AF_ACTIVE_CYCLES_PC);
885 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO);
886 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT);
887 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG);
888
889 return 0;
890 }
891
892 #define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48)
893 #define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32)
894 #define RXC_ACTIVE_THRES GENMASK_ULL(27, 16)
895 #define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0)
896 #define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48)
897 #define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48)
898
cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,int blkaddr,struct cpt_rxc_time_cfg_req * save)899 static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req,
900 int blkaddr, struct cpt_rxc_time_cfg_req *save)
901 {
902 u64 dfrg_reg;
903
904 if (save) {
905 /* Save older config */
906 dfrg_reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
907 save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg);
908 save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg);
909 save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg);
910 save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg);
911
912 save->step = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
913 }
914
915 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres);
916 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit);
917 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres);
918 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit);
919
920 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step);
921 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg);
922 }
923
rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,struct msg_rsp * rsp)924 int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu,
925 struct cpt_rxc_time_cfg_req *req,
926 struct msg_rsp *rsp)
927 {
928 int blkaddr;
929
930 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
931 if (blkaddr < 0)
932 return blkaddr;
933
934 /* This message is accepted only if sent from CPT PF/VF */
935 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
936 !is_cpt_vf(rvu, req->hdr.pcifunc))
937 return CPT_AF_ERR_ACCESS_DENIED;
938
939 cpt_rxc_time_cfg(rvu, req, blkaddr, NULL);
940
941 return 0;
942 }
943
rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)944 int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req,
945 struct msg_rsp *rsp)
946 {
947 return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc);
948 }
949
rvu_mbox_handler_cpt_lf_reset(struct rvu * rvu,struct cpt_lf_rst_req * req,struct msg_rsp * rsp)950 int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req,
951 struct msg_rsp *rsp)
952 {
953 u16 pcifunc = req->hdr.pcifunc;
954 struct rvu_block *block;
955 int cptlf, blkaddr, ret;
956 u16 actual_slot;
957 u64 ctl, ctl2;
958
959 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
960 req->slot, &actual_slot);
961 if (blkaddr < 0)
962 return CPT_AF_ERR_LF_INVALID;
963
964 block = &rvu->hw->block[blkaddr];
965
966 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
967 if (cptlf < 0)
968 return CPT_AF_ERR_LF_INVALID;
969 ctl = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
970 ctl2 = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
971
972 ret = rvu_lf_reset(rvu, block, cptlf);
973 if (ret)
974 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
975 block->addr, cptlf);
976
977 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), ctl);
978 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), ctl2);
979
980 return 0;
981 }
982
rvu_mbox_handler_cpt_flt_eng_info(struct rvu * rvu,struct cpt_flt_eng_info_req * req,struct cpt_flt_eng_info_rsp * rsp)983 int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req,
984 struct cpt_flt_eng_info_rsp *rsp)
985 {
986 struct rvu_block *block;
987 unsigned long flags;
988 int blkaddr, vec;
989 int flt_vecs;
990 u16 max_engs;
991
992 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
993 if (blkaddr < 0)
994 return blkaddr;
995
996 block = &rvu->hw->block[blkaddr];
997 max_engs = cpt_max_engines_get(rvu);
998 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);
999 for (vec = 0; vec < flt_vecs; vec++) {
1000 spin_lock_irqsave(&rvu->cpt_intr_lock, flags);
1001 rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec];
1002 rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec];
1003 if (req->reset) {
1004 block->cpt_flt_eng_map[vec] = 0x0;
1005 block->cpt_rcvrd_eng_map[vec] = 0x0;
1006 }
1007 spin_unlock_irqrestore(&rvu->cpt_intr_lock, flags);
1008 }
1009 return 0;
1010 }
1011
cpt_rxc_teardown(struct rvu * rvu,int blkaddr)1012 static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr)
1013 {
1014 struct cpt_rxc_time_cfg_req req, prev;
1015 struct rvu_hwinfo *hw = rvu->hw;
1016 int timeout = 2000;
1017 u64 reg;
1018
1019 if (!hw->cap.cpt_rxc)
1020 return;
1021
1022 /* Set time limit to minimum values, so that rxc entries will be
1023 * flushed out quickly.
1024 */
1025 req.step = 1;
1026 req.zombie_thres = 1;
1027 req.zombie_limit = 1;
1028 req.active_thres = 1;
1029 req.active_limit = 1;
1030
1031 cpt_rxc_time_cfg(rvu, &req, blkaddr, &prev);
1032
1033 do {
1034 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
1035 udelay(1);
1036 if (FIELD_GET(RXC_ACTIVE_COUNT, reg))
1037 timeout--;
1038 else
1039 break;
1040 } while (timeout);
1041
1042 if (timeout == 0)
1043 dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n");
1044
1045 timeout = 2000;
1046 do {
1047 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
1048 udelay(1);
1049 if (FIELD_GET(RXC_ZOMBIE_COUNT, reg))
1050 timeout--;
1051 else
1052 break;
1053 } while (timeout);
1054
1055 if (timeout == 0)
1056 dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n");
1057
1058 /* Restore config */
1059 cpt_rxc_time_cfg(rvu, &prev, blkaddr, NULL);
1060 }
1061
1062 #define INFLIGHT GENMASK_ULL(8, 0)
1063 #define GRB_CNT GENMASK_ULL(39, 32)
1064 #define GWB_CNT GENMASK_ULL(47, 40)
1065 #define XQ_XOR GENMASK_ULL(63, 63)
1066 #define DQPTR GENMASK_ULL(19, 0)
1067 #define NQPTR GENMASK_ULL(51, 32)
1068
cpt_lf_disable_iqueue(struct rvu * rvu,int blkaddr,int slot)1069 static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot)
1070 {
1071 int timeout = 1000000;
1072 u64 inprog, inst_ptr;
1073 u64 qsize, pending;
1074 int i = 0;
1075
1076 /* Disable instructions enqueuing */
1077 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0);
1078
1079 inprog = rvu_read64(rvu, blkaddr,
1080 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1081 inprog |= BIT_ULL(16);
1082 rvu_write64(rvu, blkaddr,
1083 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog);
1084
1085 qsize = rvu_read64(rvu, blkaddr,
1086 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF;
1087 do {
1088 inst_ptr = rvu_read64(rvu, blkaddr,
1089 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR));
1090 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +
1091 FIELD_GET(NQPTR, inst_ptr) -
1092 FIELD_GET(DQPTR, inst_ptr);
1093 udelay(1);
1094 timeout--;
1095 } while ((pending != 0) && (timeout != 0));
1096
1097 if (timeout == 0)
1098 dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n");
1099
1100 timeout = 1000000;
1101 /* Wait for CPT queue to become execution-quiescent */
1102 do {
1103 inprog = rvu_read64(rvu, blkaddr,
1104 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1105
1106 if ((FIELD_GET(INFLIGHT, inprog) == 0) &&
1107 (FIELD_GET(GRB_CNT, inprog) == 0)) {
1108 i++;
1109 } else {
1110 i = 0;
1111 timeout--;
1112 }
1113 } while ((timeout != 0) && (i < 10));
1114
1115 if (timeout == 0)
1116 dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n");
1117 /* Wait for 2 us to flush all queue writes to memory */
1118 udelay(2);
1119 }
1120
rvu_cpt_lf_teardown(struct rvu * rvu,u16 pcifunc,int blkaddr,int lf,int slot)1121 int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot)
1122 {
1123 u64 reg;
1124
1125 if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc))
1126 cpt_rxc_teardown(rvu, blkaddr);
1127
1128 mutex_lock(&rvu->alias_lock);
1129 /* Enable BAR2 ALIAS for this pcifunc. */
1130 reg = BIT_ULL(16) | pcifunc;
1131 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1132
1133 cpt_lf_disable_iqueue(rvu, blkaddr, slot);
1134
1135 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1136 mutex_unlock(&rvu->alias_lock);
1137
1138 return 0;
1139 }
1140
1141 #define CPT_RES_LEN 16
1142 #define CPT_SE_IE_EGRP 1ULL
1143
cpt_inline_inb_lf_cmd_send(struct rvu * rvu,int blkaddr,int nix_blkaddr)1144 static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr,
1145 int nix_blkaddr)
1146 {
1147 int cpt_pf_num = rvu->cpt_pf_num;
1148 struct cpt_inst_lmtst_req *req;
1149 dma_addr_t res_daddr;
1150 int timeout = 3000;
1151 u8 cpt_idx;
1152 u64 *inst;
1153 u16 *res;
1154 int rc;
1155
1156 res = kzalloc(CPT_RES_LEN, GFP_KERNEL);
1157 if (!res)
1158 return -ENOMEM;
1159
1160 res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN,
1161 DMA_BIDIRECTIONAL);
1162 if (dma_mapping_error(rvu->dev, res_daddr)) {
1163 dev_err(rvu->dev, "DMA mapping failed for CPT result\n");
1164 rc = -EFAULT;
1165 goto res_free;
1166 }
1167 *res = 0xFFFF;
1168
1169 /* Send mbox message to CPT PF */
1170 req = (struct cpt_inst_lmtst_req *)
1171 otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up,
1172 cpt_pf_num, sizeof(*req),
1173 sizeof(struct msg_rsp));
1174 if (!req) {
1175 rc = -ENOMEM;
1176 goto res_daddr_unmap;
1177 }
1178 req->hdr.sig = OTX2_MBOX_REQ_SIG;
1179 req->hdr.id = MBOX_MSG_CPT_INST_LMTST;
1180
1181 inst = req->inst;
1182 /* Prepare CPT_INST_S */
1183 inst[0] = 0;
1184 inst[1] = res_daddr;
1185 /* AF PF FUNC */
1186 inst[2] = 0;
1187 /* Set QORD */
1188 inst[3] = 1;
1189 inst[4] = 0;
1190 inst[5] = 0;
1191 inst[6] = 0;
1192 /* Set EGRP */
1193 inst[7] = CPT_SE_IE_EGRP << 61;
1194
1195 /* Subtract 1 from the NIX-CPT credit count to preserve
1196 * credit counts.
1197 */
1198 cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1;
1199 rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),
1200 BIT_ULL(22) - 1);
1201
1202 otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1203 rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1204 if (rc)
1205 dev_warn(rvu->dev, "notification to pf %d failed\n",
1206 cpt_pf_num);
1207 /* Wait for CPT instruction to be completed */
1208 do {
1209 mdelay(1);
1210 if (*res == 0xFFFF)
1211 timeout--;
1212 else
1213 break;
1214 } while (timeout);
1215
1216 if (timeout == 0)
1217 dev_warn(rvu->dev, "Poll for result hits hard loop counter\n");
1218
1219 res_daddr_unmap:
1220 dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL);
1221 res_free:
1222 kfree(res);
1223
1224 return 0;
1225 }
1226
1227 #define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46)
1228 #define CTX_CAM_CPTR GENMASK_ULL(45, 0)
1229
rvu_cpt_ctx_flush(struct rvu * rvu,u16 pcifunc)1230 int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc)
1231 {
1232 int nix_blkaddr, blkaddr;
1233 u16 max_ctx_entries, i;
1234 int slot = 0, num_lfs;
1235 u64 reg, cam_data;
1236 int rc;
1237
1238 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1239 if (nix_blkaddr < 0)
1240 return -EINVAL;
1241
1242 if (is_rvu_otx2(rvu))
1243 return 0;
1244
1245 blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0;
1246
1247 /* Submit CPT_INST_S to track when all packets have been
1248 * flushed through for the NIX PF FUNC in inline inbound case.
1249 */
1250 rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr);
1251 if (rc)
1252 return rc;
1253
1254 /* Wait for rxc entries to be flushed out */
1255 cpt_rxc_teardown(rvu, blkaddr);
1256
1257 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);
1258 max_ctx_entries = (reg >> 48) & 0xFFF;
1259
1260 mutex_lock(&rvu->rsrc_lock);
1261
1262 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1263 blkaddr);
1264 if (num_lfs == 0) {
1265 dev_warn(rvu->dev, "CPT LF is not configured\n");
1266 goto unlock;
1267 }
1268
1269 /* Enable BAR2 ALIAS for this pcifunc. */
1270 reg = BIT_ULL(16) | pcifunc;
1271 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1272
1273 for (i = 0; i < max_ctx_entries; i++) {
1274 cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i));
1275
1276 if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) &&
1277 FIELD_GET(CTX_CAM_CPTR, cam_data)) {
1278 reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data);
1279 rvu_write64(rvu, blkaddr,
1280 CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH),
1281 reg);
1282 }
1283 }
1284 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1285
1286 unlock:
1287 mutex_unlock(&rvu->rsrc_lock);
1288
1289 return 0;
1290 }
1291
1292 #define MAX_RXC_ICB_CNT GENMASK_ULL(40, 32)
1293
rvu_cpt_init(struct rvu * rvu)1294 int rvu_cpt_init(struct rvu *rvu)
1295 {
1296 struct rvu_hwinfo *hw = rvu->hw;
1297 u64 reg_val;
1298
1299 /* Retrieve CPT PF number */
1300 rvu->cpt_pf_num = get_cpt_pf_num(rvu);
1301 if (is_block_implemented(rvu->hw, BLKADDR_CPT0) && !is_rvu_otx2(rvu) &&
1302 !is_cn10kb(rvu))
1303 hw->cap.cpt_rxc = true;
1304
1305 if (hw->cap.cpt_rxc && !is_cn10ka_a0(rvu) && !is_cn10ka_a1(rvu)) {
1306 /* Set CPT_AF_RXC_CFG1:max_rxc_icb_cnt to 0xc0 to not effect
1307 * inline inbound peak performance
1308 */
1309 reg_val = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1);
1310 reg_val &= ~MAX_RXC_ICB_CNT;
1311 reg_val |= FIELD_PREP(MAX_RXC_ICB_CNT,
1312 CPT_DFLT_MAX_RXC_ICB_CNT);
1313 rvu_write64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1, reg_val);
1314 }
1315
1316 spin_lock_init(&rvu->cpt_intr_lock);
1317
1318 return 0;
1319 }
1320