1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Admin Function driver
3 *
4 * Copyright (C) 2026 Marvell.
5 *
6 */
7 #include <linux/xarray.h>
8 #include <linux/bitfield.h>
9
10 #include "rvu.h"
11 #include "npc.h"
12 #include "npc_profile.h"
13 #include "rvu_npc_hash.h"
14 #include "rvu_npc.h"
15 #include "cn20k/npc.h"
16 #include "cn20k/reg.h"
17 #include "rvu_npc_fs.h"
18
19 static struct npc_priv_t *npc_priv;
20
21 static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = {
22 [NPC_MCAM_KEY_DYN] = "DYNAMIC",
23 [NPC_MCAM_KEY_X2] = "X2",
24 [NPC_MCAM_KEY_X4] = "X4",
25 };
26
27 static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = {
28 [NPC_DFT_RULE_PROMISC_ID] = "Promisc",
29 [NPC_DFT_RULE_MCAST_ID] = "Mcast",
30 [NPC_DFT_RULE_BCAST_ID] = "Bcast",
31 [NPC_DFT_RULE_UCAST_ID] = "Ucast",
32 };
33
34 #define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \
35 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \
36 ((key_ofs) & 0x3F))
37
38 #define NPC_DFT_RULE_ID_MK(pcifunc, id) \
39 ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id))
40
41 #define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \
42 FIELD_GET(GENMASK_ULL(15, 0), rid)
43
44 #define NPC_DFT_RULE_ID_2_ID(rid) \
45 FIELD_GET(GENMASK_ULL(31, 16), rid)
46
47 #define NPC_DFT_RULE_PRIO 127
48
49 static const char cn20k_def_pfl_name[] = "default";
50
51 static struct npc_mcam_kex_extr npc_mkex_extr_default = {
52 .mkex_sign = MKEX_CN20K_SIGN,
53 .name = "default",
54 .kpu_version = NPC_KPU_PROFILE_VER,
55 .keyx_cfg = {
56 /* nibble: LA..LE (ltype only) + Error code + Channel */
57 [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) |
58 NPC_CN20K_PARSE_NIBBLE_INTF_RX |
59 NPC_CN20K_PARSE_NIBBLE_ERRCODE,
60
61 /* nibble: LA..LE (ltype only) */
62 [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) |
63 NPC_CN20K_PARSE_NIBBLE_INTF_TX,
64 },
65 .intf_extr_lid = {
66 /* Default RX MCAM KEX profile */
67 [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
68 NPC_LID_LC, NPC_LID_LC, NPC_LID_LD },
69 [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
70 NPC_LID_LC, NPC_LID_LD },
71 },
72 .intf_extr_lt = {
73 /* Default RX MCAM KEX profile */
74 [NIX_INTF_RX] = {
75 [0] = {
76 /* Layer A: Ethernet: */
77 [NPC_LT_LA_ETHER] =
78 /* DMAC: 6 bytes, KW1[63:15] */
79 KEX_EXTR_CFG(0x05, 0x0, 0x1,
80 NPC_KEXOF_DMAC + 1),
81 [NPC_LT_LA_CPT_HDR] =
82 /* DMAC: 6 bytes, KW1[63:15] */
83 KEX_EXTR_CFG(0x05, 0x0, 0x1,
84 NPC_KEXOF_DMAC + 1),
85 },
86 [1] = {
87 /* Layer A: Ethernet: */
88 [NPC_LT_LA_ETHER] =
89 /* Ethertype: 2 bytes, KW0[63:48] */
90 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
91 [NPC_LT_LA_CPT_HDR] =
92 /* Ethertype: 2 bytes, KW0[63:48] */
93 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
94 },
95 [2] = {
96 /* Layer B: Single VLAN (CTAG) */
97 [NPC_LT_LB_CTAG] =
98 /* CTAG VLAN: 2 bytes, KW1[15:0] */
99 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
100 /* Layer B: Stacked VLAN (STAG|QinQ) */
101 [NPC_LT_LB_STAG_QINQ] =
102 /* Outer VLAN: 2 bytes, KW1[15:0] */
103 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
104 [NPC_LT_LB_FDSA] =
105 /* SWITCH PORT: 1 byte, KW1[7:0] */
106 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8),
107 },
108 [3] = {
109 [NPC_LT_LB_CTAG] =
110 /* Ethertype: 2 bytes, KW0[63:48] */
111 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
112 [NPC_LT_LB_STAG_QINQ] =
113 /* Ethertype: 2 bytes, KW0[63:48] */
114 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6),
115 [NPC_LT_LB_FDSA] =
116 /* Ethertype: 2 bytes, KW0[63:48] */
117 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
118 },
119 [4] = {
120 /* Layer C: IPv4 */
121 [NPC_LT_LC_IP] =
122 /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */
123 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11),
124 /* Layer C: IPv6 */
125 [NPC_LT_LC_IP6] =
126 /* Everything up to SADDR: 8 bytes, KW3[7:0],
127 * KW2[63:8]
128 */
129 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11),
130 },
131 [5] = {
132 [NPC_LT_LC_IP] =
133 /* TOS: 1 byte, KW2[7:0] */
134 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10),
135 },
136 [6] = {
137 /* Layer D:UDP */
138 [NPC_LT_LD_UDP] =
139 /* SPORT+DPORT: 4 bytes, KW3[39:8] */
140 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
141 /* Layer D:TCP */
142 [NPC_LT_LD_TCP] =
143 /* SPORT+DPORT: 4 bytes, KW3[39:8] */
144 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
145 },
146 },
147 /* Default TX MCAM KEX profile */
148 [NIX_INTF_TX] = {
149 [0] = {
150 /* Layer A: NIX_INST_HDR_S + Ethernet */
151 /* NIX appends 8 bytes of NIX_INST_HDR_S at the
152 * start of each TX packet supplied to NPC.
153 */
154 [NPC_LT_LA_IH_NIX_ETHER] =
155 /* PF_FUNC: 2B , KW0 [47:32] */
156 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
157 /* Layer A: HiGig2: */
158 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
159 /* PF_FUNC: 2B , KW0 [47:32] */
160 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
161 },
162 [1] = {
163 [NPC_LT_LA_IH_NIX_ETHER] =
164 /* SQ_ID 3 bytes, KW1[63:16] */
165 KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa),
166 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
167 /* VID: 2 bytes, KW1[31:16] */
168 KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa),
169 },
170 [2] = {
171 /* Layer B: Single VLAN (CTAG) */
172 [NPC_LT_LB_CTAG] =
173 /* CTAG VLAN[2..3] KW0[63:48] */
174 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
175 /* Layer B: Stacked VLAN (STAG|QinQ) */
176 [NPC_LT_LB_STAG_QINQ] =
177 /* Outer VLAN: 2 bytes, KW0[63:48] */
178 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
179 },
180 [3] = {
181 [NPC_LT_LB_CTAG] =
182 /* CTAG VLAN[2..3] KW1[15:0] */
183 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8),
184 [NPC_LT_LB_STAG_QINQ] =
185 /* Outer VLAN: 2 Bytes, KW1[15:0] */
186 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8),
187 },
188 [4] = {
189 /* Layer C: IPv4 */
190 [NPC_LT_LC_IP] =
191 /* SIP+DIP: 8 bytes, KW2[63:0] */
192 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10),
193 /* Layer C: IPv6 */
194 [NPC_LT_LC_IP6] =
195 /* Everything up to SADDR: 8 bytes, KW2[63:0] */
196 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10),
197 },
198 [5] = {
199 /* Layer D:UDP */
200 [NPC_LT_LD_UDP] =
201 /* SPORT+DPORT: 4 bytes, KW3[31:0] */
202 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
203 /* Layer D:TCP */
204 [NPC_LT_LD_TCP] =
205 /* SPORT+DPORT: 4 bytes, KW3[31:0] */
206 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
207 },
208 },
209 },
210 };
211
npc_mkex_extr_default_get(void)212 struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void)
213 {
214 return &npc_mkex_extr_default;
215 }
216
npc_idx2vidx(u16 idx)217 static u16 npc_idx2vidx(u16 idx)
218 {
219 unsigned long index;
220 void *map;
221 u16 vidx;
222 int val;
223
224 vidx = idx;
225 index = idx;
226
227 map = xa_load(&npc_priv->xa_idx2vidx_map, index);
228 if (!map)
229 goto done;
230
231 val = xa_to_value(map);
232 if (val == -1)
233 goto done;
234
235 vidx = val;
236
237 done:
238 return vidx;
239 }
240
npc_is_vidx(u16 vidx)241 static bool npc_is_vidx(u16 vidx)
242 {
243 return vidx >= npc_priv->bank_depth * 2;
244 }
245
npc_vidx2idx(u16 vidx)246 static u16 npc_vidx2idx(u16 vidx)
247 {
248 unsigned long sentinel = (unsigned long)-1;
249 unsigned long index;
250 void *map;
251 int val;
252 u16 idx;
253
254 idx = vidx;
255 index = vidx;
256
257 map = xa_load(&npc_priv->xa_vidx2idx_map, index);
258 if (!map)
259 goto done;
260
261 val = xa_to_value(map);
262 if (val == sentinel)
263 goto done;
264
265 idx = val;
266
267 done:
268 return idx;
269 }
270
npc_cn20k_vidx2idx(u16 idx)271 u16 npc_cn20k_vidx2idx(u16 idx)
272 {
273 if (!npc_priv)
274 return idx;
275
276 if (!npc_is_vidx(idx))
277 return idx;
278
279 return npc_vidx2idx(idx);
280 }
281
npc_cn20k_idx2vidx(u16 idx)282 u16 npc_cn20k_idx2vidx(u16 idx)
283 {
284 if (!npc_priv)
285 return idx;
286
287 if (npc_is_vidx(idx))
288 return idx;
289
290 return npc_idx2vidx(idx);
291 }
292
npc_vidx_maps_del_entry(struct rvu * rvu,u16 vidx,u16 * old_midx)293 static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx)
294 {
295 u16 mcam_idx;
296 void *map;
297
298 if (!npc_is_vidx(vidx)) {
299 dev_err(rvu->dev,
300 "%s: vidx(%u) does not map to proper mcam idx\n",
301 __func__, vidx);
302 return -ESRCH;
303 }
304
305 mcam_idx = npc_vidx2idx(vidx);
306
307 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx);
308 if (!map) {
309 dev_err(rvu->dev,
310 "%s: vidx(%u) does not map to proper mcam idx\n",
311 __func__, vidx);
312 return -ESRCH;
313 }
314
315 map = xa_erase(&npc_priv->xa_idx2vidx_map, mcam_idx);
316 if (!map) {
317 dev_err(rvu->dev,
318 "%s: vidx(%u) is not valid\n",
319 __func__, vidx);
320 return -ESRCH;
321 }
322
323 if (old_midx)
324 *old_midx = mcam_idx;
325
326 return 0;
327 }
328
npc_vidx_maps_modify(struct rvu * rvu,u16 vidx,u16 new_midx)329 static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx)
330 {
331 u16 old_midx;
332 void *map;
333 int rc;
334
335 if (!npc_is_vidx(vidx)) {
336 dev_err(rvu->dev,
337 "%s: vidx(%u) does not map to proper mcam idx\n",
338 __func__, vidx);
339 return -ESRCH;
340 }
341
342 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx);
343 if (!map) {
344 dev_err(rvu->dev,
345 "%s: vidx(%u) could not be deleted from vidx2idx map\n",
346 __func__, vidx);
347 return -ESRCH;
348 }
349
350 old_midx = xa_to_value(map);
351
352 rc = xa_insert(&npc_priv->xa_vidx2idx_map, vidx,
353 xa_mk_value(new_midx), GFP_KERNEL);
354 if (rc) {
355 dev_err(rvu->dev,
356 "%s: vidx(%u) cannot be added to vidx2idx map\n",
357 __func__, vidx);
358 goto fail1;
359 }
360
361 map = xa_erase(&npc_priv->xa_idx2vidx_map, old_midx);
362 if (!map) {
363 dev_err(rvu->dev,
364 "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
365 __func__, old_midx, vidx);
366 rc = -ESRCH;
367 goto fail2;
368 }
369
370 rc = xa_insert(&npc_priv->xa_idx2vidx_map, new_midx,
371 xa_mk_value(vidx), GFP_KERNEL);
372 if (rc) {
373 dev_err(rvu->dev,
374 "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
375 __func__, new_midx, vidx);
376 goto fail3;
377 }
378
379 return 0;
380
381 fail3:
382 /* Restore vidx at old_midx location */
383 if (xa_insert(&npc_priv->xa_idx2vidx_map, old_midx,
384 xa_mk_value(vidx), GFP_KERNEL))
385 dev_err(rvu->dev,
386 "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n",
387 __func__, old_midx, vidx);
388 fail2:
389 /* Erase new_midx inserted at vidx */
390 if (!xa_erase(&npc_priv->xa_vidx2idx_map, vidx))
391 dev_err(rvu->dev,
392 "%s: Failed to roll back vidx2idx vidx=%u\n",
393 __func__, vidx);
394
395 fail1:
396 /* Restore old_midx at vidx location */
397 if (xa_insert(&npc_priv->xa_vidx2idx_map, vidx,
398 xa_mk_value(old_midx), GFP_KERNEL))
399 dev_err(rvu->dev,
400 "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n",
401 __func__, old_midx, vidx);
402
403 return rc;
404 }
405
npc_vidx_maps_add_entry(struct rvu * rvu,u16 mcam_idx,int pcifunc,u16 * vidx)406 static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc,
407 u16 *vidx)
408 {
409 int rc, max, min;
410 u32 id;
411
412 /* Virtual index start from maximum mcam index + 1 */
413 max = npc_priv->bank_depth * 2 * 2 - 1;
414 min = npc_priv->bank_depth * 2;
415
416 rc = xa_alloc(&npc_priv->xa_vidx2idx_map, &id,
417 xa_mk_value(mcam_idx),
418 XA_LIMIT(min, max), GFP_KERNEL);
419 if (rc) {
420 dev_err(rvu->dev,
421 "%s: Failed to add to vidx2idx map (%u)\n",
422 __func__, mcam_idx);
423 goto fail1;
424 }
425
426 rc = xa_insert(&npc_priv->xa_idx2vidx_map, mcam_idx,
427 xa_mk_value(id), GFP_KERNEL);
428 if (rc) {
429 dev_err(rvu->dev,
430 "%s: Failed to add to idx2vidx map (%u)\n",
431 __func__, mcam_idx);
432 goto fail2;
433 }
434
435 if (vidx)
436 *vidx = id;
437
438 return 0;
439
440 fail2:
441 xa_erase(&npc_priv->xa_vidx2idx_map, id);
442 fail1:
443 return rc;
444 }
445
npc_config_kpmcam(struct rvu * rvu,int blkaddr,const struct npc_kpu_profile_cam * kpucam,int kpm,int entry)446 static void npc_config_kpmcam(struct rvu *rvu, int blkaddr,
447 const struct npc_kpu_profile_cam *kpucam,
448 int kpm, int entry)
449 {
450 struct npc_kpu_cam cam0 = {0};
451 struct npc_kpu_cam cam1 = {0};
452
453 cam1.state = kpucam->state & kpucam->state_mask;
454 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask;
455 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask;
456 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask;
457
458 cam0.state = ~kpucam->state & kpucam->state_mask;
459 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask;
460 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask;
461 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask;
462
463 rvu_write64(rvu, blkaddr,
464 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0);
465 rvu_write64(rvu, blkaddr,
466 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1);
467 }
468
469 static void
npc_config_kpmaction(struct rvu * rvu,int blkaddr,const struct npc_kpu_profile_action * kpuaction,int kpm,int entry,bool pkind)470 npc_config_kpmaction(struct rvu *rvu, int blkaddr,
471 const struct npc_kpu_profile_action *kpuaction,
472 int kpm, int entry, bool pkind)
473 {
474 struct npc_kpm_action0 action0 = {0};
475 struct npc_kpu_action1 action1 = {0};
476 u64 reg;
477
478 action1.errlev = kpuaction->errlev;
479 action1.errcode = kpuaction->errcode;
480 action1.dp0_offset = kpuaction->dp0_offset;
481 action1.dp1_offset = kpuaction->dp1_offset;
482 action1.dp2_offset = kpuaction->dp2_offset;
483
484 if (pkind)
485 reg = NPC_AF_PKINDX_ACTION1(entry);
486 else
487 reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry);
488
489 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1);
490
491 action0.byp_count = kpuaction->bypass_count & 0x7;
492 action0.capture_ena = kpuaction->cap_ena & 1;
493 action0.parse_done = kpuaction->parse_done & 1;
494 action0.next_state = kpuaction->next_state & 0xf;
495 action0.capture_lid = kpuaction->lid & 0x7;
496
497 /* Parser functionality will work correctly even though
498 * upper flag bits are silently discarded
499 */
500 action0.capture_ltype = kpuaction->ltype & 0xf;
501 action0.capture_flags = kpuaction->flags & 0xf;
502 action0.ptr_advance = kpuaction->ptr_advance;
503
504 action0.var_len_offset = kpuaction->offset;
505 action0.var_len_mask = kpuaction->mask;
506 action0.var_len_right = kpuaction->right & 1;
507 action0.var_len_shift = kpuaction->shift & 1;
508
509 if (pkind)
510 reg = NPC_AF_PKINDX_ACTION0(entry);
511 else
512 reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry);
513
514 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0);
515 }
516
517 static void
npc_program_single_kpm_profile(struct rvu * rvu,int blkaddr,int kpm,int start_entry,const struct npc_kpu_profile * profile)518 npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr,
519 int kpm, int start_entry,
520 const struct npc_kpu_profile *profile)
521 {
522 int num_cam_entries, num_action_entries;
523 int entry, num_entries, max_entries;
524 u64 idx;
525
526 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile);
527 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile);
528
529 if (num_cam_entries != num_action_entries) {
530 dev_err(rvu->dev,
531 "kpm%d: CAM and action entries [%d != %d] not equal\n",
532 kpm, num_cam_entries, num_action_entries);
533
534 WARN(1, "Fatal error\n");
535 return;
536 }
537
538 max_entries = rvu->hw->npc_kpu_entries / 2;
539 entry = start_entry;
540 /* Program CAM match entries for previous kpm extracted data */
541 num_entries = min_t(int, num_cam_entries, max_entries);
542 for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
543 npc_config_kpmcam(rvu, blkaddr,
544 npc_get_kpu_cam_nth_entry(rvu, profile, idx),
545 kpm, entry);
546
547 entry = start_entry;
548 /* Program this kpm's actions */
549 num_entries = min_t(int, num_action_entries, max_entries);
550 for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
551 npc_config_kpmaction(rvu, blkaddr,
552 npc_get_kpu_action_nth_entry(rvu, profile, idx),
553 kpm, entry, false);
554 }
555
556 static void
npc_enable_kpm_entry(struct rvu * rvu,int blkaddr,int kpm,int num_entries)557 npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries)
558 {
559 u64 entry_mask;
560
561 entry_mask = npc_enable_mask(num_entries);
562 /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */
563 if (!rvu->kpu.custom)
564 entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0);
565 rvu_write64(rvu, blkaddr,
566 NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask);
567 if (num_entries <= 64) {
568 /* Disable all the entries in W1, W2 and W3 */
569 rvu_write64(rvu, blkaddr,
570 NPC_AF_KPMX_ENTRY_DISX(kpm, 1),
571 npc_enable_mask(0));
572 rvu_write64(rvu, blkaddr,
573 NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
574 npc_enable_mask(0));
575 rvu_write64(rvu, blkaddr,
576 NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
577 npc_enable_mask(0));
578 return;
579 }
580
581 num_entries = num_entries - 64;
582 entry_mask = npc_enable_mask(num_entries);
583 rvu_write64(rvu, blkaddr,
584 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask);
585 if (num_entries <= 64) {
586 /* Disable all the entries in W2 and W3 */
587 rvu_write64(rvu, blkaddr,
588 NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
589 npc_enable_mask(0));
590 rvu_write64(rvu, blkaddr,
591 NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
592 npc_enable_mask(0));
593 return;
594 }
595
596 num_entries = num_entries - 64;
597 entry_mask = npc_enable_mask(num_entries);
598 rvu_write64(rvu, blkaddr,
599 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask);
600 if (num_entries <= 64) {
601 /* Disable all the entries in W3 */
602 rvu_write64(rvu, blkaddr,
603 NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
604 npc_enable_mask(0));
605 return;
606 }
607
608 num_entries = num_entries - 64;
609 entry_mask = npc_enable_mask(num_entries);
610 rvu_write64(rvu, blkaddr,
611 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask);
612 }
613
614 #define KPU_OFFSET 8
npc_program_kpm_profile(struct rvu * rvu,int blkaddr,int num_kpms)615 static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms)
616 {
617 const struct npc_kpu_profile *profile1, *profile2;
618 int pfl1_num_cam_entries, pfl2_num_cam_entries;
619 int idx, total_cam_entries;
620
621 for (idx = 0; idx < num_kpms; idx++) {
622 profile1 = &rvu->kpu.kpu[idx];
623 pfl1_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile1);
624 npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1);
625 profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET];
626 pfl2_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile2);
627
628 npc_program_single_kpm_profile(rvu, blkaddr, idx,
629 pfl1_num_cam_entries,
630 profile2);
631 total_cam_entries = pfl1_num_cam_entries + pfl2_num_cam_entries;
632 npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries);
633 rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx),
634 pfl1_num_cam_entries);
635 /* Enable the KPUs associated with this KPM */
636 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01);
637 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET),
638 0x01);
639 }
640 }
641
npc_cn20k_parser_profile_init(struct rvu * rvu,int blkaddr)642 void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr)
643 {
644 struct npc_kpu_profile_action *act;
645 struct rvu_hwinfo *hw = rvu->hw;
646 int num_pkinds, idx;
647
648 /* Disable all KPMs and their entries */
649 for (idx = 0; idx < hw->npc_kpms; idx++) {
650 rvu_write64(rvu, blkaddr,
651 NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL);
652 rvu_write64(rvu, blkaddr,
653 NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL);
654 rvu_write64(rvu, blkaddr,
655 NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL);
656 rvu_write64(rvu, blkaddr,
657 NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL);
658 }
659
660 for (idx = 0; idx < hw->npc_kpus; idx++)
661 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00);
662
663 /* Load and customize KPU profile. */
664 npc_load_kpu_profile(rvu);
665
666 /* Configure KPU and KPM mapping for second pass */
667 rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210);
668
669 /* First program IKPU profile i.e PKIND configs.
670 * Check HW max count to avoid configuring junk or
671 * writing to unsupported CSR addresses.
672 */
673 num_pkinds = rvu->kpu.pkinds;
674 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds);
675
676 /* Cn20k does not support Custom profile from filesystem */
677 for (idx = 0; idx < num_pkinds; idx++) {
678 act = npc_get_ikpu_nth_entry(rvu, idx);
679 if (!act)
680 continue;
681
682 npc_config_kpmaction(rvu, blkaddr, act,
683 0, idx, true);
684 }
685
686 /* Program KPM CAM and Action profiles */
687 npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms);
688 }
689
npc_priv_get(void)690 struct npc_priv_t *npc_priv_get(void)
691 {
692 return npc_priv;
693 }
694
npc_program_mkex_rx(struct rvu * rvu,int blkaddr,const struct npc_mcam_kex_extr * mkex_extr,u8 intf)695 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr,
696 const struct npc_mcam_kex_extr *mkex_extr,
697 u8 intf)
698 {
699 u8 num_extr = rvu->hw->npc_kex_extr;
700 int extr, lt;
701 u64 val;
702
703 if (is_npc_intf_tx(intf))
704 return;
705
706 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
707 mkex_extr->keyx_cfg[NIX_INTF_RX]);
708
709 /* Program EXTRACTOR */
710 for (extr = 0; extr < num_extr; extr++)
711 rvu_write64(rvu, blkaddr,
712 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
713 mkex_extr->intf_extr_lid[intf][extr]);
714
715 /* Program EXTRACTOR_LTYPE */
716 for (extr = 0; extr < num_extr; extr++) {
717 for (lt = 0; lt < NPC_MAX_LT; lt++) {
718 val = mkex_extr->intf_extr_lt[intf][extr][lt];
719 CN20K_SET_EXTR_LT(intf, extr, lt, val);
720 }
721 }
722 }
723
npc_program_mkex_tx(struct rvu * rvu,int blkaddr,const struct npc_mcam_kex_extr * mkex_extr,u8 intf)724 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr,
725 const struct npc_mcam_kex_extr *mkex_extr,
726 u8 intf)
727 {
728 u8 num_extr = rvu->hw->npc_kex_extr;
729 int extr, lt;
730 u64 val;
731
732 if (is_npc_intf_rx(intf))
733 return;
734
735 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
736 mkex_extr->keyx_cfg[NIX_INTF_TX]);
737
738 /* Program EXTRACTOR */
739 for (extr = 0; extr < num_extr; extr++)
740 rvu_write64(rvu, blkaddr,
741 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
742 mkex_extr->intf_extr_lid[intf][extr]);
743
744 /* Program EXTRACTOR_LTYPE */
745 for (extr = 0; extr < num_extr; extr++) {
746 for (lt = 0; lt < NPC_MAX_LT; lt++) {
747 val = mkex_extr->intf_extr_lt[intf][extr][lt];
748 CN20K_SET_EXTR_LT(intf, extr, lt, val);
749 }
750 }
751 }
752
npc_program_mkex_profile(struct rvu * rvu,int blkaddr,const struct npc_mcam_kex_extr * mkex_extr)753 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,
754 const struct npc_mcam_kex_extr *mkex_extr)
755 {
756 struct rvu_hwinfo *hw = rvu->hw;
757 u8 intf;
758
759 for (intf = 0; intf < hw->npc_intfs; intf++) {
760 npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf);
761 npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf);
762 }
763
764 /* Programme mkex hash profile */
765 npc_program_mkex_hash(rvu, blkaddr);
766 }
767
npc_cn20k_load_mkex_profile(struct rvu * rvu,int blkaddr,const char * mkex_profile)768 void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr,
769 const char *mkex_profile)
770 {
771 struct npc_mcam_kex_extr *mcam_kex_extr;
772 struct device *dev = &rvu->pdev->dev;
773 void __iomem *mkex_prfl_addr = NULL;
774 u64 prfl_sz;
775 int ret;
776
777 /* If user not selected mkex profile */
778 if (rvu->kpu_fwdata_sz ||
779 !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN))
780 goto program_mkex_extr;
781
782 /* Setting up the mapping for mkex profile image */
783 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz);
784 if (ret < 0)
785 goto program_mkex_extr;
786
787 mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr;
788
789 while (((s64)prfl_sz > 0) &&
790 (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) {
791 /* Compare with mkex mod_param name string */
792 if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN &&
793 !strncmp(mcam_kex_extr->name, mkex_profile,
794 MKEX_NAME_LEN)) {
795 rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr;
796 goto program_mkex_extr;
797 }
798
799 mcam_kex_extr++;
800 prfl_sz -= sizeof(struct npc_mcam_kex_extr);
801 }
802 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);
803 rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get();
804
805 program_mkex_extr:
806 dev_info(rvu->dev, "Using %s mkex profile\n",
807 rvu->kpu.mcam_kex_prfl.mkex_extr->name);
808 /* Program selected mkex profile */
809 npc_program_mkex_profile(rvu, blkaddr,
810 rvu->kpu.mcam_kex_prfl.mkex_extr);
811 if (mkex_prfl_addr)
812 iounmap(mkex_prfl_addr);
813 }
814
815 int
npc_cn20k_enable_mcam_entry(struct rvu * rvu,int blkaddr,int index,bool enable)816 npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
817 int index, bool enable)
818 {
819 struct npc_mcam *mcam = &rvu->hw->mcam;
820 int mcam_idx = index % mcam->banksize;
821 int bank = index / mcam->banksize;
822 u64 cfg, hw_prio;
823 u8 kw_type;
824
825 if (index < 0 || index >= mcam->total_entries)
826 return -EINVAL;
827
828 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
829 return -EINVAL;
830
831 if (kw_type == NPC_MCAM_KEY_X2) {
832 cfg = rvu_read64(rvu, blkaddr,
833 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
834 bank));
835 hw_prio = cfg & GENMASK_ULL(30, 24);
836 cfg = enable ? 1 : 0;
837 cfg |= hw_prio;
838 rvu_write64(rvu, blkaddr,
839 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
840 cfg);
841 goto update_en_map;
842 }
843
844 /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks
845 * need to be programmed with the same value.
846 */
847 for (bank = 0; bank < mcam->banks_per_entry; bank++) {
848 cfg = rvu_read64(rvu, blkaddr,
849 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
850 bank));
851 hw_prio = cfg & GENMASK_ULL(30, 24);
852 cfg = enable ? 1 : 0;
853 cfg |= hw_prio;
854 rvu_write64(rvu, blkaddr,
855 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
856 cfg);
857 }
858
859 update_en_map:
860 if (enable)
861 set_bit(index, npc_priv->en_map);
862 else
863 clear_bit(index, npc_priv->en_map);
864
865 return 0;
866 }
867
868 static void
npc_clear_x2_entry(struct rvu * rvu,int blkaddr,int bank,int index)869 npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index)
870 {
871 rvu_write64(rvu, blkaddr,
872 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
873 0);
874 rvu_write64(rvu, blkaddr,
875 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
876 0);
877
878 rvu_write64(rvu, blkaddr,
879 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0);
880 rvu_write64(rvu, blkaddr,
881 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0);
882
883 rvu_write64(rvu, blkaddr,
884 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0);
885 rvu_write64(rvu, blkaddr,
886 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0);
887
888 rvu_write64(rvu, blkaddr,
889 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0);
890 rvu_write64(rvu, blkaddr,
891 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0);
892
893 rvu_write64(rvu, blkaddr,
894 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0);
895 rvu_write64(rvu, blkaddr,
896 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0);
897
898 /* Clear corresponding stats register */
899 rvu_write64(rvu, blkaddr,
900 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0);
901 }
902
903 int
npc_cn20k_clear_mcam_entry(struct rvu * rvu,int blkaddr,int mcam_idx)904 npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx)
905 {
906 struct npc_mcam *mcam = &rvu->hw->mcam;
907 int bank = npc_get_bank(mcam, mcam_idx);
908 u8 kw_type;
909 int index;
910
911 if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type))
912 return -EINVAL;
913
914 index = mcam_idx & (mcam->banksize - 1);
915
916 if (kw_type == NPC_MCAM_KEY_X2) {
917 npc_clear_x2_entry(rvu, blkaddr, bank, index);
918 return 0;
919 }
920
921 /* For NPC_MCAM_KEY_X4 keys, both the banks
922 * need to be programmed with the same value.
923 */
924 for (bank = 0; bank < mcam->banks_per_entry; bank++)
925 npc_clear_x2_entry(rvu, blkaddr, bank, index);
926
927 return 0;
928 }
929
npc_cn20k_get_keyword(struct cn20k_mcam_entry * entry,int idx,u64 * cam0,u64 * cam1)930 static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx,
931 u64 *cam0, u64 *cam1)
932 {
933 u64 kw_mask;
934
935 /* The two banks of every MCAM entry are used as a single double-wide
936 * entry that is compared with the search key as follows:
937 *
938 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0]
939 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1]
940 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2]
941 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3]
942 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4]
943 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5]
944 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6]
945 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7]
946 */
947 *cam1 = entry->kw[idx];
948 kw_mask = entry->kw_mask[idx];
949 *cam1 &= kw_mask;
950 *cam0 = ~*cam1 & kw_mask;
951 }
952
953 /*-------------------------------------------------------------------------
954 *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > |
955 * in | | | | || | |
956 * profile| len | | | ||len | type |
957 *-------------------------------------------------------------------------
958 *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 |
959 *------------------------------------------------------------------------|
960 *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN |
961 *------------------------------------------------------------------------|
962 *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 |
963 *------------------------------------------------------------------------|
964 *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN |
965 *------------------------------------------------------------------------|
966 *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 |
967 *------------------------------------------------------------------------|
968 *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN |
969 *------------------------------------------------------------------------|
970 *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN |
971 *------------------------------------------------------------------------|
972 * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL |
973 *------------------------------------------------------------------------|
974 */
npc_cn20k_config_kw_x2(struct rvu * rvu,struct npc_mcam * mcam,int blkaddr,int index,u8 intf,struct cn20k_mcam_entry * entry,int bank,u8 kw_type,int kw,u8 req_kw_type)975 static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam,
976 int blkaddr, int index, u8 intf,
977 struct cn20k_mcam_entry *entry,
978 int bank, u8 kw_type, int kw,
979 u8 req_kw_type)
980 {
981 u64 intf_ext = 0, intf_ext_mask = 0;
982 u8 tx_intf_mask = ~intf & 0x3;
983 u8 tx_intf = intf, kex_type;
984 u8 kw_type_mask = ~kw_type;
985 u64 cam0, cam1, kex_cfg;
986
987 if (is_npc_intf_tx(intf)) {
988 /* Last bit must be set and rest don't care
989 * for TX interfaces
990 */
991 tx_intf_mask = 0x1;
992 tx_intf = intf & tx_intf_mask;
993 tx_intf_mask = ~tx_intf & tx_intf_mask;
994 }
995
996 kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
997 kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32;
998 if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) &&
999 kw_type == NPC_MCAM_KEY_X2) {
1000 kw_type = 0;
1001 kw_type_mask = 0;
1002 }
1003
1004 /* Say, we need to write x2 keyword in an x4 subbank.
1005 * req_kw_type will be x2, and kw_type will be x4.
1006 * So in the case ignore kw bits in mcam.
1007 */
1008 if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) {
1009 kw_type = 0;
1010 kw_type_mask = 0;
1011 }
1012
1013 intf_ext = ((u64)kw_type << 16) | tx_intf;
1014 intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) |
1015 tx_intf_mask;
1016 rvu_write64(rvu, blkaddr,
1017 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
1018 intf_ext);
1019 rvu_write64(rvu, blkaddr,
1020 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
1021 intf_ext_mask);
1022
1023 /* Set the match key */
1024 npc_cn20k_get_keyword(entry, kw, &cam0, &cam1);
1025 rvu_write64(rvu, blkaddr,
1026 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1),
1027 cam1);
1028 rvu_write64(rvu, blkaddr,
1029 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0),
1030 cam0);
1031
1032 npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1);
1033 rvu_write64(rvu, blkaddr,
1034 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1),
1035 cam1);
1036 rvu_write64(rvu, blkaddr,
1037 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0),
1038 cam0);
1039
1040 npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1);
1041 rvu_write64(rvu, blkaddr,
1042 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1),
1043 cam1);
1044 rvu_write64(rvu, blkaddr,
1045 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0),
1046 cam0);
1047
1048 npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1);
1049 rvu_write64(rvu, blkaddr,
1050 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1),
1051 cam1);
1052 rvu_write64(rvu, blkaddr,
1053 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0),
1054 cam0);
1055 }
1056
npc_cn20k_config_kw_x4(struct rvu * rvu,struct npc_mcam * mcam,int blkaddr,int index,u8 intf,struct cn20k_mcam_entry * entry,u8 kw_type,u8 req_kw_type)1057 static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam,
1058 int blkaddr, int index, u8 intf,
1059 struct cn20k_mcam_entry *entry,
1060 u8 kw_type, u8 req_kw_type)
1061 {
1062 int kw = 0, bank;
1063
1064 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4)
1065 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
1066 index, intf,
1067 entry, bank, kw_type,
1068 kw, req_kw_type);
1069 }
1070
npc_cn20k_config_mcam_entry(struct rvu * rvu,int blkaddr,int index,u8 intf,struct cn20k_mcam_entry * entry,bool enable,u8 hw_prio,u8 req_kw_type)1071 int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
1072 u8 intf, struct cn20k_mcam_entry *entry,
1073 bool enable, u8 hw_prio, u8 req_kw_type)
1074 {
1075 struct npc_mcam *mcam = &rvu->hw->mcam;
1076 int mcam_idx = index % mcam->banksize;
1077 int bank = index / mcam->banksize;
1078 u64 bank_cfg = (u64)hw_prio << 24;
1079 int kw = 0;
1080 u8 kw_type;
1081
1082 if (index < 0 || index >= mcam->total_entries)
1083 return -EINVAL;
1084
1085 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
1086 return -EINVAL;
1087
1088 /* Disable before mcam entry update */
1089 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false))
1090 return -EINVAL;
1091
1092 /* CAM1 takes the comparison value and
1093 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'.
1094 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0
1095 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1
1096 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare.
1097 */
1098 if (kw_type == NPC_MCAM_KEY_X2) {
1099 /* Clear mcam entry to avoid writes being suppressed by NPC */
1100 npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx);
1101 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
1102 mcam_idx, intf, entry,
1103 bank, kw_type, kw, req_kw_type);
1104 /* Set 'action' */
1105 rvu_write64(rvu, blkaddr,
1106 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1107 bank, 0),
1108 entry->action);
1109
1110 /* Set 'action2' for inline receive */
1111 rvu_write64(rvu, blkaddr,
1112 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1113 bank, 2),
1114 entry->action2);
1115
1116 /* Set TAG 'action' */
1117 rvu_write64(rvu, blkaddr,
1118 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1119 bank, 1),
1120 entry->vtag_action);
1121
1122 /* Set HW priority */
1123 rvu_write64(rvu, blkaddr,
1124 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
1125 bank_cfg);
1126
1127 } else {
1128 /* Clear mcam entry to avoid writes being suppressed by NPC */
1129 npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx);
1130 npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx);
1131
1132 npc_cn20k_config_kw_x4(rvu, mcam, blkaddr,
1133 mcam_idx, intf, entry,
1134 kw_type, req_kw_type);
1135 for (bank = 0; bank < mcam->banks_per_entry; bank++) {
1136 /* Set 'action' */
1137 rvu_write64(rvu, blkaddr,
1138 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1139 bank, 0),
1140 entry->action);
1141
1142 /* Set TAG 'action' */
1143 rvu_write64(rvu, blkaddr,
1144 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1145 bank, 1),
1146 entry->vtag_action);
1147
1148 /* Set 'action2' for inline receive */
1149 rvu_write64(rvu, blkaddr,
1150 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
1151 bank, 2),
1152 entry->action2);
1153
1154 /* Set HW priority */
1155 rvu_write64(rvu, blkaddr,
1156 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
1157 bank_cfg);
1158 }
1159 }
1160
1161 /* TODO: */
1162 /* PF installing VF rule */
1163 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable))
1164 return -EINVAL;
1165
1166 return 0;
1167 }
1168
npc_cn20k_copy_mcam_entry(struct rvu * rvu,int blkaddr,u16 src,u16 dest)1169 int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
1170 {
1171 struct npc_mcam *mcam = &rvu->hw->mcam;
1172 u64 cfg, sreg, dreg, soff, doff;
1173 u8 src_kwtype, dest_kwtype;
1174 int bank, i, sb, db;
1175 int dbank, sbank;
1176
1177 if (src >= mcam->total_entries || dest >= mcam->total_entries)
1178 return -EINVAL;
1179
1180 dbank = npc_get_bank(mcam, dest);
1181 sbank = npc_get_bank(mcam, src);
1182
1183 if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype))
1184 return -EINVAL;
1185
1186 if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype))
1187 return -EINVAL;
1188
1189 if (src_kwtype != dest_kwtype)
1190 return -EINVAL;
1191
1192 src &= (mcam->banksize - 1);
1193 dest &= (mcam->banksize - 1);
1194
1195 /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */
1196 for (bank = 0; bank < mcam->banks_per_entry; bank++) {
1197 sb = sbank + bank;
1198 sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0);
1199 db = dbank + bank;
1200 dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0);
1201 for (i = 0; i < 10; i++) {
1202 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8));
1203 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg);
1204 }
1205
1206 /* Copy action */
1207 for (i = 0; i < 3; i++) {
1208 soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src,
1209 sb, i);
1210 cfg = rvu_read64(rvu, blkaddr, soff);
1211
1212 doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db,
1213 i);
1214 rvu_write64(rvu, blkaddr, doff, cfg);
1215 }
1216
1217 /* Copy bank configuration */
1218 cfg = rvu_read64(rvu, blkaddr,
1219 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb));
1220 rvu_write64(rvu, blkaddr,
1221 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg);
1222 if (src_kwtype == NPC_MCAM_KEY_X2)
1223 break;
1224 }
1225
1226 return 0;
1227 }
1228
npc_cn20k_fill_entryword(struct cn20k_mcam_entry * entry,int idx,u64 cam0,u64 cam1)1229 static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx,
1230 u64 cam0, u64 cam1)
1231 {
1232 entry->kw[idx] = cam1;
1233 entry->kw_mask[idx] = cam1 ^ cam0;
1234 }
1235
npc_cn20k_read_mcam_entry(struct rvu * rvu,int blkaddr,u16 index,struct cn20k_mcam_entry * entry,u8 * intf,u8 * ena,u8 * hw_prio)1236 int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
1237 struct cn20k_mcam_entry *entry,
1238 u8 *intf, u8 *ena, u8 *hw_prio)
1239 {
1240 struct npc_mcam *mcam = &rvu->hw->mcam;
1241 u64 cam0, cam1, bank_cfg, cfg;
1242 int kw = 0, bank;
1243 u8 kw_type;
1244
1245 if (index >= mcam->total_entries)
1246 return -EINVAL;
1247
1248 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
1249 return -EINVAL;
1250
1251 bank = npc_get_bank(mcam, index);
1252 index &= (mcam->banksize - 1);
1253
1254 cfg = rvu_read64(rvu, blkaddr,
1255 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0));
1256 entry->action = cfg;
1257
1258 cfg = rvu_read64(rvu, blkaddr,
1259 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1));
1260 entry->vtag_action = cfg;
1261
1262 cfg = rvu_read64(rvu, blkaddr,
1263 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2));
1264 entry->action2 = cfg;
1265
1266 cfg = rvu_read64(rvu, blkaddr,
1267 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index,
1268 bank, 1)) & 3;
1269 *intf = cfg;
1270
1271 bank_cfg = rvu_read64(rvu, blkaddr,
1272 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank));
1273 *ena = bank_cfg & 0x1;
1274 *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24;
1275 if (kw_type == NPC_MCAM_KEY_X2) {
1276 cam1 = rvu_read64(rvu, blkaddr,
1277 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
1278 bank,
1279 1));
1280 cam0 = rvu_read64(rvu, blkaddr,
1281 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
1282 bank,
1283 0));
1284 npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
1285
1286 cam1 = rvu_read64(rvu, blkaddr,
1287 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
1288 bank,
1289 1));
1290 cam0 = rvu_read64(rvu, blkaddr,
1291 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
1292 bank,
1293 0));
1294 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
1295
1296 cam1 = rvu_read64(rvu, blkaddr,
1297 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
1298 bank,
1299 1));
1300 cam0 = rvu_read64(rvu, blkaddr,
1301 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
1302 bank,
1303 0));
1304 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
1305
1306 cam1 = rvu_read64(rvu, blkaddr,
1307 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
1308 bank,
1309 1));
1310 cam0 = rvu_read64(rvu, blkaddr,
1311 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
1312 bank,
1313 0));
1314 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
1315 return 0;
1316 }
1317
1318 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) {
1319 cam1 = rvu_read64(rvu, blkaddr,
1320 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
1321 bank,
1322 1));
1323 cam0 = rvu_read64(rvu, blkaddr,
1324 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
1325 bank,
1326 0));
1327 npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
1328
1329 cam1 = rvu_read64(rvu, blkaddr,
1330 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
1331 bank,
1332 1));
1333 cam0 = rvu_read64(rvu, blkaddr,
1334 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
1335 bank,
1336 0));
1337 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
1338
1339 cam1 = rvu_read64(rvu, blkaddr,
1340 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
1341 bank,
1342 1));
1343 cam0 = rvu_read64(rvu, blkaddr,
1344 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
1345 bank,
1346 0));
1347 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
1348
1349 cam1 = rvu_read64(rvu, blkaddr,
1350 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
1351 bank,
1352 1));
1353 cam0 = rvu_read64(rvu, blkaddr,
1354 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
1355 bank,
1356 0));
1357 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
1358 }
1359
1360 return 0;
1361 }
1362
rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu * rvu,struct npc_cn20k_mcam_write_entry_req * req,struct msg_rsp * rsp)1363 int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu,
1364 struct npc_cn20k_mcam_write_entry_req *req,
1365 struct msg_rsp *rsp)
1366 {
1367 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
1368 struct npc_mcam *mcam = &rvu->hw->mcam;
1369 u16 pcifunc = req->hdr.pcifunc;
1370 int blkaddr, rc;
1371 u8 nix_intf;
1372
1373 req->entry = npc_cn20k_vidx2idx(req->entry);
1374
1375 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1376 if (blkaddr < 0)
1377 return NPC_MCAM_INVALID_REQ;
1378
1379 mutex_lock(&mcam->lock);
1380 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
1381 if (rc)
1382 goto exit;
1383
1384 if (!is_npc_interface_valid(rvu, req->intf)) {
1385 rc = NPC_MCAM_INVALID_REQ;
1386 goto exit;
1387 }
1388
1389 if (is_npc_intf_tx(req->intf))
1390 nix_intf = pfvf->nix_tx_intf;
1391 else
1392 nix_intf = pfvf->nix_rx_intf;
1393
1394 /* For AF installed rules, the nix_intf should be set to target NIX */
1395 if (is_pffunc_af(req->hdr.pcifunc))
1396 nix_intf = req->intf;
1397
1398 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf,
1399 &req->entry_data, req->enable_entry,
1400 req->hw_prio, req->req_kw_type);
1401
1402 exit:
1403 mutex_unlock(&mcam->lock);
1404 return rc;
1405 }
1406
rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu * rvu,struct npc_mcam_read_entry_req * req,struct npc_cn20k_mcam_read_entry_rsp * rsp)1407 int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu,
1408 struct npc_mcam_read_entry_req *req,
1409 struct npc_cn20k_mcam_read_entry_rsp *rsp)
1410 {
1411 struct npc_mcam *mcam = &rvu->hw->mcam;
1412 u16 pcifunc = req->hdr.pcifunc;
1413 int blkaddr, rc;
1414
1415 req->entry = npc_cn20k_vidx2idx(req->entry);
1416
1417 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1418 if (blkaddr < 0)
1419 return NPC_MCAM_INVALID_REQ;
1420
1421 mutex_lock(&mcam->lock);
1422 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
1423 if (rc)
1424 goto fail;
1425
1426 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry,
1427 &rsp->entry_data, &rsp->intf,
1428 &rsp->enable, &rsp->hw_prio);
1429 fail:
1430 mutex_unlock(&mcam->lock);
1431 return rc;
1432 }
1433
rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu * rvu,struct npc_cn20k_mcam_alloc_and_write_entry_req * req,struct npc_mcam_alloc_and_write_entry_rsp * rsp)1434 int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu,
1435 struct npc_cn20k_mcam_alloc_and_write_entry_req *req,
1436 struct npc_mcam_alloc_and_write_entry_rsp *rsp)
1437 {
1438 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
1439 struct npc_mcam_free_entry_req free_req = { 0 };
1440 struct npc_mcam_alloc_entry_req entry_req;
1441 struct npc_mcam_alloc_entry_rsp entry_rsp;
1442 struct npc_mcam *mcam = &rvu->hw->mcam;
1443 u16 entry = NPC_MCAM_ENTRY_INVALID;
1444 struct msg_rsp free_rsp;
1445 int blkaddr, rc, err;
1446 u8 nix_intf;
1447
1448 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1449 if (blkaddr < 0)
1450 return NPC_MCAM_INVALID_REQ;
1451
1452 if (!is_npc_interface_valid(rvu, req->intf))
1453 return NPC_MCAM_INVALID_REQ;
1454
1455 /* Try to allocate a MCAM entry */
1456 entry_req.hdr.pcifunc = req->hdr.pcifunc;
1457 entry_req.contig = true;
1458 entry_req.ref_prio = req->ref_prio;
1459 entry_req.ref_entry = req->ref_entry;
1460 entry_req.count = 1;
1461 entry_req.virt = req->virt;
1462
1463 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,
1464 &entry_req, &entry_rsp);
1465 if (rc)
1466 return rc;
1467
1468 if (!entry_rsp.count)
1469 return NPC_MCAM_ALLOC_FAILED;
1470
1471 /* entry_req.count is 1, so single entry is allocated */
1472 entry = npc_cn20k_vidx2idx(entry_rsp.entry);
1473
1474 mutex_lock(&mcam->lock);
1475
1476 if (is_npc_intf_tx(req->intf))
1477 nix_intf = pfvf->nix_tx_intf;
1478 else
1479 nix_intf = pfvf->nix_rx_intf;
1480
1481 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf,
1482 &req->entry_data, req->enable_entry,
1483 req->hw_prio, req->req_kw_type);
1484
1485 mutex_unlock(&mcam->lock);
1486
1487 if (rc) {
1488 free_req.hdr.pcifunc = req->hdr.pcifunc;
1489 free_req.entry = entry_rsp.entry;
1490 err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
1491 if (err)
1492 dev_err(rvu->dev,
1493 "%s: Error to free mcam idx %u\n",
1494 __func__, entry_rsp.entry);
1495 return rc;
1496 }
1497
1498 rsp->entry = entry_rsp.entry;
1499 return 0;
1500 }
1501
rvu_npc_get_base_steer_rule_type(struct rvu * rvu,u16 pcifunc)1502 static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc)
1503 {
1504 if (is_lbk_vf(rvu, pcifunc))
1505 return NIXLF_PROMISC_ENTRY;
1506
1507 return NIXLF_UCAST_ENTRY;
1508 }
1509
rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu * rvu,struct msg_req * req,struct npc_cn20k_mcam_read_base_rule_rsp * rsp)1510 int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu,
1511 struct msg_req *req,
1512 struct npc_cn20k_mcam_read_base_rule_rsp *rsp)
1513 {
1514 struct npc_mcam *mcam = &rvu->hw->mcam;
1515 int index, blkaddr, nixlf, rc = 0;
1516 u16 pcifunc = req->hdr.pcifunc;
1517 u8 intf, enable, hw_prio;
1518 struct rvu_pfvf *pfvf;
1519 int rl_type;
1520
1521 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1522 if (blkaddr < 0)
1523 return NPC_MCAM_INVALID_REQ;
1524
1525 /* Return the channel number in case of PF */
1526 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
1527 pfvf = rvu_get_pfvf(rvu, pcifunc);
1528 rsp->entry.kw[0] = pfvf->rx_chan_base;
1529 rsp->entry.kw_mask[0] = 0xFFFULL;
1530 goto out;
1531 }
1532
1533 /* Find the pkt steering rule installed by PF to this VF */
1534 mutex_lock(&mcam->lock);
1535 for (index = 0; index < mcam->bmap_entries; index++) {
1536 if (mcam->entry2target_pffunc[index] == pcifunc)
1537 goto read_entry;
1538 }
1539
1540 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
1541 if (rc < 0) {
1542 mutex_unlock(&mcam->lock);
1543 goto out;
1544 }
1545
1546 rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc);
1547
1548 /* Read the default ucast entry if there is no pkt steering rule */
1549 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type);
1550 if (index < 0) {
1551 mutex_unlock(&mcam->lock);
1552 goto out;
1553 }
1554
1555 read_entry:
1556 /* Read the mcam entry */
1557 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index,
1558 &rsp->entry, &intf,
1559 &enable, &hw_prio);
1560 mutex_unlock(&mcam->lock);
1561 out:
1562 return rc;
1563 }
1564
npc_map2cn20k_flag(u8 flag)1565 static u8 npc_map2cn20k_flag(u8 flag)
1566 {
1567 switch (flag) {
1568 case NPC_F_LC_U_IP_FRAG:
1569 return NPC_CN20K_F_LC_L_IP_FRAG;
1570
1571 case NPC_F_LC_U_IP6_FRAG:
1572 return NPC_CN20K_F_LC_L_IP6_FRAG;
1573
1574 case NPC_F_LC_L_6TO4:
1575 return NPC_CN20K_F_LC_L_6TO4;
1576
1577 case NPC_F_LC_L_MPLS_IN_IP:
1578 return NPC_CN20K_F_LC_U_MPLS_IN_IP;
1579
1580 case NPC_F_LC_L_IP6_TUN_IP6:
1581 return NPC_CN20K_F_LC_U_IP6_TUN_IP6;
1582
1583 case NPC_F_LC_L_IP6_MPLS_IN_IP:
1584 return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP;
1585
1586 default:
1587 break;
1588 }
1589
1590 WARN(1, "%s: Invalid flag=%u\n", __func__, flag);
1591 return 0xff;
1592 }
1593
npc_cn20k_translate_action_flags(struct npc_kpu_profile_action * act)1594 static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act)
1595 {
1596 u8 ltype, val;
1597
1598 if (act->lid != NPC_LID_LC)
1599 return;
1600
1601 ltype = act->ltype;
1602 if (ltype != NPC_LT_LC_IP &&
1603 ltype != NPC_LT_LC_IP6 &&
1604 ltype != NPC_LT_LC_IP_OPT &&
1605 ltype != NPC_LT_LC_IP6_EXT)
1606 return;
1607
1608 switch (act->flags) {
1609 case NPC_F_LC_U_IP_FRAG:
1610 case NPC_F_LC_U_IP6_FRAG:
1611 case NPC_F_LC_L_6TO4:
1612 case NPC_F_LC_L_MPLS_IN_IP:
1613 case NPC_F_LC_L_IP6_TUN_IP6:
1614 case NPC_F_LC_L_IP6_MPLS_IN_IP:
1615 val = npc_map2cn20k_flag(act->flags);
1616 if (val != 0xFF)
1617 act->flags = val;
1618 break;
1619 default:
1620 break;
1621 }
1622 }
1623
1624 void
npc_cn20k_update_action_entries_n_flags(struct rvu * rvu,struct npc_kpu_profile_adapter * pfl)1625 npc_cn20k_update_action_entries_n_flags(struct rvu *rvu,
1626 struct npc_kpu_profile_adapter *pfl)
1627 {
1628 struct npc_kpu_profile_action *action;
1629 int entries;
1630
1631 for (int i = 0; i < pfl->kpus; i++) {
1632 action = pfl->kpu[i].action;
1633 entries = pfl->kpu[i].action_entries;
1634
1635 for (int j = 0; j < entries; j++)
1636 npc_cn20k_translate_action_flags(&action[j]);
1637 }
1638 }
1639
npc_cn20k_apply_custom_kpu(struct rvu * rvu,struct npc_kpu_profile_adapter * profile)1640 int npc_cn20k_apply_custom_kpu(struct rvu *rvu,
1641 struct npc_kpu_profile_adapter *profile)
1642 {
1643 const struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata;
1644 size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
1645 struct npc_kpu_profile_action *action;
1646 struct npc_kpu_profile_cam *cam;
1647 struct npc_kpu_fwdata *fw_kpu;
1648 size_t offset = 0;
1649 u16 kpu, entry;
1650 int entries;
1651
1652 hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
1653
1654 if (rvu->kpu_fwdata_sz < hdr_sz) {
1655 dev_warn(rvu->dev, "Invalid KPU profile size\n");
1656 return -EINVAL;
1657 }
1658
1659 if (le64_to_cpu(fw->signature) != KPU_SIGN) {
1660 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n",
1661 fw->signature);
1662 return -EINVAL;
1663 }
1664
1665 /* Verify if the using known profile structure */
1666 if (NPC_KPU_VER_MAJ(profile->version) >
1667 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) {
1668 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n",
1669 NPC_KPU_VER_MAJ(profile->version),
1670 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER));
1671 return -EINVAL;
1672 }
1673
1674 /* Verify if profile is aligned with the required kernel changes */
1675 if (NPC_KPU_VER_MIN(profile->version) <
1676 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) {
1677 dev_warn(rvu->dev,
1678 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n",
1679 NPC_KPU_VER_MAJ(profile->version),
1680 NPC_KPU_VER_MIN(profile->version),
1681 NPC_KPU_VER_PATCH(profile->version),
1682 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER),
1683 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER),
1684 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER));
1685 return -EINVAL;
1686 }
1687
1688 /* Verify if profile fits the HW */
1689 if (fw->kpus > rvu->hw->npc_kpus) {
1690 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus,
1691 rvu->hw->npc_kpus);
1692 return -EINVAL;
1693 }
1694
1695 /* Check if there is enough memory */
1696 if (fw->kpus > profile->kpus) {
1697 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus,
1698 profile->kpus);
1699 return -EINVAL;
1700 }
1701
1702 profile->mcam_kex_prfl.mkex_extr = &fw->mkex;
1703 if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) {
1704 dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n",
1705 profile->mcam_kex_prfl.mkex_extr->mkex_sign);
1706 return -EINVAL;
1707 }
1708
1709 profile->custom = 1;
1710 profile->name = fw->name;
1711 profile->version = le64_to_cpu(fw->version);
1712 profile->lt_def = &fw->lt_def;
1713
1714 for (kpu = 0; kpu < fw->kpus; kpu++) {
1715 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset);
1716 if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT)
1717 dev_warn(rvu->dev,
1718 "Too many custom entries on KPU%d: %d > %d\n",
1719 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
1720 entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
1721 cam = (struct npc_kpu_profile_cam *)fw_kpu->data;
1722 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam);
1723 action = (struct npc_kpu_profile_action *)(fw->data + offset);
1724 offset += fw_kpu->entries * sizeof(*action);
1725 if (rvu->kpu_fwdata_sz < hdr_sz + offset) {
1726 dev_warn(rvu->dev,
1727 "Profile size mismatch on KPU%i parsing.\n",
1728 kpu + 1);
1729 return -EINVAL;
1730 }
1731
1732 for (entry = 0; entry < entries; entry++) {
1733 profile->kpu[kpu].cam[entry] = cam[entry];
1734 profile->kpu[kpu].action[entry] = action[entry];
1735 npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]);
1736 }
1737 }
1738
1739 return 0;
1740 }
1741
npc_mcam_idx_2_key_type(struct rvu * rvu,u16 mcam_idx,u8 * key_type)1742 int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type)
1743 {
1744 struct npc_subbank *sb;
1745 int bank_off, sb_id;
1746
1747 /* mcam_idx should be less than (2 * bank depth) */
1748 if (mcam_idx >= npc_priv->bank_depth * 2) {
1749 dev_err(rvu->dev, "%s: bad params\n",
1750 __func__);
1751 return -EINVAL;
1752 }
1753
1754 /* find mcam offset per bank */
1755 bank_off = mcam_idx & (npc_priv->bank_depth - 1);
1756
1757 /* Find subbank id */
1758 sb_id = bank_off / npc_priv->subbank_depth;
1759
1760 /* Check if subbank id is more than maximum
1761 * number of subbanks available
1762 */
1763 if (sb_id >= npc_priv->num_subbanks) {
1764 dev_err(rvu->dev, "%s: invalid subbank %d\n",
1765 __func__, sb_id);
1766 return -EINVAL;
1767 }
1768
1769 sb = &npc_priv->sb[sb_id];
1770
1771 *key_type = sb->key_type;
1772
1773 return 0;
1774 }
1775
npc_subbank_idx_2_mcam_idx(struct rvu * rvu,struct npc_subbank * sb,u16 sub_off,u16 * mcam_idx)1776 static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb,
1777 u16 sub_off, u16 *mcam_idx)
1778 {
1779 int off, bot;
1780
1781 /* for x4 section, maximum allowed subbank index =
1782 * subsection depth - 1
1783 */
1784 if (sb->key_type == NPC_MCAM_KEY_X4 &&
1785 sub_off >= npc_priv->subbank_depth) {
1786 dev_err(rvu->dev,
1787 "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u",
1788 __func__, sb->idx, sub_off);
1789 return -EINVAL;
1790 }
1791
1792 /* for x2 section, maximum allowed subbank index =
1793 * 2 * subsection depth - 1
1794 */
1795 if (sb->key_type == NPC_MCAM_KEY_X2 &&
1796 sub_off >= npc_priv->subbank_depth * 2) {
1797 dev_err(rvu->dev,
1798 "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u",
1799 __func__, sb->idx, sub_off);
1800 return -EINVAL;
1801 }
1802
1803 /* Find subbank offset from respective subbank (w.r.t bank) */
1804 off = sub_off & (npc_priv->subbank_depth - 1);
1805
1806 /* if subsection idx is in bank1, add bank depth,
1807 * which is part of sb->b1b
1808 */
1809 bot = sub_off >= npc_priv->subbank_depth ? sb->b1b : sb->b0b;
1810
1811 *mcam_idx = bot + off;
1812 return 0;
1813 }
1814
npc_mcam_idx_2_subbank_idx(struct rvu * rvu,u16 mcam_idx,struct npc_subbank ** sb,int * sb_off)1815 int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx,
1816 struct npc_subbank **sb,
1817 int *sb_off)
1818 {
1819 int bank_off, sb_id;
1820
1821 /* mcam_idx should be less than (2 * bank depth) */
1822 if (mcam_idx >= npc_priv->bank_depth * 2) {
1823 dev_err(rvu->dev, "%s: Invalid mcam idx %u\n",
1824 __func__, mcam_idx);
1825 return -EINVAL;
1826 }
1827
1828 /* find mcam offset per bank */
1829 bank_off = mcam_idx & (npc_priv->bank_depth - 1);
1830
1831 /* Find subbank id */
1832 sb_id = bank_off / npc_priv->subbank_depth;
1833
1834 /* Check if subbank id is more than maximum
1835 * number of subbanks available
1836 */
1837 if (sb_id >= npc_priv->num_subbanks) {
1838 dev_err(rvu->dev, "%s: invalid subbank %d\n",
1839 __func__, sb_id);
1840 return -EINVAL;
1841 }
1842
1843 *sb = &npc_priv->sb[sb_id];
1844
1845 /* Subbank offset per bank */
1846 *sb_off = bank_off % npc_priv->subbank_depth;
1847
1848 /* Index in a subbank should add subbank depth
1849 * if it is in bank1
1850 */
1851 if (mcam_idx >= npc_priv->bank_depth)
1852 *sb_off += npc_priv->subbank_depth;
1853
1854 return 0;
1855 }
1856
__npc_subbank_contig_alloc(struct rvu * rvu,struct npc_subbank * sb,int key_type,int sidx,int eidx,int prio,int count,int t,int b,unsigned long * bmap,u16 * save)1857 static int __npc_subbank_contig_alloc(struct rvu *rvu,
1858 struct npc_subbank *sb,
1859 int key_type, int sidx,
1860 int eidx, int prio,
1861 int count, int t, int b,
1862 unsigned long *bmap,
1863 u16 *save)
1864 {
1865 int k, offset, delta = 0;
1866 int cnt = 0, sbd;
1867
1868 sbd = npc_priv->subbank_depth;
1869
1870 if (sidx >= npc_priv->bank_depth)
1871 delta = sbd;
1872
1873 switch (prio) {
1874 case NPC_MCAM_LOWER_PRIO:
1875 case NPC_MCAM_ANY_PRIO:
1876 /* Find an area of size 'count' from sidx to eidx */
1877 offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b,
1878 count, 0);
1879
1880 if (offset >= sbd) {
1881 dev_err(rvu->dev,
1882 "%s: Could not find contiguous(%d) entries\n",
1883 __func__, count);
1884 return -EFAULT;
1885 }
1886
1887 dev_dbg(rvu->dev,
1888 "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n",
1889 __func__, sidx, eidx, t, b, offset,
1890 count, delta);
1891
1892 for (cnt = 0; cnt < count; cnt++)
1893 save[cnt] = offset + cnt + delta;
1894
1895 break;
1896
1897 case NPC_MCAM_HIGHER_PRIO:
1898 /* Find an area of 'count' from eidx to sidx */
1899 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
1900 /* If an intermediate slot is not free,
1901 * reset the counter (cnt) to zero as
1902 * request is for contiguous.
1903 */
1904 if (test_bit(k, bmap)) {
1905 cnt = 0;
1906 continue;
1907 }
1908
1909 save[cnt++] = k + delta;
1910 }
1911 break;
1912 }
1913
1914 /* Found 'count' number of free slots */
1915 if (cnt == count)
1916 return 0;
1917
1918 dev_dbg(rvu->dev,
1919 "%s: Could not find contiguous(%d) entries in subbank=%u\n",
1920 __func__, count, sb->idx);
1921 return -EFAULT;
1922 }
1923
__npc_subbank_non_contig_alloc(struct rvu * rvu,struct npc_subbank * sb,int key_type,int sidx,int eidx,int prio,int t,int b,unsigned long * bmap,int count,u16 * save,bool max_alloc,int * alloc_cnt)1924 static int __npc_subbank_non_contig_alloc(struct rvu *rvu,
1925 struct npc_subbank *sb,
1926 int key_type, int sidx,
1927 int eidx, int prio,
1928 int t, int b,
1929 unsigned long *bmap,
1930 int count, u16 *save,
1931 bool max_alloc, int *alloc_cnt)
1932 {
1933 unsigned long index;
1934 int cnt = 0, delta;
1935 int k, sbd;
1936
1937 sbd = npc_priv->subbank_depth;
1938 delta = sidx >= npc_priv->bank_depth ? sbd : 0;
1939
1940 switch (prio) {
1941 /* Find an area of size 'count' from sidx to eidx */
1942 case NPC_MCAM_LOWER_PRIO:
1943 case NPC_MCAM_ANY_PRIO:
1944 index = find_next_zero_bit(bmap, sbd, sidx - b);
1945 if (index >= sbd) {
1946 dev_err(rvu->dev,
1947 "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n",
1948 __func__, count,
1949 bitmap_weight(bmap, sbd),
1950 sb->idx);
1951 break;
1952 }
1953
1954 for (k = index; cnt < count && k <= (eidx - b); k++) {
1955 /* Skip used slots */
1956 if (test_bit(k, bmap))
1957 continue;
1958
1959 save[cnt++] = k + delta;
1960 }
1961 break;
1962
1963 /* Find an area of 'count' from eidx to sidx */
1964 case NPC_MCAM_HIGHER_PRIO:
1965 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
1966 /* Skip used slots */
1967 if (test_bit(k, bmap))
1968 continue;
1969
1970 save[cnt++] = k + delta;
1971 }
1972 break;
1973 }
1974
1975 /* Update allocated 'cnt' to alloc_cnt */
1976 *alloc_cnt = cnt;
1977
1978 /* Successfully allocated requested count slots */
1979 if (cnt == count)
1980 return 0;
1981
1982 /* Allocation successful for cnt < count */
1983 if (max_alloc && cnt > 0)
1984 return 0;
1985
1986 dev_dbg(rvu->dev,
1987 "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n",
1988 __func__, count, sb->idx, cnt, max_alloc);
1989
1990 return -EFAULT;
1991 }
1992
__npc_subbank_sboff_2_off(struct rvu * rvu,struct npc_subbank * sb,int sb_off,unsigned long ** bmap,int * off)1993 static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb,
1994 int sb_off, unsigned long **bmap,
1995 int *off)
1996 {
1997 int sbd;
1998
1999 sbd = npc_priv->subbank_depth;
2000
2001 *off = sb_off & (sbd - 1);
2002 *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map;
2003 }
2004
2005 /* set/clear bitmap */
__npc_subbank_mark_slot(struct rvu * rvu,struct npc_subbank * sb,int sb_off,bool set)2006 static bool __npc_subbank_mark_slot(struct rvu *rvu,
2007 struct npc_subbank *sb,
2008 int sb_off, bool set)
2009 {
2010 unsigned long *bmap;
2011 int off;
2012
2013 /* if sb_off >= subbank.depth, then slots are in
2014 * bank1
2015 */
2016 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
2017
2018 dev_dbg(rvu->dev,
2019 "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n",
2020 __func__, set, sb_off, sb->idx, off);
2021
2022 if (set) {
2023 /* Slot is already used */
2024 if (test_bit(off, bmap))
2025 return false;
2026
2027 sb->free_cnt--;
2028 set_bit(off, bmap);
2029 return true;
2030 }
2031
2032 /* Slot is already free */
2033 if (!test_bit(off, bmap))
2034 return false;
2035
2036 sb->free_cnt++;
2037 clear_bit(off, bmap);
2038 return true;
2039 }
2040
__npc_subbank_mark_free(struct rvu * rvu,struct npc_subbank * sb)2041 static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb)
2042 {
2043 int rc, blkaddr;
2044
2045 sb->flags = NPC_SUBBANK_FLAG_FREE;
2046 sb->key_type = 0;
2047
2048 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth);
2049 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth);
2050
2051 if (!xa_erase(&npc_priv->xa_sb_used, sb->arr_idx)) {
2052 dev_err(rvu->dev,
2053 "%s: Error to delete from xa_sb_used array\n",
2054 __func__);
2055 return -EFAULT;
2056 }
2057
2058 rc = xa_insert(&npc_priv->xa_sb_free, sb->arr_idx,
2059 xa_mk_value(sb->idx), GFP_KERNEL);
2060 if (rc) {
2061 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx,
2062 xa_mk_value(sb->idx), GFP_KERNEL);
2063 if (rc)
2064 dev_err(rvu->dev,
2065 "%s: Failed to roll back sb(%u) arr_idx=%d\n",
2066 __func__, sb->idx, sb->arr_idx);
2067
2068 dev_err(rvu->dev,
2069 "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n",
2070 __func__, sb->idx, sb->arr_idx);
2071 return rc;
2072 }
2073
2074 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2075 rvu_write64(rvu, blkaddr,
2076 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
2077 NPC_MCAM_KEY_X2);
2078
2079 return rc;
2080 }
2081
__npc_subbank_mark_used(struct rvu * rvu,struct npc_subbank * sb,int key_type)2082 static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb,
2083 int key_type)
2084 {
2085 int rc;
2086
2087 sb->flags = NPC_SUBBANK_FLAG_USED;
2088 sb->key_type = key_type;
2089 if (key_type == NPC_MCAM_KEY_X4)
2090 sb->free_cnt = npc_priv->subbank_depth;
2091 else
2092 sb->free_cnt = 2 * npc_priv->subbank_depth;
2093
2094 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth);
2095 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth);
2096
2097 if (!xa_erase(&npc_priv->xa_sb_free, sb->arr_idx)) {
2098 dev_err(rvu->dev,
2099 "%s: Error to delete from xa_sb_free array\n",
2100 __func__);
2101 return -EFAULT;
2102 }
2103
2104 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx,
2105 xa_mk_value(sb->idx), GFP_KERNEL);
2106 if (rc)
2107 dev_err(rvu->dev,
2108 "%s: Error to add to xa_sb_used array\n", __func__);
2109
2110 return rc;
2111 }
2112
__npc_subbank_free(struct rvu * rvu,struct npc_subbank * sb,u16 sb_off)2113 static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb,
2114 u16 sb_off)
2115 {
2116 bool deleted = false;
2117 unsigned long *bmap;
2118 int rc, off;
2119
2120 deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false);
2121 if (!deleted)
2122 goto done;
2123
2124 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
2125
2126 /* Check whether we can mark whole subbank as free */
2127 if (sb->key_type == NPC_MCAM_KEY_X4) {
2128 if (sb->free_cnt < npc_priv->subbank_depth)
2129 goto done;
2130 } else {
2131 if (sb->free_cnt < 2 * npc_priv->subbank_depth)
2132 goto done;
2133 }
2134
2135 /* All slots in subbank are unused. Mark the subbank as free
2136 * and add to free pool
2137 */
2138 rc = __npc_subbank_mark_free(rvu, sb);
2139 if (rc)
2140 dev_err(rvu->dev, "%s: Error to free subbank\n", __func__);
2141
2142 done:
2143 return deleted;
2144 }
2145
2146 static int
npc_subbank_free(struct rvu * rvu,struct npc_subbank * sb,u16 sb_off)2147 npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off)
2148 {
2149 bool deleted;
2150
2151 mutex_lock(&sb->lock);
2152 deleted = __npc_subbank_free(rvu, sb, sb_off);
2153 mutex_unlock(&sb->lock);
2154
2155 return deleted ? 0 : -EFAULT;
2156 }
2157
__npc_subbank_alloc(struct rvu * rvu,struct npc_subbank * sb,int key_type,int ref,int limit,int prio,bool contig,int count,u16 * mcam_idx,int idx_sz,bool max_alloc,int * alloc_cnt)2158 static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
2159 int key_type, int ref, int limit, int prio,
2160 bool contig, int count, u16 *mcam_idx,
2161 int idx_sz, bool max_alloc, int *alloc_cnt)
2162 {
2163 int cnt, t, b, i, blkaddr;
2164 bool new_sub_bank = false;
2165 unsigned long *bmap;
2166 u16 *save = NULL;
2167 int sidx, eidx;
2168 bool diffbank;
2169 int bw, bfree;
2170 int rc = 0;
2171 bool ret;
2172
2173 /* Check if enough space is there to return requested number of
2174 * mcam indexes in case of contiguous allocation
2175 */
2176 if (!max_alloc && count > idx_sz) {
2177 dev_err(rvu->dev,
2178 "%s: Less space, count=%d idx_sz=%d sb_id=%d\n",
2179 __func__, count, idx_sz, sb->idx);
2180 return -ENOSPC;
2181 }
2182
2183 /* Allocation on multiple subbank is not supported by this function.
2184 * it means that ref and limit should be on same subbank.
2185 *
2186 * ref and limit values should be validated w.r.t prio as below.
2187 * say ref = 100, limit = 200,
2188 * if NPC_MCAM_LOWER_PRIO, allocate index 100
2189 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error.
2190 * if NPC_MCAM_ANY_PRIO, allocate index 100
2191 *
2192 * say ref = 200, limit = 100
2193 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error.
2194 * if NPC_MCAM_HIGHER_PRIO, allocate index 200
2195 * if NPC_MCAM_ANY_PRIO, allocate index 100
2196 *
2197 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction
2198 * on "ref" and "limit" values. ie, ref > limit and limit > ref
2199 * are valid cases.
2200 */
2201 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
2202 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
2203 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
2204 __func__, ref, limit);
2205 return -EINVAL;
2206 }
2207
2208 /* x4 indexes are from 0 to bank size as it combines two x2 banks */
2209 if (key_type == NPC_MCAM_KEY_X4 &&
2210 (ref >= npc_priv->bank_depth || limit >= npc_priv->bank_depth)) {
2211 dev_err(rvu->dev,
2212 "%s: Wrong ref_enty(%d) or limit(%d) for x4\n",
2213 __func__, ref, limit);
2214 return -EINVAL;
2215 }
2216
2217 /* This function is called either bank0 or bank1 portion of a subbank.
2218 * so ref and limit should be on same bank.
2219 */
2220 diffbank = !!((ref & npc_priv->bank_depth) ^
2221 (limit & npc_priv->bank_depth));
2222 if (diffbank) {
2223 dev_err(rvu->dev,
2224 "%s: request ref and limit should be from same bank\n",
2225 __func__);
2226 return -EINVAL;
2227 }
2228
2229 sidx = min_t(int, limit, ref);
2230 eidx = max_t(int, limit, ref);
2231
2232 /* Find total number of slots available; both used and free */
2233 cnt = eidx - sidx + 1;
2234 if (contig && cnt < count) {
2235 dev_err(rvu->dev,
2236 "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n",
2237 __func__, ref, limit, count);
2238 return -EINVAL;
2239 }
2240
2241 /* If subbank is free, check if requested number of indexes is less than
2242 * or equal to mcam entries available in the subbank if contig.
2243 */
2244 if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
2245 if (contig && count > npc_priv->subbank_depth) {
2246 dev_err(rvu->dev, "%s: Less number of entries\n",
2247 __func__);
2248 return -ENOSPC;
2249 }
2250
2251 new_sub_bank = true;
2252 goto process;
2253 }
2254
2255 /* Flag should be set for all used subbanks */
2256 WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED),
2257 "Used flag is not set(%#x)\n", sb->flags);
2258
2259 /* If subbank key type does not match with requested key_type,
2260 * return error
2261 */
2262 if (sb->key_type != key_type) {
2263 dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__);
2264 return -EINVAL;
2265 }
2266
2267 process:
2268 /* if ref or limit >= npc_priv->bank_depth, index are in bank1.
2269 * else bank0.
2270 */
2271 if (ref >= npc_priv->bank_depth) {
2272 bmap = sb->b1map;
2273 t = sb->b1t;
2274 b = sb->b1b;
2275 } else {
2276 bmap = sb->b0map;
2277 t = sb->b0t;
2278 b = sb->b0b;
2279 }
2280
2281 /* Calculate free slots */
2282 bw = bitmap_weight(bmap, npc_priv->subbank_depth);
2283 bfree = npc_priv->subbank_depth - bw;
2284
2285 if (!bfree) {
2286 dev_dbg(rvu->dev, "%s: subbank is full\n", __func__);
2287 return -ENOSPC;
2288 }
2289
2290 /* If request is for contiguous , then max we can allocate is
2291 * equal to subbank_depth
2292 */
2293 if (contig && bfree < count) {
2294 dev_dbg(rvu->dev, "%s: no space for entry\n", __func__);
2295 return -ENOSPC;
2296 }
2297
2298 /* 'save' array stores available indexes temporarily before
2299 * marking it as allocated
2300 */
2301 save = kcalloc(count, sizeof(u16), GFP_KERNEL);
2302 if (!save) {
2303 rc = -ENOMEM;
2304 goto err1;
2305 }
2306
2307 if (contig) {
2308 rc = __npc_subbank_contig_alloc(rvu, sb, key_type,
2309 sidx, eidx, prio,
2310 count, t, b,
2311 bmap, save);
2312 /* contiguous allocation success means that
2313 * requested number of free slots got
2314 * allocated
2315 */
2316 if (!rc)
2317 *alloc_cnt = count;
2318
2319 } else {
2320 rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type,
2321 sidx, eidx, prio,
2322 t, b, bmap,
2323 count, save,
2324 max_alloc, alloc_cnt);
2325 }
2326
2327 if (rc)
2328 goto err1;
2329
2330 /* Mark new subbank bank as used */
2331 if (new_sub_bank) {
2332 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2333 if (blkaddr < 0) {
2334 dev_err(rvu->dev,
2335 "%s: NPC block not implemented\n", __func__);
2336 rc = -EFAULT;
2337 goto err1;
2338 }
2339
2340 rc = __npc_subbank_mark_used(rvu, sb, key_type);
2341 if (rc) {
2342 dev_err(rvu->dev,
2343 "%s: Error to mark subbank as used\n",
2344 __func__);
2345 goto err2;
2346 }
2347
2348 /* Configure section type to key_type */
2349 rvu_write64(rvu, blkaddr,
2350 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
2351 key_type);
2352 }
2353
2354 for (i = 0; i < *alloc_cnt; i++) {
2355 rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i],
2356 &mcam_idx[i]);
2357 if (rc) {
2358 dev_err(rvu->dev,
2359 "%s: Error to find mcam idx for %u\n",
2360 __func__, save[i]);
2361 /* TODO: handle err case gracefully */
2362 goto err3;
2363 }
2364
2365 /* Mark all slots as used */
2366 ret = __npc_subbank_mark_slot(rvu, sb, save[i], true);
2367 if (!ret) {
2368 dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n",
2369 __func__, mcam_idx[i]);
2370 rc = -EFAULT;
2371 goto err3;
2372 }
2373 }
2374 kfree(save);
2375 return 0;
2376
2377 err3:
2378 for (int j = 0; j < i; j++)
2379 __npc_subbank_mark_slot(rvu, sb, save[j], false);
2380 err2:
2381 if (new_sub_bank)
2382 __npc_subbank_mark_free(rvu, sb);
2383 err1:
2384 kfree(save);
2385 *alloc_cnt = 0;
2386 return rc;
2387 }
2388
2389 static int
npc_subbank_alloc(struct rvu * rvu,struct npc_subbank * sb,int key_type,int ref,int limit,int prio,bool contig,int count,u16 * mcam_idx,int idx_sz,bool max_alloc,int * alloc_cnt)2390 npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
2391 int key_type, int ref, int limit, int prio,
2392 bool contig, int count, u16 *mcam_idx,
2393 int idx_sz, bool max_alloc, int *alloc_cnt)
2394 {
2395 int rc;
2396
2397 mutex_lock(&sb->lock);
2398 rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio,
2399 contig, count, mcam_idx, idx_sz,
2400 max_alloc, alloc_cnt);
2401 mutex_unlock(&sb->lock);
2402
2403 return rc;
2404 }
2405
2406 static int
npc_del_from_pf_maps(struct rvu * rvu,u16 mcam_idx)2407 npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx)
2408 {
2409 int pcifunc, idx;
2410 void *map;
2411
2412 map = xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx);
2413 if (!map) {
2414 dev_err(rvu->dev,
2415 "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n",
2416 __func__, mcam_idx);
2417 return -EFAULT;
2418 }
2419
2420 pcifunc = xa_to_value(map);
2421 map = xa_load(&npc_priv->xa_pf_map, pcifunc);
2422 if (!map) {
2423 dev_err(rvu->dev,
2424 "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n",
2425 __func__, pcifunc, mcam_idx);
2426 return -ESRCH;
2427 }
2428
2429 idx = xa_to_value(map);
2430
2431 map = xa_erase(&npc_priv->xa_pf2idx_map[idx], mcam_idx);
2432 if (!map) {
2433 dev_err(rvu->dev,
2434 "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n",
2435 __func__, mcam_idx);
2436 return -EFAULT;
2437 }
2438 return 0;
2439 }
2440
2441 static int
npc_add_to_pf_maps(struct rvu * rvu,u16 mcam_idx,int pcifunc)2442 npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc)
2443 {
2444 int rc, idx;
2445 void *map;
2446
2447 dev_dbg(rvu->dev,
2448 "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n",
2449 __func__, mcam_idx, pcifunc);
2450
2451 rc = xa_insert(&npc_priv->xa_idx2pf_map, mcam_idx,
2452 xa_mk_value(pcifunc), GFP_KERNEL);
2453
2454 if (rc) {
2455 map = xa_load(&npc_priv->xa_idx2pf_map, mcam_idx);
2456 dev_err(rvu->dev,
2457 "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n",
2458 __func__, mcam_idx, xa_to_value(map));
2459 return -EFAULT;
2460 }
2461
2462 map = xa_load(&npc_priv->xa_pf_map, pcifunc);
2463 if (!map) {
2464 dev_err(rvu->dev,
2465 "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n",
2466 __func__, pcifunc, mcam_idx);
2467 return -ESRCH;
2468 }
2469
2470 idx = xa_to_value(map);
2471
2472 rc = xa_insert(&npc_priv->xa_pf2idx_map[idx], mcam_idx,
2473 xa_mk_value(pcifunc), GFP_KERNEL);
2474
2475 if (rc) {
2476 map = xa_load(&npc_priv->xa_pf2idx_map[idx], mcam_idx);
2477 xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx);
2478 dev_err(rvu->dev,
2479 "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n",
2480 __func__, mcam_idx, xa_to_value(map), idx);
2481
2482 return -EFAULT;
2483 }
2484
2485 return 0;
2486 }
2487
2488 static bool
npc_subbank_suits(struct npc_subbank * sb,int key_type)2489 npc_subbank_suits(struct npc_subbank *sb, int key_type)
2490 {
2491 mutex_lock(&sb->lock);
2492
2493 if (!sb->key_type) {
2494 mutex_unlock(&sb->lock);
2495 return true;
2496 }
2497
2498 if (sb->key_type == key_type) {
2499 mutex_unlock(&sb->lock);
2500 return true;
2501 }
2502
2503 mutex_unlock(&sb->lock);
2504 return false;
2505 }
2506
2507 #define SB_ALIGN_UP(val) (((val) + npc_priv->subbank_depth) & \
2508 ~((npc_priv->subbank_depth) - 1))
2509 #define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv->subbank_depth)
2510
npc_subbank_iter_down(struct rvu * rvu,int ref,int limit,int * cur_ref,int * cur_limit,bool * start,bool * stop)2511 static void npc_subbank_iter_down(struct rvu *rvu,
2512 int ref, int limit,
2513 int *cur_ref, int *cur_limit,
2514 bool *start, bool *stop)
2515 {
2516 int align;
2517
2518 *stop = false;
2519
2520 /* ALIGN_DOWN the limit to current subbank boundary bottom index */
2521 if (*start) {
2522 *start = false;
2523 *cur_ref = ref;
2524 align = SB_ALIGN_DOWN(ref);
2525 if (align < limit) {
2526 *stop = true;
2527 *cur_limit = limit;
2528 return;
2529 }
2530 *cur_limit = align;
2531 return;
2532 }
2533
2534 *cur_ref = *cur_limit - 1;
2535 align = *cur_ref - npc_priv->subbank_depth + 1;
2536 if (align <= limit) {
2537 *stop = true;
2538 *cur_limit = limit;
2539 return;
2540 }
2541
2542 *cur_limit = align;
2543 }
2544
npc_subbank_iter_up(struct rvu * rvu,int ref,int limit,int * cur_ref,int * cur_limit,bool * start,bool * stop)2545 static void npc_subbank_iter_up(struct rvu *rvu,
2546 int ref, int limit,
2547 int *cur_ref, int *cur_limit,
2548 bool *start, bool *stop)
2549 {
2550 int align;
2551
2552 *stop = false;
2553
2554 /* ALIGN_UP the limit to current subbank boundary top index */
2555 if (*start) {
2556 *start = false;
2557 *cur_ref = ref;
2558
2559 /* Find next lower prio subbank's bottom index */
2560 align = SB_ALIGN_UP(ref);
2561
2562 /* Crosses limit ? */
2563 if (align - 1 > limit) {
2564 *stop = true;
2565 *cur_limit = limit;
2566 return;
2567 }
2568
2569 /* Current subbank's top index */
2570 *cur_limit = align - 1;
2571 return;
2572 }
2573
2574 *cur_ref = *cur_limit + 1;
2575 align = *cur_ref + npc_priv->subbank_depth - 1;
2576
2577 if (align >= limit) {
2578 *stop = true;
2579 *cur_limit = limit;
2580 return;
2581 }
2582
2583 *cur_limit = align;
2584 }
2585
2586 static int
npc_subbank_iter(struct rvu * rvu,int key_type,int ref,int limit,int prio,int * cur_ref,int * cur_limit,bool * start,bool * stop)2587 npc_subbank_iter(struct rvu *rvu, int key_type,
2588 int ref, int limit, int prio,
2589 int *cur_ref, int *cur_limit,
2590 bool *start, bool *stop)
2591 {
2592 if (prio != NPC_MCAM_HIGHER_PRIO)
2593 npc_subbank_iter_up(rvu, ref, limit,
2594 cur_ref, cur_limit,
2595 start, stop);
2596 else
2597 npc_subbank_iter_down(rvu, ref, limit,
2598 cur_ref, cur_limit,
2599 start, stop);
2600
2601 /* limit and ref should < bank_depth for x4 */
2602 if (key_type == NPC_MCAM_KEY_X4) {
2603 if (*cur_ref >= npc_priv->bank_depth)
2604 return -EINVAL;
2605
2606 if (*cur_limit >= npc_priv->bank_depth)
2607 return -EINVAL;
2608 }
2609 /* limit and ref should < 2 * bank_depth, for x2 */
2610 if (*cur_ref >= 2 * npc_priv->bank_depth)
2611 return -EINVAL;
2612
2613 if (*cur_limit >= 2 * npc_priv->bank_depth)
2614 return -EINVAL;
2615
2616 return 0;
2617 }
2618
npc_idx_free(struct rvu * rvu,u16 * mcam_idx,int count,bool maps_del)2619 static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count,
2620 bool maps_del)
2621 {
2622 struct npc_subbank *sb;
2623 u16 vidx, midx;
2624 int sb_off, i;
2625 bool ret;
2626 int rc;
2627
2628 /* Check if we can dealloc indexes properly ? */
2629 for (i = 0; i < count; i++) {
2630 rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]),
2631 &sb, &sb_off);
2632 if (rc) {
2633 dev_err(rvu->dev,
2634 "Failed to free mcam idx=%u\n", mcam_idx[i]);
2635 return rc;
2636 }
2637 }
2638
2639 for (i = 0; i < count; i++) {
2640 if (npc_is_vidx(mcam_idx[i])) {
2641 vidx = mcam_idx[i];
2642 midx = npc_vidx2idx(vidx);
2643 } else {
2644 midx = mcam_idx[i];
2645 vidx = npc_idx2vidx(midx);
2646 }
2647
2648 if (midx >= npc_priv->bank_depth * npc_priv->num_banks) {
2649 dev_err(rvu->dev,
2650 "%s: Invalid mcam_idx=%u cannot be deleted\n",
2651 __func__, mcam_idx[i]);
2652 return -EINVAL;
2653 }
2654
2655 rc = npc_mcam_idx_2_subbank_idx(rvu, midx,
2656 &sb, &sb_off);
2657 if (rc) {
2658 dev_err(rvu->dev,
2659 "%s: Failed to find subbank info for vidx=%u\n",
2660 __func__, vidx);
2661 return rc;
2662 }
2663
2664 ret = npc_subbank_free(rvu, sb, sb_off);
2665 if (ret) {
2666 dev_err(rvu->dev,
2667 "%s: Failed to find subbank info for vidx=%u\n",
2668 __func__, vidx);
2669 return -EINVAL;
2670 }
2671
2672 if (!maps_del)
2673 continue;
2674
2675 rc = npc_del_from_pf_maps(rvu, midx);
2676 if (rc)
2677 return rc;
2678
2679 /* If there is no vidx mapping; continue */
2680 if (vidx == midx)
2681 continue;
2682
2683 rc = npc_vidx_maps_del_entry(rvu, vidx, NULL);
2684 if (rc)
2685 return rc;
2686 }
2687
2688 return 0;
2689 }
2690
npc_multi_subbank_ref_alloc(struct rvu * rvu,int key_type,int ref,int limit,int prio,bool contig,int count,u16 * mcam_idx)2691 static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type,
2692 int ref, int limit, int prio,
2693 bool contig, int count,
2694 u16 *mcam_idx)
2695 {
2696 struct npc_subbank *sb;
2697 unsigned long *bmap;
2698 int sb_off, off, rc;
2699 int cnt = 0;
2700 bool bitset;
2701
2702 if (prio != NPC_MCAM_HIGHER_PRIO) {
2703 while (ref <= limit) {
2704 /* Calculate subbank and subbank index */
2705 rc = npc_mcam_idx_2_subbank_idx(rvu, ref,
2706 &sb, &sb_off);
2707 if (rc)
2708 goto err;
2709
2710 /* If subbank is not suitable for requested key type
2711 * restart search from next subbank
2712 */
2713 if (!npc_subbank_suits(sb, key_type)) {
2714 ref = SB_ALIGN_UP(ref);
2715 if (contig) {
2716 rc = npc_idx_free(rvu, mcam_idx,
2717 cnt, false);
2718 if (rc)
2719 return rc;
2720 cnt = 0;
2721 }
2722 continue;
2723 }
2724
2725 mutex_lock(&sb->lock);
2726
2727 /* If subbank is free; mark it as used */
2728 if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
2729 rc = __npc_subbank_mark_used(rvu, sb,
2730 key_type);
2731 if (rc) {
2732 mutex_unlock(&sb->lock);
2733 dev_err(rvu->dev,
2734 "%s:Error to add to use array\n",
2735 __func__);
2736 goto err;
2737 }
2738 }
2739
2740 /* Find correct bmap */
2741 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
2742
2743 /* if bit is already set, reset 'cnt' */
2744 bitset = test_bit(off, bmap);
2745 if (bitset) {
2746 mutex_unlock(&sb->lock);
2747 if (contig) {
2748 rc = npc_idx_free(rvu, mcam_idx,
2749 cnt, false);
2750 if (rc)
2751 return rc;
2752 cnt = 0;
2753 }
2754
2755 ref++;
2756 continue;
2757 }
2758
2759 set_bit(off, bmap);
2760 sb->free_cnt--;
2761 mcam_idx[cnt++] = ref;
2762 mutex_unlock(&sb->lock);
2763
2764 if (cnt == count)
2765 return 0;
2766 ref++;
2767 }
2768
2769 /* Could not allocate request count slots */
2770 goto err;
2771 }
2772 while (ref >= limit) {
2773 rc = npc_mcam_idx_2_subbank_idx(rvu, ref,
2774 &sb, &sb_off);
2775 if (rc)
2776 goto err;
2777
2778 if (!npc_subbank_suits(sb, key_type)) {
2779 ref = SB_ALIGN_DOWN(ref) - 1;
2780 if (contig) {
2781 rc = npc_idx_free(rvu, mcam_idx, cnt, false);
2782 if (rc)
2783 return rc;
2784
2785 cnt = 0;
2786 }
2787 continue;
2788 }
2789
2790 mutex_lock(&sb->lock);
2791
2792 if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
2793 rc = __npc_subbank_mark_used(rvu, sb, key_type);
2794 if (rc) {
2795 mutex_unlock(&sb->lock);
2796 dev_err(rvu->dev,
2797 "%s:Error to add to use array\n",
2798 __func__);
2799 goto err;
2800 }
2801 }
2802
2803 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
2804 bitset = test_bit(off, bmap);
2805 if (bitset) {
2806 mutex_unlock(&sb->lock);
2807 if (contig) {
2808 rc = npc_idx_free(rvu, mcam_idx, cnt, false);
2809 if (rc)
2810 return rc;
2811
2812 cnt = 0;
2813 }
2814 ref--;
2815 continue;
2816 }
2817
2818 mcam_idx[cnt++] = ref;
2819 sb->free_cnt--;
2820 set_bit(off, bmap);
2821 mutex_unlock(&sb->lock);
2822
2823 if (cnt == count)
2824 return 0;
2825 ref--;
2826 }
2827
2828 err:
2829 rc = npc_idx_free(rvu, mcam_idx, cnt, false);
2830 if (rc)
2831 dev_err(rvu->dev,
2832 "%s: Error happened while freeing cnt=%u indexes\n",
2833 __func__, cnt);
2834
2835 return -ENOSPC;
2836 }
2837
npc_subbank_free_cnt(struct rvu * rvu,struct npc_subbank * sb,int key_type)2838 static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb,
2839 int key_type)
2840 {
2841 int cnt, spd;
2842
2843 spd = npc_priv->subbank_depth;
2844 mutex_lock(&sb->lock);
2845
2846 if (sb->flags & NPC_SUBBANK_FLAG_FREE)
2847 cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd;
2848 else
2849 cnt = sb->free_cnt;
2850
2851 mutex_unlock(&sb->lock);
2852 return cnt;
2853 }
2854
npc_subbank_ref_alloc(struct rvu * rvu,int key_type,int ref,int limit,int prio,bool contig,int count,u16 * mcam_idx)2855 static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type,
2856 int ref, int limit, int prio,
2857 bool contig, int count,
2858 u16 *mcam_idx)
2859 {
2860 struct npc_subbank *sb1, *sb2;
2861 bool max_alloc, start, stop;
2862 int r, l, sb_idx1, sb_idx2;
2863 int tot = 0, rc;
2864 int alloc_cnt;
2865
2866 max_alloc = !contig;
2867
2868 start = true;
2869 stop = false;
2870
2871 /* Loop until we cross the ref/limit boundary */
2872 while (!stop) {
2873 rc = npc_subbank_iter(rvu, key_type, ref, limit, prio,
2874 &r, &l, &start, &stop);
2875
2876 dev_dbg(rvu->dev,
2877 "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n",
2878 __func__, ref, limit, r, l,
2879 start, stop, tot, count, rc);
2880
2881 if (rc)
2882 goto err;
2883
2884 /* Find subbank and subbank index for ref */
2885 rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1,
2886 &sb_idx1);
2887 if (rc)
2888 goto err;
2889
2890 dev_dbg(rvu->dev,
2891 "%s: ref subbank=%d off=%d\n",
2892 __func__, sb1->idx, sb_idx1);
2893
2894 /* Skip subbank if it is not available for the keytype */
2895 if (!npc_subbank_suits(sb1, key_type)) {
2896 dev_dbg(rvu->dev,
2897 "%s: not suitable sb=%d key_type=%d\n",
2898 __func__, sb1->idx, key_type);
2899 continue;
2900 }
2901
2902 /* Find subbank and subbank index for limit */
2903 rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2,
2904 &sb_idx2);
2905 if (rc)
2906 goto err;
2907
2908 dev_dbg(rvu->dev,
2909 "%s: limit subbank=%d off=%d\n",
2910 __func__, sb_idx1, sb_idx2);
2911
2912 /* subbank of ref and limit should be same */
2913 if (sb1 != sb2) {
2914 dev_err(rvu->dev,
2915 "%s: l(%d) and r(%d) are not in same subbank\n",
2916 __func__, r, l);
2917 goto err;
2918 }
2919
2920 if (contig &&
2921 npc_subbank_free_cnt(rvu, sb1, key_type) < count) {
2922 dev_dbg(rvu->dev, "%s: less count =%d\n",
2923 __func__,
2924 npc_subbank_free_cnt(rvu, sb1, key_type));
2925 continue;
2926 }
2927
2928 /* Try in one bank of a subbank */
2929 alloc_cnt = 0;
2930 rc = npc_subbank_alloc(rvu, sb1, key_type,
2931 r, l, prio, contig,
2932 count - tot, mcam_idx + tot,
2933 count - tot, max_alloc,
2934 &alloc_cnt);
2935
2936 tot += alloc_cnt;
2937
2938 dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n",
2939 __func__, tot, alloc_cnt);
2940
2941 if (!rc && count == tot)
2942 return 0;
2943 }
2944 err:
2945 dev_dbg(rvu->dev, "%s: Error to allocate\n",
2946 __func__);
2947
2948 /* non contiguous allocation fails. We need to do clean up */
2949 if (max_alloc) {
2950 rc = npc_idx_free(rvu, mcam_idx, tot, false);
2951 if (rc)
2952 dev_err(rvu->dev,
2953 "%s: failed to free %u indexes\n",
2954 __func__, tot);
2955 }
2956
2957 return -EFAULT;
2958 }
2959
2960 /* Minimize allocation from bottom and top subbanks for noref allocations.
2961 * Default allocations are ref based, and will be allocated from top
2962 * subbanks (least priority subbanks). Since default allocation is at very
2963 * early stage of kernel netdev probes, this subbanks will be moved to
2964 * used subbanks list. This will pave a way for noref allocation from these
2965 * used subbanks. Skip allocation for these top and bottom, and try free
2966 * bank next. If none slot is available, come back and search in these
2967 * subbanks.
2968 */
2969
2970 static int npc_subbank_restricted_idxs[2];
2971 static bool restrict_valid = true;
2972
npc_subbank_restrict_usage(struct rvu * rvu,int index)2973 static bool npc_subbank_restrict_usage(struct rvu *rvu, int index)
2974 {
2975 int i;
2976
2977 if (!restrict_valid)
2978 return false;
2979
2980 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) {
2981 if (index == npc_subbank_restricted_idxs[i])
2982 return true;
2983 }
2984
2985 return false;
2986 }
2987
npc_subbank_noref_alloc(struct rvu * rvu,int key_type,bool contig,int count,u16 * mcam_idx)2988 static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig,
2989 int count, u16 *mcam_idx)
2990 {
2991 struct npc_subbank *sb;
2992 unsigned long index;
2993 int tot = 0, rc;
2994 bool max_alloc;
2995 int alloc_cnt;
2996 int idx, i;
2997 void *val;
2998
2999 max_alloc = !contig;
3000
3001 /* Check used subbanks for free slots */
3002 xa_for_each(&npc_priv->xa_sb_used, index, val) {
3003 idx = xa_to_value(val);
3004
3005 /* Minimize allocation from restricted subbanks
3006 * in noref allocations.
3007 */
3008 if (npc_subbank_restrict_usage(rvu, idx))
3009 continue;
3010
3011 sb = &npc_priv->sb[idx];
3012
3013 /* Skip if not suitable subbank */
3014 if (!npc_subbank_suits(sb, key_type))
3015 continue;
3016
3017 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
3018 continue;
3019
3020 /* try in bank 0. Try passing ref and limit equal to
3021 * subbank boundaries
3022 */
3023 alloc_cnt = 0;
3024 rc = npc_subbank_alloc(rvu, sb, key_type,
3025 sb->b0b, sb->b0t, 0,
3026 contig, count - tot,
3027 mcam_idx + tot,
3028 count - tot,
3029 max_alloc, &alloc_cnt);
3030
3031 /* Non contiguous allocation may allocate less than
3032 * requested 'count'.
3033 */
3034 tot += alloc_cnt;
3035
3036 dev_dbg(rvu->dev,
3037 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3038 __func__, alloc_cnt, sb->idx, tot, count);
3039
3040 /* Successfully allocated */
3041 if (!rc && count == tot)
3042 return 0;
3043
3044 /* x4 entries can be allocated from bank 0 only */
3045 if (key_type == NPC_MCAM_KEY_X4)
3046 continue;
3047
3048 /* try in bank 1 for x2 */
3049 alloc_cnt = 0;
3050 rc = npc_subbank_alloc(rvu, sb, key_type,
3051 sb->b1b, sb->b1t, 0,
3052 contig, count - tot,
3053 mcam_idx + tot,
3054 count - tot, max_alloc,
3055 &alloc_cnt);
3056
3057 tot += alloc_cnt;
3058
3059 dev_dbg(rvu->dev,
3060 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3061 __func__, alloc_cnt, sb->idx, tot, count);
3062
3063 if (!rc && count == tot)
3064 return 0;
3065 }
3066
3067 /* Allocate in free subbanks */
3068 xa_for_each(&npc_priv->xa_sb_free, index, val) {
3069 idx = xa_to_value(val);
3070 sb = &npc_priv->sb[idx];
3071
3072 /* Minimize allocation from restricted subbanks
3073 * in noref allocations.
3074 */
3075 if (npc_subbank_restrict_usage(rvu, idx))
3076 continue;
3077
3078 if (!npc_subbank_suits(sb, key_type))
3079 continue;
3080
3081 /* try in bank 0 */
3082 alloc_cnt = 0;
3083 rc = npc_subbank_alloc(rvu, sb, key_type,
3084 sb->b0b, sb->b0t, 0,
3085 contig, count - tot,
3086 mcam_idx + tot,
3087 count - tot,
3088 max_alloc, &alloc_cnt);
3089
3090 tot += alloc_cnt;
3091
3092 dev_dbg(rvu->dev,
3093 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3094 __func__, alloc_cnt, sb->idx, tot, count);
3095
3096 /* Successfully allocated */
3097 if (!rc && count == tot)
3098 return 0;
3099
3100 /* x4 entries can be allocated from bank 0 only */
3101 if (key_type == NPC_MCAM_KEY_X4)
3102 continue;
3103
3104 /* try in bank 1 for x2 */
3105 alloc_cnt = 0;
3106 rc = npc_subbank_alloc(rvu, sb,
3107 key_type, sb->b1b, sb->b1t, 0,
3108 contig, count - tot,
3109 mcam_idx + tot, count - tot,
3110 max_alloc, &alloc_cnt);
3111
3112 tot += alloc_cnt;
3113
3114 dev_dbg(rvu->dev,
3115 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3116 __func__, alloc_cnt, sb->idx, tot, count);
3117
3118 if (!rc && count == tot)
3119 return 0;
3120 }
3121
3122 /* Allocate from restricted subbanks */
3123 for (i = 0; restrict_valid &&
3124 (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) {
3125 idx = npc_subbank_restricted_idxs[i];
3126 sb = &npc_priv->sb[idx];
3127
3128 /* Skip if not suitable subbank */
3129 if (!npc_subbank_suits(sb, key_type))
3130 continue;
3131
3132 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
3133 continue;
3134
3135 /* try in bank 0. Try passing ref and limit equal to
3136 * subbank boundaries
3137 */
3138 alloc_cnt = 0;
3139 rc = npc_subbank_alloc(rvu, sb, key_type,
3140 sb->b0b, sb->b0t, 0,
3141 contig, count - tot,
3142 mcam_idx + tot,
3143 count - tot,
3144 max_alloc, &alloc_cnt);
3145
3146 /* Non contiguous allocation may allocate less than
3147 * requested 'count'.
3148 */
3149 tot += alloc_cnt;
3150
3151 dev_dbg(rvu->dev,
3152 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3153 __func__, alloc_cnt, sb->idx, tot, count);
3154
3155 /* Successfully allocated */
3156 if (!rc && count == tot)
3157 return 0;
3158
3159 /* x4 entries can be allocated from bank 0 only */
3160 if (key_type == NPC_MCAM_KEY_X4)
3161 continue;
3162
3163 /* try in bank 1 for x2 */
3164 alloc_cnt = 0;
3165 rc = npc_subbank_alloc(rvu, sb, key_type,
3166 sb->b1b, sb->b1t, 0,
3167 contig, count - tot,
3168 mcam_idx + tot,
3169 count - tot, max_alloc,
3170 &alloc_cnt);
3171
3172 tot += alloc_cnt;
3173
3174 dev_dbg(rvu->dev,
3175 "%s: Allocated %d from subbank %d, tot=%d count=%d\n",
3176 __func__, alloc_cnt, sb->idx, tot, count);
3177
3178 if (!rc && count == tot)
3179 return 0;
3180 }
3181
3182 /* non contiguous allocation fails. We need to do clean up */
3183 if (max_alloc)
3184 npc_idx_free(rvu, mcam_idx, tot, false);
3185
3186 dev_dbg(rvu->dev, "%s: non-contig allocation fails\n",
3187 __func__);
3188
3189 return -EFAULT;
3190 }
3191
npc_cn20k_idx_free(struct rvu * rvu,u16 * mcam_idx,int count)3192 int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count)
3193 {
3194 return npc_idx_free(rvu, mcam_idx, count, true);
3195 }
3196
npc_cn20k_ref_idx_alloc(struct rvu * rvu,int pcifunc,int key_type,int prio,u16 * mcam_idx,int ref,int limit,bool contig,int count,bool virt)3197 int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type,
3198 int prio, u16 *mcam_idx, int ref, int limit,
3199 bool contig, int count, bool virt)
3200 {
3201 bool defrag_candidate = false;
3202 int i, eidx, rc, bd;
3203 bool ref_valid;
3204 u16 vidx;
3205
3206 bd = npc_priv->bank_depth;
3207
3208 /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1
3209 * Here user wants to allocate 0th entry
3210 */
3211 if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO &&
3212 count == 1) {
3213 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
3214 prio, contig, count, mcam_idx);
3215
3216 if (rc)
3217 return rc;
3218 goto add2map;
3219 }
3220
3221 ref_valid = !!(limit || ref);
3222 defrag_candidate = !ref_valid && !contig && virt;
3223 if (!ref_valid) {
3224 if (contig && count > npc_priv->subbank_depth)
3225 goto try_noref_multi_subbank;
3226
3227 rc = npc_subbank_noref_alloc(rvu, key_type, contig,
3228 count, mcam_idx);
3229 if (!rc)
3230 goto add2map;
3231
3232 try_noref_multi_subbank:
3233 eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1;
3234
3235 if (prio == NPC_MCAM_HIGHER_PRIO)
3236 rc = npc_multi_subbank_ref_alloc(rvu, key_type,
3237 eidx, 0,
3238 NPC_MCAM_HIGHER_PRIO,
3239 contig, count,
3240 mcam_idx);
3241 else
3242 rc = npc_multi_subbank_ref_alloc(rvu, key_type,
3243 0, eidx,
3244 NPC_MCAM_LOWER_PRIO,
3245 contig, count,
3246 mcam_idx);
3247
3248 if (!rc)
3249 goto add2map;
3250
3251 return rc;
3252 }
3253
3254 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
3255 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
3256 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
3257 __func__, ref, limit);
3258 return -EINVAL;
3259 }
3260
3261 if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) ||
3262 (key_type == NPC_MCAM_KEY_X2 &&
3263 (ref >= 2 * bd || limit >= 2 * bd))) {
3264 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
3265 __func__, ref, limit);
3266 return -EINVAL;
3267 }
3268
3269 if (contig && count > npc_priv->subbank_depth)
3270 goto try_ref_multi_subbank;
3271
3272 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
3273 prio, contig, count, mcam_idx);
3274 if (!rc)
3275 goto add2map;
3276
3277 try_ref_multi_subbank:
3278 rc = npc_multi_subbank_ref_alloc(rvu, key_type,
3279 ref, limit, prio,
3280 contig, count, mcam_idx);
3281 if (!rc)
3282 goto add2map;
3283
3284 return rc;
3285
3286 add2map:
3287 for (i = 0; i < count; i++) {
3288 rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc);
3289 if (rc)
3290 goto err;
3291
3292 if (!defrag_candidate)
3293 continue;
3294
3295 rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx);
3296 if (rc) {
3297 npc_del_from_pf_maps(rvu, mcam_idx[i]);
3298 goto err;
3299 }
3300
3301 /* Return vidx to caller */
3302 mcam_idx[i] = vidx;
3303 }
3304
3305 return 0;
3306 err:
3307 for (int j = 0; j < i; j++) {
3308 npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j]));
3309
3310 if (!defrag_candidate)
3311 continue;
3312
3313 npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL);
3314 }
3315
3316 return rc;
3317
3318 }
3319
npc_cn20k_subbank_calc_free(struct rvu * rvu,int * x2_free,int * x4_free,int * sb_free)3320 void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free,
3321 int *x4_free, int *sb_free)
3322 {
3323 struct npc_subbank *sb;
3324 int i;
3325
3326 /* Reset all stats to zero */
3327 *x2_free = 0;
3328 *x4_free = 0;
3329 *sb_free = 0;
3330
3331 for (i = 0; i < npc_priv->num_subbanks; i++) {
3332 sb = &npc_priv->sb[i];
3333 mutex_lock(&sb->lock);
3334
3335 /* Count number of free subbanks */
3336 if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
3337 (*sb_free)++;
3338 goto next;
3339 }
3340
3341 /* Sumup x4 free count */
3342 if (sb->key_type == NPC_MCAM_KEY_X4) {
3343 (*x4_free) += sb->free_cnt;
3344 goto next;
3345 }
3346
3347 /* Sumup x2 free counts */
3348 (*x2_free) += sb->free_cnt;
3349 next:
3350 mutex_unlock(&sb->lock);
3351 }
3352 }
3353
3354 int
rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu * rvu,struct msg_req * req,struct npc_cn20k_get_fcnt_rsp * rsp)3355 rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu,
3356 struct msg_req *req,
3357 struct npc_cn20k_get_fcnt_rsp *rsp)
3358 {
3359 npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2,
3360 &rsp->free_x4, &rsp->free_subbanks);
3361 return 0;
3362 }
3363
3364 int
rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu * rvu,struct msg_req * req,struct npc_cn20k_get_kex_cfg_rsp * rsp)3365 rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu,
3366 struct msg_req *req,
3367 struct npc_cn20k_get_kex_cfg_rsp *rsp)
3368 {
3369 int extr, lt;
3370
3371 rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX);
3372 rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX);
3373
3374 /* Get EXTRACTOR LID */
3375 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
3376 rsp->intf_extr_lid[NIX_INTF_RX][extr] =
3377 CN20K_GET_EXTR_LID(NIX_INTF_RX, extr);
3378 rsp->intf_extr_lid[NIX_INTF_TX][extr] =
3379 CN20K_GET_EXTR_LID(NIX_INTF_TX, extr);
3380 }
3381
3382 /* Get EXTRACTOR LTYPE */
3383 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
3384 for (lt = 0; lt < NPC_MAX_LT; lt++) {
3385 rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] =
3386 CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt);
3387 rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] =
3388 CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt);
3389 }
3390 }
3391
3392 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN);
3393 return 0;
3394 }
3395
3396 static u32 *subbank_srch_order;
3397
npc_populate_restricted_idxs(int num_subbanks)3398 static void npc_populate_restricted_idxs(int num_subbanks)
3399 {
3400 npc_subbank_restricted_idxs[0] = num_subbanks - 1;
3401 npc_subbank_restricted_idxs[1] = 0;
3402 }
3403
npc_create_srch_order(int cnt)3404 static int npc_create_srch_order(int cnt)
3405 {
3406 int val = 0;
3407
3408 subbank_srch_order = kcalloc(cnt, sizeof(u32),
3409 GFP_KERNEL);
3410 if (!subbank_srch_order)
3411 return -ENOMEM;
3412
3413 /* cnt(subbank depth) is always a power of 2. There is a check in
3414 * npc_priv_init() to check the same.
3415 */
3416 for (int i = 0; i < cnt; i += 2) {
3417 subbank_srch_order[i] = cnt / 2 - val - 1;
3418 subbank_srch_order[i + 1] = cnt / 2 + 1 + val;
3419 val++;
3420 }
3421
3422 subbank_srch_order[cnt - 1] = cnt / 2;
3423 return 0;
3424 }
3425
npc_subbanks_srch_order_init(struct rvu * rvu)3426 static int npc_subbanks_srch_order_init(struct rvu *rvu)
3427 {
3428 struct npc_subbank *sb;
3429 int sb_idx;
3430 int i, j;
3431 int rc;
3432
3433 for (i = 0; i < npc_priv->num_subbanks; i++) {
3434 sb_idx = subbank_srch_order[i];
3435 sb = &npc_priv->sb[sb_idx];
3436 sb->arr_idx = i;
3437
3438 dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n",
3439 __func__, sb->idx, sb->arr_idx);
3440
3441 rc = xa_err(xa_store(&npc_priv->xa_sb_free, sb->arr_idx,
3442 xa_mk_value(sb->idx), GFP_KERNEL));
3443 if (rc) {
3444 dev_err(rvu->dev,
3445 "%s: xa_store(xa_sb_free) failed at slot %d (sb=%d): %d\n",
3446 __func__, i, sb_idx, rc);
3447 for (j = 0; j < i; j++)
3448 xa_erase(&npc_priv->xa_sb_free, j);
3449 return rc;
3450 }
3451 }
3452
3453 return 0;
3454 }
3455
npc_subbank_init(struct rvu * rvu,struct npc_subbank * sb,int idx)3456 static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx)
3457 {
3458 mutex_init(&sb->lock);
3459
3460 sb->b0b = idx * npc_priv->subbank_depth;
3461 sb->b0t = sb->b0b + npc_priv->subbank_depth - 1;
3462
3463 sb->b1b = npc_priv->bank_depth + idx * npc_priv->subbank_depth;
3464 sb->b1t = sb->b1b + npc_priv->subbank_depth - 1;
3465
3466 sb->flags = NPC_SUBBANK_FLAG_FREE;
3467 sb->idx = idx;
3468 }
3469
npc_pcifunc_map_create(struct rvu * rvu)3470 static int npc_pcifunc_map_create(struct rvu *rvu)
3471 {
3472 int pf, vf, numvfs;
3473 int cnt = 0;
3474 u16 pcifunc;
3475 u64 cfg;
3476
3477 for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
3478 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
3479 numvfs = (cfg >> 12) & 0xFF;
3480
3481 /* Skip not enabled PFs */
3482 if (!(cfg & BIT_ULL(20)))
3483 goto chk_vfs;
3484
3485 /* If Admin function, check on VFs */
3486 if (cfg & BIT_ULL(21))
3487 goto chk_vfs;
3488
3489 pcifunc = pf << 9;
3490
3491 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc,
3492 xa_mk_value(cnt), GFP_KERNEL);
3493
3494 cnt++;
3495
3496 chk_vfs:
3497 for (vf = 0; vf < numvfs; vf++) {
3498 pcifunc = (pf << 9) | (vf + 1);
3499
3500 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc,
3501 xa_mk_value(cnt), GFP_KERNEL);
3502 cnt++;
3503 }
3504 }
3505
3506 return cnt;
3507 }
3508
3509 struct npc_defrag_node {
3510 u8 idx;
3511 u8 key_type;
3512 bool valid;
3513 bool refs;
3514 u16 free_cnt;
3515 u16 vidx_cnt;
3516 u16 *vidx;
3517 struct list_head list;
3518 };
3519
npc_defrag_skip_restricted_sb(int sb_id)3520 static bool npc_defrag_skip_restricted_sb(int sb_id)
3521 {
3522 int i;
3523
3524 if (!restrict_valid)
3525 return false;
3526
3527 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++)
3528 if (sb_id == npc_subbank_restricted_idxs[i])
3529 return true;
3530 return false;
3531 }
3532
3533 /* Find subbank with minimum number of virtual indexes */
npc_subbank_min_vidx(struct list_head * lh)3534 static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh)
3535 {
3536 struct npc_defrag_node *node, *tnode = NULL;
3537 int min = INT_MAX;
3538
3539 list_for_each_entry(node, lh, list) {
3540 if (!node->valid)
3541 continue;
3542
3543 /* if subbank has ref allocated mcam indexes, that subbank
3544 * is not a good candidate to move out indexes.
3545 */
3546 if (node->refs)
3547 continue;
3548
3549 if (min > node->vidx_cnt) {
3550 min = node->vidx_cnt;
3551 tnode = node;
3552 }
3553 }
3554
3555 return tnode;
3556 }
3557
3558 /* Find subbank with maximum number of free spaces */
npc_subbank_max_free(struct list_head * lh)3559 static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh)
3560 {
3561 struct npc_defrag_node *node, *tnode = NULL;
3562 int max = INT_MIN;
3563
3564 list_for_each_entry(node, lh, list) {
3565 if (!node->valid)
3566 continue;
3567
3568 if (max < node->free_cnt) {
3569 max = node->free_cnt;
3570 tnode = node;
3571 }
3572 }
3573
3574 return tnode;
3575 }
3576
npc_defrag_alloc_free_slots(struct rvu * rvu,struct npc_defrag_node * f,int cnt,u16 * save)3577 static int npc_defrag_alloc_free_slots(struct rvu *rvu,
3578 struct npc_defrag_node *f,
3579 int cnt, u16 *save)
3580 {
3581 int alloc_cnt1, alloc_cnt2;
3582 struct npc_subbank *sb;
3583 int rc, sb_off, i, err;
3584 bool deleted;
3585
3586 sb = &npc_priv->sb[f->idx];
3587
3588 alloc_cnt1 = 0;
3589 alloc_cnt2 = 0;
3590
3591 rc = __npc_subbank_alloc(rvu, sb,
3592 f->key_type, sb->b0b,
3593 sb->b0t,
3594 NPC_MCAM_LOWER_PRIO,
3595 false, cnt, save, cnt, true,
3596 &alloc_cnt1);
3597
3598 /* X4 entries only occupy bank 0 (b0b..b0t); see npc_subbank_idx_2_mcam_idx().
3599 * X2 uses both halves of the subbank, so spill into bank 1 if needed.
3600 */
3601 if (alloc_cnt1 < cnt && f->key_type == NPC_MCAM_KEY_X2) {
3602 rc = __npc_subbank_alloc(rvu, sb,
3603 f->key_type, sb->b1b,
3604 sb->b1t,
3605 NPC_MCAM_LOWER_PRIO,
3606 false, cnt - alloc_cnt1,
3607 save + alloc_cnt1,
3608 cnt - alloc_cnt1,
3609 true, &alloc_cnt2);
3610 }
3611
3612 if (alloc_cnt1 + alloc_cnt2 != cnt) {
3613 dev_err(rvu->dev,
3614 "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n",
3615 __func__, cnt, alloc_cnt1, alloc_cnt2);
3616 rc = -ENOSPC;
3617 goto fail_free_alloc;
3618 }
3619
3620 return 0;
3621
3622 fail_free_alloc:
3623 for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) {
3624 err = npc_mcam_idx_2_subbank_idx(rvu, save[i],
3625 &sb, &sb_off);
3626 if (err) {
3627 dev_err(rvu->dev,
3628 "%s: Error to find subbank for mcam idx=%u\n",
3629 __func__, save[i]);
3630 break;
3631 }
3632
3633 deleted = __npc_subbank_free(rvu, sb, sb_off);
3634 if (!deleted) {
3635 dev_err(rvu->dev,
3636 "%s: Error to free mcam idx=%u\n",
3637 __func__, save[i]);
3638 break;
3639 }
3640 }
3641
3642 return rc;
3643 }
3644
npc_defrag_add_2_show_list(struct rvu * rvu,u16 old_midx,u16 new_midx,u16 vidx)3645 static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx,
3646 u16 new_midx, u16 vidx)
3647 {
3648 struct npc_defrag_show_node *node;
3649
3650 node = kcalloc(1, sizeof(*node), GFP_KERNEL);
3651 if (!node)
3652 return -ENOMEM;
3653
3654 node->old_midx = old_midx;
3655 node->new_midx = new_midx;
3656 node->vidx = vidx;
3657 INIT_LIST_HEAD(&node->list);
3658
3659 mutex_lock(&npc_priv->lock);
3660 list_add_tail(&node->list, &npc_priv->defrag_lh);
3661 mutex_unlock(&npc_priv->lock);
3662
3663 return 0;
3664 }
3665
3666 static
npc_defrag_move_vdx_to_free(struct rvu * rvu,struct npc_defrag_node * f,struct npc_defrag_node * v,int cnt,u16 * save)3667 int npc_defrag_move_vdx_to_free(struct rvu *rvu,
3668 struct npc_defrag_node *f,
3669 struct npc_defrag_node *v,
3670 int cnt, u16 *save)
3671 {
3672 u16 new_midx, old_midx, vidx, target_pf;
3673 struct npc_mcam *mcam = &rvu->hw->mcam;
3674 struct rvu_npc_mcam_rule *rule, *tmp;
3675 int i, vidx_cnt, rc, sb_off;
3676 struct npc_subbank *sb;
3677 bool deleted;
3678 u16 pcifunc;
3679 int blkaddr;
3680 void *map;
3681 u8 bank;
3682 u16 midx;
3683 u64 stats;
3684
3685 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3686
3687 vidx_cnt = v->vidx_cnt;
3688 for (i = 0; i < cnt; i++) {
3689 vidx = v->vidx[vidx_cnt - i - 1];
3690 old_midx = npc_vidx2idx(vidx);
3691 new_midx = save[cnt - i - 1];
3692
3693 dev_dbg(rvu->dev,
3694 "%s: Moving %u ---> %u (vidx=%u)\n",
3695 __func__,
3696 old_midx, new_midx, vidx);
3697
3698 rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx);
3699 if (rc)
3700 dev_err(rvu->dev,
3701 "%s: Error happened to add to show list vidx=%u\n",
3702 __func__, vidx);
3703
3704 /* Modify vidx to point to new mcam idx */
3705 rc = npc_vidx_maps_modify(rvu, vidx, new_midx);
3706 if (rc)
3707 return rc;
3708
3709 midx = old_midx % mcam->banksize;
3710 bank = old_midx / mcam->banksize;
3711 stats = rvu_read64(rvu, blkaddr,
3712 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx,
3713 bank));
3714
3715 /* If bug happened during copy/enable mcam, then there is a bug in allocation
3716 * algorithm itself. There is no point in rewinding and returning, as it
3717 * will face further issue. Return error after printing error
3718 */
3719 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) {
3720 dev_err(rvu->dev,
3721 "%s: Error happened while disabling old_mid=%u\n",
3722 __func__, old_midx);
3723 return -EFAULT;
3724 }
3725
3726 if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) {
3727 dev_err(rvu->dev,
3728 "%s: Error happened while copying old_midx=%u new_midx=%u\n",
3729 __func__, old_midx, new_midx);
3730 return -EFAULT;
3731 }
3732
3733 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) {
3734 dev_err(rvu->dev,
3735 "%s: Error happened while enabling new_mid=%u\n",
3736 __func__, new_midx);
3737 return -EFAULT;
3738 }
3739
3740 midx = new_midx % mcam->banksize;
3741 bank = new_midx / mcam->banksize;
3742 rvu_write64(rvu, blkaddr,
3743 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank),
3744 stats);
3745
3746 /* Free the old mcam idx */
3747 rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx,
3748 &sb, &sb_off);
3749 if (rc) {
3750 dev_err(rvu->dev,
3751 "%s: Unable to calculate subbank off for mcamidx=%u\n",
3752 __func__, old_midx);
3753 return rc;
3754 }
3755
3756 deleted = __npc_subbank_free(rvu, sb, sb_off);
3757 if (!deleted) {
3758 dev_err(rvu->dev,
3759 "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n",
3760 __func__, old_midx, sb->idx, sb_off);
3761 return -EFAULT;
3762 }
3763
3764 /* save pcifunc */
3765 map = xa_load(&npc_priv->xa_idx2pf_map, old_midx);
3766 pcifunc = xa_to_value(map);
3767
3768 /* delete from pf maps */
3769 rc = npc_del_from_pf_maps(rvu, old_midx);
3770 if (rc) {
3771 dev_err(rvu->dev,
3772 "%s: Failed to delete pf maps for mcamidx=%u\n",
3773 __func__, old_midx);
3774 return rc;
3775 }
3776
3777 /* add new mcam_idx to pf map */
3778 rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc);
3779 if (rc) {
3780 dev_err(rvu->dev,
3781 "%s: Failed to add pf maps for mcamidx=%u\n",
3782 __func__, new_midx);
3783 return rc;
3784 }
3785
3786 /* Remove from mcam maps */
3787 mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP;
3788 mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP;
3789 npc_mcam_clear_bit(mcam, old_midx);
3790
3791 mcam->entry2pfvf_map[new_midx] = pcifunc;
3792 /* Counter is not preserved */
3793 mcam->entry2cntr_map[new_midx] = new_midx;
3794 target_pf = mcam->entry2target_pffunc[old_midx];
3795 mcam->entry2target_pffunc[new_midx] = target_pf;
3796 mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP;
3797
3798 npc_mcam_set_bit(mcam, new_midx);
3799
3800 /* Note: list order is not functionally required for mcam_rules */
3801 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
3802 if (rule->entry != old_midx)
3803 continue;
3804
3805 rule->entry = new_midx;
3806 break;
3807 }
3808
3809 /* Mark as invalid */
3810 v->vidx[vidx_cnt - i - 1] = -1;
3811 save[cnt - i - 1] = -1;
3812
3813 f->free_cnt--;
3814 v->vidx_cnt--;
3815 }
3816
3817 return 0;
3818 }
3819
npc_defrag_process(struct rvu * rvu,struct list_head * lh)3820 static int npc_defrag_process(struct rvu *rvu, struct list_head *lh)
3821 {
3822 struct npc_defrag_node *v = NULL;
3823 struct npc_defrag_node *f = NULL;
3824 int rc = 0, cnt;
3825 u16 *save;
3826
3827 while (1) {
3828 /* Find subbank with minimum vidx */
3829 if (!v) {
3830 v = npc_subbank_min_vidx(lh);
3831 if (!v)
3832 break;
3833 }
3834
3835 /* Find subbank with maximum free slots */
3836 if (!f) {
3837 f = npc_subbank_max_free(lh);
3838 if (!f)
3839 break;
3840 }
3841
3842 if (!v->vidx_cnt) {
3843 list_del_init(&v->list);
3844 v = NULL;
3845 continue;
3846 }
3847
3848 if (!f->free_cnt) {
3849 list_del_init(&f->list);
3850 f = NULL;
3851 continue;
3852 }
3853
3854 /* If both subbanks are same, choose vidx and
3855 * search for free list again
3856 */
3857 if (f == v) {
3858 list_del_init(&f->list);
3859 f = NULL;
3860 continue;
3861 }
3862
3863 /* Calculate minimum free slots needs to be allocated */
3864 cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt :
3865 f->free_cnt;
3866
3867 dev_dbg(rvu->dev,
3868 "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n",
3869 __func__, cnt, f->free_cnt, f->idx,
3870 v->vidx_cnt, v->idx);
3871
3872 /* Allocate an array to store newly allocated
3873 * free slots (mcam indexes)
3874 */
3875 save = kcalloc(cnt, sizeof(*save), GFP_KERNEL);
3876 if (!save) {
3877 rc = -ENOMEM;
3878 goto err;
3879 }
3880
3881 /* Alloc free slots for existing vidx */
3882 rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save);
3883 if (rc) {
3884 kfree(save);
3885 goto err;
3886 }
3887
3888 /* Move vidx to free slots; update pf_map and vidx maps,
3889 * and free existing vidx mcam slots
3890 */
3891 rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save);
3892 if (rc) {
3893 kfree(save);
3894 goto err;
3895 }
3896
3897 kfree(save);
3898
3899 if (!f->free_cnt) {
3900 list_del_init(&f->list);
3901 f = NULL;
3902 }
3903
3904 if (!v->vidx_cnt) {
3905 list_del_init(&v->list);
3906 v = NULL;
3907 }
3908 }
3909
3910 err:
3911 /* Whole defragmentation process is done within locks. if there
3912 * is an error, it would be hard to roll back as index remove/add
3913 * can fail again if it failed before. This would mean that there
3914 * is bug in the index management algorithm.
3915 * Return from here than rolling back.
3916 */
3917 return rc;
3918 }
3919
npc_defrag_list_clear(void)3920 static void npc_defrag_list_clear(void)
3921 {
3922 struct npc_defrag_show_node *node, *next;
3923
3924 mutex_lock(&npc_priv->lock);
3925 list_for_each_entry_safe(node, next, &npc_priv->defrag_lh, list) {
3926 list_del_init(&node->list);
3927 kfree(node);
3928 }
3929
3930 mutex_unlock(&npc_priv->lock);
3931 }
3932
npc_lock_all_subbank(void)3933 static void npc_lock_all_subbank(void)
3934 {
3935 int i;
3936
3937 for (i = 0; i < npc_priv->num_subbanks; i++)
3938 mutex_lock(&npc_priv->sb[i].lock);
3939 }
3940
npc_unlock_all_subbank(void)3941 static void npc_unlock_all_subbank(void)
3942 {
3943 int i;
3944
3945 for (i = npc_priv->num_subbanks - 1; i >= 0; i--)
3946 mutex_unlock(&npc_priv->sb[i].lock);
3947 }
3948
npc_cn20k_search_order_set(struct rvu * rvu,u64 narr[MAX_NUM_SUB_BANKS],int cnt)3949 int npc_cn20k_search_order_set(struct rvu *rvu,
3950 u64 narr[MAX_NUM_SUB_BANKS], int cnt)
3951 {
3952 struct npc_mcam *mcam = &rvu->hw->mcam;
3953 int rsrc[2][MAX_NUM_SUB_BANKS] = { };
3954 u8 save[MAX_NUM_SUB_BANKS] = { };
3955 struct npc_subbank *sb;
3956 struct xarray *xa;
3957 int prio, rc, err;
3958 int sb_idx;
3959 enum {
3960 FREE = 0,
3961 USED = 1,
3962 };
3963
3964 if (cnt != npc_priv->num_subbanks) {
3965 dev_err(rvu->dev, "Number of entries(%u) != %u\n",
3966 cnt, npc_priv->num_subbanks);
3967 return -EINVAL;
3968 }
3969
3970 mutex_lock(&mcam->lock);
3971 npc_lock_all_subbank();
3972
3973 for (sb_idx = 0; sb_idx < cnt; sb_idx++) {
3974 sb = &npc_priv->sb[sb_idx];
3975 save[sb->idx] = sb->arr_idx;
3976 }
3977
3978 for (prio = 0; prio < cnt; prio++) {
3979 sb_idx = narr[prio];
3980 sb = &npc_priv->sb[sb_idx];
3981
3982 if (sb->flags & NPC_SUBBANK_FLAG_USED)
3983 xa = &npc_priv->xa_sb_used;
3984 else
3985 xa = &npc_priv->xa_sb_free;
3986
3987 rc = xa_err(xa_store(xa, prio,
3988 xa_mk_value(sb_idx), GFP_KERNEL));
3989 if (rc) {
3990 dev_err(rvu->dev,
3991 "Setting arr_idx=%d for sb=%d failed\n",
3992 sb->arr_idx, sb_idx);
3993 goto fail;
3994 }
3995
3996 if (sb->flags & NPC_SUBBANK_FLAG_USED) {
3997 rsrc[USED][sb->arr_idx] -= 1;
3998 rsrc[USED][prio] += 1;
3999 } else {
4000 rsrc[FREE][sb->arr_idx] -= 1;
4001 rsrc[FREE][prio] += 1;
4002 }
4003
4004 sb->arr_idx = prio;
4005 }
4006
4007 for (prio = 0; prio < cnt; prio++) {
4008 if (rsrc[FREE][prio] == -1)
4009 xa_erase(&npc_priv->xa_sb_free, prio);
4010
4011 if (rsrc[USED][prio] == -1)
4012 xa_erase(&npc_priv->xa_sb_used, prio);
4013 }
4014
4015 for (int i = 0; i < cnt; i++)
4016 subbank_srch_order[i] = (u32)narr[i];
4017
4018 restrict_valid = false;
4019
4020 npc_unlock_all_subbank();
4021 mutex_unlock(&mcam->lock);
4022
4023 return 0;
4024
4025 fail:
4026 for (prio = 0; prio < cnt; prio++) {
4027 if (rsrc[FREE][prio] == 1)
4028 xa_erase(&npc_priv->xa_sb_free, prio);
4029
4030 if (rsrc[USED][prio] == 1)
4031 xa_erase(&npc_priv->xa_sb_used, prio);
4032 }
4033
4034 for (sb_idx = 0; sb_idx < cnt; sb_idx++) {
4035 sb = &npc_priv->sb[sb_idx];
4036 sb->arr_idx = save[sb_idx];
4037
4038 if (sb->flags & NPC_SUBBANK_FLAG_USED)
4039 xa = &npc_priv->xa_sb_used;
4040 else
4041 xa = &npc_priv->xa_sb_free;
4042
4043 /* Since the entry already exists, xa_store() replaces
4044 * the value without a kmalloc(), making failure highly unlikely.
4045 */
4046 err = xa_err(xa_store(xa, sb->arr_idx,
4047 xa_mk_value(sb->idx), GFP_KERNEL));
4048 WARN(!!err, "Failed to rollback sb=%u idx=%u\n",
4049 sb->idx, sb->arr_idx);
4050 }
4051
4052 npc_unlock_all_subbank();
4053 mutex_unlock(&mcam->lock);
4054
4055 return rc;
4056 }
4057
npc_cn20k_search_order_get(bool * restricted_order,u32 * sz)4058 const u32 *npc_cn20k_search_order_get(bool *restricted_order, u32 *sz)
4059 {
4060 *restricted_order = restrict_valid;
4061 *sz = npc_priv->num_subbanks;
4062 return subbank_srch_order;
4063 }
4064
4065 /* Only non-ref non-contigous mcam indexes
4066 * are picked for defrag process
4067 */
npc_cn20k_defrag(struct rvu * rvu)4068 int npc_cn20k_defrag(struct rvu *rvu)
4069 {
4070 struct npc_mcam *mcam = &rvu->hw->mcam;
4071 struct npc_defrag_node *node, *tnode;
4072 struct list_head x4lh, x2lh, *lh;
4073 int rc = 0, i, sb_off, tot;
4074 struct npc_subbank *sb;
4075 unsigned long index;
4076 void *map;
4077 u16 midx;
4078
4079 /* Free previous show list */
4080 npc_defrag_list_clear();
4081
4082 INIT_LIST_HEAD(&x4lh);
4083 INIT_LIST_HEAD(&x2lh);
4084
4085 node = kcalloc(npc_priv->num_subbanks, sizeof(*node), GFP_KERNEL);
4086 if (!node)
4087 return -ENOMEM;
4088
4089 /* Lock mcam */
4090 mutex_lock(&mcam->lock);
4091 npc_lock_all_subbank();
4092
4093 /* Fill in node with subbank properties */
4094 for (i = 0; i < npc_priv->num_subbanks; i++) {
4095 sb = &npc_priv->sb[i];
4096
4097 node[i].idx = i;
4098 node[i].key_type = sb->key_type;
4099 node[i].free_cnt = sb->free_cnt;
4100 node[i].vidx = kcalloc(npc_priv->subbank_depth * 2,
4101 sizeof(*node[i].vidx),
4102 GFP_KERNEL);
4103 if (!node[i].vidx) {
4104 rc = -ENOMEM;
4105 goto free_vidx;
4106 }
4107
4108 /* If subbank is empty, dont include it in defrag
4109 * process
4110 */
4111 if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
4112 node[i].valid = false;
4113 continue;
4114 }
4115
4116 if (npc_defrag_skip_restricted_sb(i)) {
4117 node[i].valid = false;
4118 continue;
4119 }
4120
4121 node[i].valid = true;
4122 INIT_LIST_HEAD(&node[i].list);
4123
4124 /* Add node to x2 or x4 list */
4125 lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh;
4126 list_add_tail(&node[i].list, lh);
4127 }
4128
4129 /* Filling vidx[] array with all vidx in that subbank */
4130 xa_for_each_start(&npc_priv->xa_vidx2idx_map, index, map,
4131 npc_priv->bank_depth * 2) {
4132 midx = xa_to_value(map);
4133 rc = npc_mcam_idx_2_subbank_idx(rvu, midx,
4134 &sb, &sb_off);
4135 if (rc) {
4136 dev_err(rvu->dev,
4137 "%s: Error to get mcam_idx for vidx=%lu\n",
4138 __func__, index);
4139 goto free_vidx;
4140 }
4141
4142 tnode = &node[sb->idx];
4143 tnode->vidx[tnode->vidx_cnt] = index;
4144 tnode->vidx_cnt++;
4145 }
4146
4147 /* Mark all subbank which has ref allocation */
4148 for (i = 0; i < npc_priv->num_subbanks; i++) {
4149 tnode = &node[i];
4150
4151 if (!tnode->valid)
4152 continue;
4153
4154 tot = (tnode->key_type == NPC_MCAM_KEY_X2) ?
4155 npc_priv->subbank_depth * 2 : npc_priv->subbank_depth;
4156
4157 if (node[i].vidx_cnt != tot - tnode->free_cnt)
4158 tnode->refs = true;
4159 }
4160
4161 rc = npc_defrag_process(rvu, &x2lh);
4162 if (rc)
4163 goto free_vidx;
4164
4165 rc = npc_defrag_process(rvu, &x4lh);
4166 if (rc)
4167 goto free_vidx;
4168
4169 free_vidx:
4170 npc_unlock_all_subbank();
4171 mutex_unlock(&mcam->lock);
4172 for (i = 0; i < npc_priv->num_subbanks; i++)
4173 kfree(node[i].vidx);
4174 kfree(node);
4175 return rc;
4176 }
4177
rvu_mbox_handler_npc_defrag(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)4178 int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req,
4179 struct msg_rsp *rsp)
4180 {
4181 return npc_cn20k_defrag(rvu);
4182 }
4183
npc_cn20k_dft_rules_idx_get(struct rvu * rvu,u16 pcifunc,u16 * bcast,u16 * mcast,u16 * promisc,u16 * ucast)4184 int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
4185 u16 *mcast, u16 *promisc, u16 *ucast)
4186 {
4187 u16 *ptr[4] = {promisc, mcast, bcast, ucast};
4188 unsigned long idx;
4189 bool set = false;
4190 void *val;
4191 int i, j;
4192
4193 for (i = 0; i < ARRAY_SIZE(ptr); i++) {
4194 if (!ptr[i])
4195 continue;
4196
4197 *ptr[i] = USHRT_MAX;
4198 }
4199
4200 if (!npc_priv)
4201 return 0;
4202
4203 if (is_lbk_vf(rvu, pcifunc)) {
4204 if (!ptr[0])
4205 return -EINVAL;
4206
4207 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
4208 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
4209 if (!val) {
4210 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
4211 __func__,
4212 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
4213 pcifunc);
4214
4215 return -ESRCH;
4216 }
4217
4218 *ptr[0] = xa_to_value(val);
4219 return 0;
4220 }
4221
4222 if (is_vf(pcifunc)) {
4223 if (!ptr[3])
4224 return -EINVAL;
4225
4226 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
4227 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
4228 if (!val) {
4229 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
4230 __func__,
4231 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
4232 pcifunc);
4233
4234 return -ESRCH;
4235 }
4236
4237 *ptr[3] = xa_to_value(val);
4238 return 0;
4239 }
4240
4241 for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++,
4242 j++) {
4243 if (!ptr[j])
4244 continue;
4245
4246 idx = NPC_DFT_RULE_ID_MK(pcifunc, i);
4247 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
4248 if (!val) {
4249 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
4250 __func__,
4251 npc_dft_rule_name[i], pcifunc);
4252
4253 continue;
4254 }
4255
4256 *ptr[j] = xa_to_value(val);
4257 set = true;
4258 }
4259
4260 return set ? 0 : -ESRCH;
4261 }
4262
rvu_mbox_handler_npc_get_pfl_info(struct rvu * rvu,struct msg_req * req,struct npc_get_pfl_info_rsp * rsp)4263 int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req,
4264 struct npc_get_pfl_info_rsp *rsp)
4265 {
4266 if (!is_cn20k(rvu->pdev)) {
4267 dev_err(rvu->dev, "Mbox support is only for cn20k\n");
4268 return -EOPNOTSUPP;
4269 }
4270
4271 rsp->kw_type = npc_priv->kw;
4272 rsp->x4_slots = npc_priv->bank_depth;
4273 return 0;
4274 }
4275
rvu_mbox_handler_npc_get_num_kws(struct rvu * rvu,struct npc_get_num_kws_req * req,struct npc_get_num_kws_rsp * rsp)4276 int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu,
4277 struct npc_get_num_kws_req *req,
4278 struct npc_get_num_kws_rsp *rsp)
4279 {
4280 u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
4281 u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
4282 struct rvu_npc_mcam_rule dummy = { 0 };
4283 struct mcam_entry_mdata mdata = { };
4284 struct npc_install_flow_req *fl;
4285 int i, cnt = 0, blkaddr;
4286
4287 if (!is_cn20k(rvu->pdev)) {
4288 dev_err(rvu->dev, "Mbox support is only for cn20k\n");
4289 return -EOPNOTSUPP;
4290 }
4291
4292 fl = &req->fl;
4293
4294 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
4295 if (blkaddr < 0) {
4296 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
4297 return NPC_MCAM_INVALID_REQ;
4298 }
4299
4300 mdata.kw = kw;
4301 mdata.kw_mask = kw_mask;
4302
4303 npc_update_flow(rvu, &mdata, fl->features, &fl->packet,
4304 &fl->mask, &dummy, fl->intf, blkaddr);
4305
4306 /* Find the most significant word valid. Traverse from
4307 * MSB to LSB, check if cam0 or cam1 is set
4308 */
4309 for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) {
4310 if (kw[i] || kw_mask[i]) {
4311 cnt = i + 1;
4312 break;
4313 }
4314 }
4315
4316 rsp->kws = cnt;
4317
4318 return 0;
4319 }
4320
rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu * rvu,struct msg_req * req,struct npc_get_dft_rl_idxs_rsp * rsp)4321 int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req,
4322 struct npc_get_dft_rl_idxs_rsp *rsp)
4323 {
4324 u16 bcast, mcast, promisc, ucast;
4325 u16 pcifunc;
4326 int rc;
4327
4328 if (!is_cn20k(rvu->pdev)) {
4329 dev_err(rvu->dev, "Mbox support is only for cn20k\n");
4330 return -EOPNOTSUPP;
4331 }
4332
4333 pcifunc = req->hdr.pcifunc;
4334
4335 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast,
4336 &promisc, &ucast);
4337 if (rc)
4338 return rc;
4339
4340 rsp->bcast = bcast;
4341 rsp->mcast = mcast;
4342 rsp->promisc = promisc;
4343 rsp->ucast = ucast;
4344 return 0;
4345 }
4346
npc_is_cgx_or_lbk(struct rvu * rvu,u16 pcifunc)4347 bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
4348 {
4349 return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) ||
4350 is_lbk_vf(rvu, pcifunc);
4351 }
4352
npc_cn20k_dft_rules_free(struct rvu * rvu,u16 pcifunc)4353 void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc)
4354 {
4355 struct npc_mcam *mcam = &rvu->hw->mcam;
4356 u16 ptr[4] = {[0 ... 3] = USHRT_MAX};
4357 struct rvu_npc_mcam_rule *rule, *tmp;
4358 unsigned long index;
4359 int blkaddr, rc, i;
4360 void *map;
4361
4362 if (!npc_priv)
4363 return;
4364
4365 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
4366 dev_dbg(rvu->dev,
4367 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
4368 __func__, pcifunc);
4369 return;
4370 }
4371
4372 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1],
4373 &ptr[2], &ptr[3]);
4374 if (rc)
4375 return;
4376
4377 /* LBK */
4378 if (is_lbk_vf(rvu, pcifunc)) {
4379 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
4380 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
4381 if (!map)
4382 dev_dbg(rvu->dev,
4383 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
4384 __func__,
4385 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
4386 pcifunc);
4387
4388 goto free_rules;
4389 }
4390
4391 /* VF */
4392 if (is_vf(pcifunc)) {
4393 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
4394 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
4395 if (!map)
4396 dev_dbg(rvu->dev,
4397 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
4398 __func__,
4399 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
4400 pcifunc);
4401
4402 goto free_rules;
4403 }
4404
4405 /* PF */
4406 for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) {
4407 index = NPC_DFT_RULE_ID_MK(pcifunc, i);
4408 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
4409 if (!map)
4410 dev_dbg(rvu->dev,
4411 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
4412 __func__, npc_dft_rule_name[i],
4413 pcifunc);
4414 }
4415
4416 free_rules:
4417 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
4418 if (blkaddr < 0)
4419 return;
4420 for (int i = 0; i < 4; i++) {
4421 if (ptr[i] == USHRT_MAX)
4422 continue;
4423
4424 mutex_lock(&mcam->lock);
4425 npc_mcam_clear_bit(mcam, ptr[i]);
4426 mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP;
4427 npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false);
4428 mcam->entry2target_pffunc[ptr[i]] = 0x0;
4429 mutex_unlock(&mcam->lock);
4430
4431 rc = npc_cn20k_idx_free(rvu, &ptr[i], 1);
4432 if (rc) {
4433 /* Non recoverable error. Let us WARN and return. Keep system alive to
4434 * enable debugging
4435 */
4436 WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n",
4437 __func__, pcifunc, ptr[i]);
4438 return;
4439 }
4440 }
4441
4442 mutex_lock(&mcam->lock);
4443 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
4444 for (int i = 0; i < 4; i++) {
4445 if (ptr[i] != rule->entry)
4446 continue;
4447
4448 list_del(&rule->list);
4449 kfree(rule);
4450 break;
4451 }
4452 }
4453 mutex_unlock(&mcam->lock);
4454 }
4455
npc_cn20k_dft_rules_alloc(struct rvu * rvu,u16 pcifunc)4456 int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc)
4457 {
4458 struct npc_mcam_free_entry_req free_req = { 0 };
4459 u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc;
4460 struct npc_mcam_alloc_entry_req req = { 0 };
4461 struct npc_mcam_alloc_entry_rsp rsp = { 0 };
4462 int ret, eidx, i, k, pf, cnt;
4463 struct rvu_pfvf *pfvf;
4464 unsigned long index;
4465 struct msg_rsp free_rsp;
4466 u16 b, m, p, u;
4467
4468 if (!npc_priv)
4469 return 0;
4470
4471 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
4472 dev_dbg(rvu->dev,
4473 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
4474 __func__, pcifunc);
4475 return 0;
4476 }
4477
4478 /* Check if default rules are already alloced for this pcifunc */
4479 ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u);
4480 if (!ret) {
4481 dev_dbg(rvu->dev,
4482 "%s: default rules are already installed (pcifunc=%#x)\n",
4483 __func__, pcifunc);
4484 dev_dbg(rvu->dev,
4485 "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n",
4486 __func__, b, m, p, u);
4487 return 0;
4488 }
4489
4490 /* Set ref index as lowest priority index */
4491 eidx = 2 * npc_priv->bank_depth - 1;
4492
4493 /* Install only UCAST for VF */
4494 cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx);
4495
4496 /* For VF pcifunc, allocate default mcam indexes by taking
4497 * ref as PF's ucast index.
4498 */
4499 if (is_vf(pcifunc)) {
4500 pf = rvu_get_pf(rvu->pdev, pcifunc);
4501 pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT;
4502
4503 /* Get PF's ucast entry index */
4504 ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL,
4505 NULL, NULL, &pf_ucast);
4506
4507 /* There is no PF rules installed; and VF installation comes
4508 * first. PF may come later.
4509 * TODO: Install PF rules before installing VF rules.
4510 */
4511
4512 /* Set PF's ucast as ref entry */
4513 if (!ret)
4514 eidx = pf_ucast;
4515 }
4516
4517 pfvf = rvu_get_pfvf(rvu, pcifunc);
4518 pfvf->hw_prio = NPC_DFT_RULE_PRIO;
4519
4520 if (npc_priv->kw == NPC_MCAM_KEY_X4) {
4521 req.kw_type = NPC_MCAM_KEY_X4;
4522 req.ref_entry = eidx & (npc_priv->bank_depth - 1);
4523 } else {
4524 req.kw_type = NPC_MCAM_KEY_X2;
4525 req.ref_entry = eidx;
4526 }
4527
4528 req.contig = false;
4529 req.ref_prio = NPC_MCAM_HIGHER_PRIO;
4530 req.count = cnt;
4531 req.hdr.pcifunc = pcifunc;
4532
4533 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
4534
4535 /* successfully allocated index */
4536 if (!ret) {
4537 /* Copy indexes to local array */
4538 for (i = 0; i < cnt; i++)
4539 mcam_idx[i] = rsp.entry_list[i];
4540
4541 goto chk_sanity;
4542 }
4543
4544 /* If there is no slots available and request is for PF,
4545 * return error.
4546 */
4547 if (!is_vf(pcifunc)) {
4548 dev_err(rvu->dev,
4549 "%s: Default index allocation failed for pcifunc=%#x\n",
4550 __func__, pcifunc);
4551 return ret;
4552 }
4553
4554 /* We could not find an index with higher priority index for VF.
4555 * Find rule with lower priority index and set hardware priority
4556 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority)
4557 */
4558 req.contig = false;
4559 req.count = cnt;
4560 req.hdr.pcifunc = pcifunc;
4561 req.ref_prio = NPC_MCAM_LOWER_PRIO;
4562
4563 if (npc_priv->kw == NPC_MCAM_KEY_X4) {
4564 req.kw_type = NPC_MCAM_KEY_X4;
4565 req.ref_entry = eidx & (npc_priv->bank_depth - 1);
4566 } else {
4567 req.kw_type = NPC_MCAM_KEY_X2;
4568 req.ref_entry = eidx;
4569 }
4570
4571 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
4572 if (ret) {
4573 dev_err(rvu->dev,
4574 "%s: Default index allocation failed for pcifunc=%#x\n",
4575 __func__, pcifunc);
4576 return ret;
4577 }
4578
4579 /* Copy indexes to local array */
4580 for (i = 0; i < cnt; i++)
4581 mcam_idx[i] = rsp.entry_list[i];
4582
4583 pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1;
4584
4585 chk_sanity:
4586 /* LBK */
4587 if (is_lbk_vf(rvu, pcifunc)) {
4588 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
4589 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
4590 xa_mk_value(mcam_idx[0]), GFP_KERNEL);
4591 if (ret) {
4592 dev_err(rvu->dev,
4593 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
4594 __func__,
4595 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
4596 pcifunc);
4597 goto err;
4598 }
4599
4600 goto done;
4601 }
4602
4603 /* VF */
4604 if (is_vf(pcifunc)) {
4605 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
4606 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
4607 xa_mk_value(mcam_idx[0]), GFP_KERNEL);
4608 if (ret) {
4609 dev_err(rvu->dev,
4610 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
4611 __func__,
4612 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
4613 pcifunc);
4614 goto err;
4615 }
4616
4617 goto done;
4618 }
4619
4620 /* PF */
4621 for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID &&
4622 k < cnt; i++, k++) {
4623 index = NPC_DFT_RULE_ID_MK(pcifunc, i);
4624 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
4625 xa_mk_value(mcam_idx[k]), GFP_KERNEL);
4626 if (ret) {
4627 dev_err(rvu->dev,
4628 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
4629 __func__, npc_dft_rule_name[i],
4630 pcifunc);
4631 for (int p = NPC_DFT_RULE_START_ID; p < i; p++) {
4632 index = NPC_DFT_RULE_ID_MK(pcifunc, p);
4633 xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
4634 }
4635 goto err;
4636 }
4637 }
4638
4639 done:
4640 return 0;
4641
4642 err:
4643 free_req.hdr.pcifunc = pcifunc;
4644 free_req.all = 1;
4645 ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
4646 if (ret)
4647 dev_err(rvu->dev,
4648 "%s: Error deleting default entries (pcifunc=%#x\n",
4649 __func__, pcifunc);
4650
4651 return -EFAULT;
4652 }
4653
npc_priv_init(struct rvu * rvu)4654 static int npc_priv_init(struct rvu *rvu)
4655 {
4656 struct npc_mcam *mcam = &rvu->hw->mcam;
4657 int blkaddr, num_banks, bank_depth;
4658 int num_subbanks, subbank_depth;
4659 u64 npc_const1, npc_const2 = 0;
4660 struct npc_subbank *sb;
4661 int ret = -ENOMEM;
4662 u64 cfg;
4663 int i;
4664
4665 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
4666 if (blkaddr < 0) {
4667 dev_err(rvu->dev, "%s: NPC block not implemented\n",
4668 __func__);
4669 return -ENODEV;
4670 }
4671
4672 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1);
4673 if (npc_const1 & BIT_ULL(63))
4674 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2);
4675
4676 num_banks = mcam->banks;
4677 if (!num_banks || num_banks > MAX_NUM_BANKS) {
4678 dev_err(rvu->dev,
4679 "Number of banks(%u) is invalid\n", num_banks);
4680 return -EINVAL;
4681 }
4682
4683 num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2);
4684 if (!num_subbanks || num_subbanks > MAX_NUM_SUB_BANKS) {
4685 dev_err(rvu->dev,
4686 "Number of subbanks is invalid %u\n",
4687 num_subbanks);
4688 return -EFAULT;
4689 }
4690
4691 if (num_subbanks & (num_subbanks - 1)) {
4692 dev_err(rvu->dev,
4693 "subbanks cnt(%u) should be a power of 2\n",
4694 num_subbanks);
4695 return -EINVAL;
4696 }
4697
4698 bank_depth = mcam->banksize;
4699 if (!bank_depth || bank_depth % num_subbanks) {
4700 dev_err(rvu->dev,
4701 "Bank depth(%u) should be a multiple of num_subbanks(%u)\n",
4702 bank_depth, num_subbanks);
4703 return -EINVAL;
4704 }
4705
4706 subbank_depth = bank_depth / num_subbanks;
4707 if (!subbank_depth || subbank_depth > MAX_SUBBANK_DEPTH) {
4708 dev_err(rvu->dev,
4709 "Invalid subbank depth %u\n",
4710 subbank_depth);
4711 return -EINVAL;
4712 }
4713
4714 npc_priv = kcalloc(1, sizeof(*npc_priv), GFP_KERNEL);
4715 if (!npc_priv)
4716 return -ENOMEM;
4717
4718 npc_priv->num_banks = num_banks;
4719 npc_priv->num_subbanks = num_subbanks;
4720 npc_priv->bank_depth = bank_depth;
4721 npc_priv->subbank_depth = subbank_depth;
4722
4723 /* Get kex configured key size */
4724 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0));
4725 npc_priv->kw = FIELD_GET(GENMASK_ULL(34, 32), cfg);
4726
4727 dev_info(rvu->dev,
4728 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n",
4729 num_banks, bank_depth, num_subbanks, subbank_depth,
4730 npc_kw_name[npc_priv->kw]);
4731
4732 npc_priv->sb = kcalloc(num_subbanks, sizeof(struct npc_subbank),
4733 GFP_KERNEL);
4734 if (!npc_priv->sb)
4735 goto fail1;
4736
4737 xa_init_flags(&npc_priv->xa_sb_used, XA_FLAGS_ALLOC);
4738 xa_init_flags(&npc_priv->xa_sb_free, XA_FLAGS_ALLOC);
4739 xa_init_flags(&npc_priv->xa_idx2pf_map, XA_FLAGS_ALLOC);
4740 xa_init_flags(&npc_priv->xa_pf_map, XA_FLAGS_ALLOC);
4741 xa_init_flags(&npc_priv->xa_pf2dfl_rmap, XA_FLAGS_ALLOC);
4742 xa_init_flags(&npc_priv->xa_idx2vidx_map, XA_FLAGS_ALLOC);
4743 xa_init_flags(&npc_priv->xa_vidx2idx_map, XA_FLAGS_ALLOC);
4744
4745 if (npc_create_srch_order(num_subbanks))
4746 goto fail2;
4747
4748 npc_populate_restricted_idxs(num_subbanks);
4749
4750 /* Initialize subbanks */
4751 for (i = 0, sb = npc_priv->sb; i < num_subbanks; i++, sb++)
4752 npc_subbank_init(rvu, sb, i);
4753
4754 ret = npc_subbanks_srch_order_init(rvu);
4755 if (ret)
4756 goto fail3;
4757
4758 /* Get number of pcifuncs in the system */
4759 npc_priv->pf_cnt = npc_pcifunc_map_create(rvu);
4760 npc_priv->xa_pf2idx_map = kcalloc(npc_priv->pf_cnt,
4761 sizeof(struct xarray),
4762 GFP_KERNEL);
4763 if (!npc_priv->xa_pf2idx_map) {
4764 ret = -ENOMEM;
4765 goto fail3;
4766 }
4767
4768 for (i = 0; i < npc_priv->pf_cnt; i++)
4769 xa_init_flags(&npc_priv->xa_pf2idx_map[i], XA_FLAGS_ALLOC);
4770
4771 INIT_LIST_HEAD(&npc_priv->defrag_lh);
4772 mutex_init(&npc_priv->lock);
4773
4774 return 0;
4775
4776 fail3:
4777 kfree(subbank_srch_order);
4778 subbank_srch_order = NULL;
4779
4780 fail2:
4781 xa_destroy(&npc_priv->xa_sb_used);
4782 xa_destroy(&npc_priv->xa_sb_free);
4783 xa_destroy(&npc_priv->xa_idx2pf_map);
4784 xa_destroy(&npc_priv->xa_pf_map);
4785 xa_destroy(&npc_priv->xa_pf2dfl_rmap);
4786 xa_destroy(&npc_priv->xa_idx2vidx_map);
4787 xa_destroy(&npc_priv->xa_vidx2idx_map);
4788 kfree(npc_priv->sb);
4789 npc_priv->sb = NULL;
4790 fail1:
4791 kfree(npc_priv);
4792 npc_priv = NULL;
4793 return ret;
4794 }
4795
npc_cn20k_deinit(struct rvu * rvu)4796 void npc_cn20k_deinit(struct rvu *rvu)
4797 {
4798 int i;
4799
4800 if (!npc_priv)
4801 return;
4802
4803 xa_destroy(&npc_priv->xa_sb_used);
4804 xa_destroy(&npc_priv->xa_sb_free);
4805 xa_destroy(&npc_priv->xa_idx2pf_map);
4806 xa_destroy(&npc_priv->xa_pf_map);
4807 xa_destroy(&npc_priv->xa_pf2dfl_rmap);
4808 xa_destroy(&npc_priv->xa_idx2vidx_map);
4809 xa_destroy(&npc_priv->xa_vidx2idx_map);
4810
4811 for (i = 0; i < npc_priv->pf_cnt; i++)
4812 xa_destroy(&npc_priv->xa_pf2idx_map[i]);
4813
4814 kfree(npc_priv->xa_pf2idx_map);
4815 /* No need to destroy mutex lock as it is
4816 * part of subbank structure
4817 */
4818 kfree(npc_priv->sb);
4819 kfree(subbank_srch_order);
4820 bitmap_clear(npc_priv->en_map, 0, MAX_NUM_BANKS * MAX_NUM_SUB_BANKS *
4821 MAX_SUBBANK_DEPTH);
4822 npc_defrag_list_clear();
4823 kfree(npc_priv);
4824 npc_priv = NULL;
4825 }
4826
npc_setup_mcam_section(struct rvu * rvu,int key_type)4827 static int npc_setup_mcam_section(struct rvu *rvu, int key_type)
4828 {
4829 int blkaddr, sec;
4830
4831 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
4832 if (blkaddr < 0) {
4833 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
4834 return -ENODEV;
4835 }
4836
4837 for (sec = 0; sec < npc_priv->num_subbanks; sec++)
4838 rvu_write64(rvu, blkaddr,
4839 NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type);
4840
4841 return 0;
4842 }
4843
npc_cn20k_init(struct rvu * rvu)4844 int npc_cn20k_init(struct rvu *rvu)
4845 {
4846 int err;
4847
4848 err = npc_priv_init(rvu);
4849 if (err) {
4850 dev_err(rvu->dev, "%s: Error to init\n",
4851 __func__);
4852 return err;
4853 }
4854
4855 err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2);
4856 if (err) {
4857 dev_err(rvu->dev, "%s: mcam section configuration failure\n",
4858 __func__);
4859 goto fail;
4860 }
4861
4862 return 0;
4863
4864 fail:
4865 npc_cn20k_deinit(rvu);
4866 return err;
4867 }
4868