1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2026 Marvell. 5 * 6 */ 7 #include <linux/xarray.h> 8 #include <linux/bitfield.h> 9 10 #include "rvu.h" 11 #include "npc.h" 12 #include "npc_profile.h" 13 #include "rvu_npc_hash.h" 14 #include "rvu_npc.h" 15 #include "cn20k/npc.h" 16 #include "cn20k/reg.h" 17 #include "rvu_npc_fs.h" 18 19 static struct npc_priv_t *npc_priv; 20 21 static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { 22 [NPC_MCAM_KEY_DYN] = "DYNAMIC", 23 [NPC_MCAM_KEY_X2] = "X2", 24 [NPC_MCAM_KEY_X4] = "X4", 25 }; 26 27 static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { 28 [NPC_DFT_RULE_PROMISC_ID] = "Promisc", 29 [NPC_DFT_RULE_MCAST_ID] = "Mcast", 30 [NPC_DFT_RULE_BCAST_ID] = "Bcast", 31 [NPC_DFT_RULE_UCAST_ID] = "Ucast", 32 }; 33 34 #define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ 35 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 36 ((key_ofs) & 0x3F)) 37 38 #define NPC_DFT_RULE_ID_MK(pcifunc, id) \ 39 ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) 40 41 #define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ 42 FIELD_GET(GENMASK_ULL(15, 0), rid) 43 44 #define NPC_DFT_RULE_ID_2_ID(rid) \ 45 FIELD_GET(GENMASK_ULL(31, 16), rid) 46 47 #define NPC_DFT_RULE_PRIO 127 48 49 static const char cn20k_def_pfl_name[] = "default"; 50 51 static struct npc_mcam_kex_extr npc_mkex_extr_default = { 52 .mkex_sign = MKEX_CN20K_SIGN, 53 .name = "default", 54 .kpu_version = NPC_KPU_PROFILE_VER, 55 .keyx_cfg = { 56 /* nibble: LA..LE (ltype only) + Error code + Channel */ 57 [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | 58 NPC_CN20K_PARSE_NIBBLE_INTF_RX | 59 NPC_CN20K_PARSE_NIBBLE_ERRCODE, 60 61 /* nibble: LA..LE (ltype only) */ 62 [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | 63 NPC_CN20K_PARSE_NIBBLE_INTF_TX, 64 }, 65 .intf_extr_lid = { 66 /* Default RX MCAM KEX profile */ 67 [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 68 NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, 69 [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 70 NPC_LID_LC, NPC_LID_LD }, 71 }, 72 .intf_extr_lt = { 73 /* Default RX MCAM KEX profile */ 74 [NIX_INTF_RX] = { 75 [0] = { 76 /* Layer A: Ethernet: */ 77 [NPC_LT_LA_ETHER] = 78 /* DMAC: 6 bytes, KW1[63:15] */ 79 KEX_EXTR_CFG(0x05, 0x0, 0x1, 80 NPC_KEXOF_DMAC + 1), 81 [NPC_LT_LA_CPT_HDR] = 82 /* DMAC: 6 bytes, KW1[63:15] */ 83 KEX_EXTR_CFG(0x05, 0x0, 0x1, 84 NPC_KEXOF_DMAC + 1), 85 }, 86 [1] = { 87 /* Layer A: Ethernet: */ 88 [NPC_LT_LA_ETHER] = 89 /* Ethertype: 2 bytes, KW0[63:48] */ 90 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 91 [NPC_LT_LA_CPT_HDR] = 92 /* Ethertype: 2 bytes, KW0[63:48] */ 93 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 94 }, 95 [2] = { 96 /* Layer B: Single VLAN (CTAG) */ 97 [NPC_LT_LB_CTAG] = 98 /* CTAG VLAN: 2 bytes, KW1[15:0] */ 99 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 100 /* Layer B: Stacked VLAN (STAG|QinQ) */ 101 [NPC_LT_LB_STAG_QINQ] = 102 /* Outer VLAN: 2 bytes, KW1[15:0] */ 103 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 104 [NPC_LT_LB_FDSA] = 105 /* SWITCH PORT: 1 byte, KW1[7:0] */ 106 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), 107 }, 108 [3] = { 109 [NPC_LT_LB_CTAG] = 110 /* Ethertype: 2 bytes, KW0[63:48] */ 111 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 112 [NPC_LT_LB_STAG_QINQ] = 113 /* Ethertype: 2 bytes, KW0[63:48] */ 114 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), 115 [NPC_LT_LB_FDSA] = 116 /* Ethertype: 2 bytes, KW0[63:48] */ 117 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 118 }, 119 [4] = { 120 /* Layer C: IPv4 */ 121 [NPC_LT_LC_IP] = 122 /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ 123 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), 124 /* Layer C: IPv6 */ 125 [NPC_LT_LC_IP6] = 126 /* Everything up to SADDR: 8 bytes, KW3[7:0], 127 * KW2[63:8] 128 */ 129 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), 130 }, 131 [5] = { 132 [NPC_LT_LC_IP] = 133 /* TOS: 1 byte, KW2[7:0] */ 134 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), 135 }, 136 [6] = { 137 /* Layer D:UDP */ 138 [NPC_LT_LD_UDP] = 139 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 140 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 141 /* Layer D:TCP */ 142 [NPC_LT_LD_TCP] = 143 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 144 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 145 }, 146 }, 147 /* Default TX MCAM KEX profile */ 148 [NIX_INTF_TX] = { 149 [0] = { 150 /* Layer A: NIX_INST_HDR_S + Ethernet */ 151 /* NIX appends 8 bytes of NIX_INST_HDR_S at the 152 * start of each TX packet supplied to NPC. 153 */ 154 [NPC_LT_LA_IH_NIX_ETHER] = 155 /* PF_FUNC: 2B , KW0 [47:32] */ 156 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 157 /* Layer A: HiGig2: */ 158 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 159 /* PF_FUNC: 2B , KW0 [47:32] */ 160 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 161 }, 162 [1] = { 163 [NPC_LT_LA_IH_NIX_ETHER] = 164 /* SQ_ID 3 bytes, KW1[63:16] */ 165 KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), 166 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 167 /* VID: 2 bytes, KW1[31:16] */ 168 KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), 169 }, 170 [2] = { 171 /* Layer B: Single VLAN (CTAG) */ 172 [NPC_LT_LB_CTAG] = 173 /* CTAG VLAN[2..3] KW0[63:48] */ 174 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 175 /* Layer B: Stacked VLAN (STAG|QinQ) */ 176 [NPC_LT_LB_STAG_QINQ] = 177 /* Outer VLAN: 2 bytes, KW0[63:48] */ 178 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 179 }, 180 [3] = { 181 [NPC_LT_LB_CTAG] = 182 /* CTAG VLAN[2..3] KW1[15:0] */ 183 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), 184 [NPC_LT_LB_STAG_QINQ] = 185 /* Outer VLAN: 2 Bytes, KW1[15:0] */ 186 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), 187 }, 188 [4] = { 189 /* Layer C: IPv4 */ 190 [NPC_LT_LC_IP] = 191 /* SIP+DIP: 8 bytes, KW2[63:0] */ 192 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), 193 /* Layer C: IPv6 */ 194 [NPC_LT_LC_IP6] = 195 /* Everything up to SADDR: 8 bytes, KW2[63:0] */ 196 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), 197 }, 198 [5] = { 199 /* Layer D:UDP */ 200 [NPC_LT_LD_UDP] = 201 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 202 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 203 /* Layer D:TCP */ 204 [NPC_LT_LD_TCP] = 205 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 206 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 207 }, 208 }, 209 }, 210 }; 211 212 struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) 213 { 214 return &npc_mkex_extr_default; 215 } 216 217 static u16 npc_idx2vidx(u16 idx) 218 { 219 unsigned long index; 220 void *map; 221 u16 vidx; 222 int val; 223 224 vidx = idx; 225 index = idx; 226 227 map = xa_load(&npc_priv->xa_idx2vidx_map, index); 228 if (!map) 229 goto done; 230 231 val = xa_to_value(map); 232 if (val == -1) 233 goto done; 234 235 vidx = val; 236 237 done: 238 return vidx; 239 } 240 241 static bool npc_is_vidx(u16 vidx) 242 { 243 return vidx >= npc_priv->bank_depth * 2; 244 } 245 246 static u16 npc_vidx2idx(u16 vidx) 247 { 248 unsigned long sentinel = (unsigned long)-1; 249 unsigned long index; 250 void *map; 251 int val; 252 u16 idx; 253 254 idx = vidx; 255 index = vidx; 256 257 map = xa_load(&npc_priv->xa_vidx2idx_map, index); 258 if (!map) 259 goto done; 260 261 val = xa_to_value(map); 262 if (val == sentinel) 263 goto done; 264 265 idx = val; 266 267 done: 268 return idx; 269 } 270 271 u16 npc_cn20k_vidx2idx(u16 idx) 272 { 273 if (!npc_priv) 274 return idx; 275 276 if (!npc_is_vidx(idx)) 277 return idx; 278 279 return npc_vidx2idx(idx); 280 } 281 282 u16 npc_cn20k_idx2vidx(u16 idx) 283 { 284 if (!npc_priv) 285 return idx; 286 287 if (npc_is_vidx(idx)) 288 return idx; 289 290 return npc_idx2vidx(idx); 291 } 292 293 static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) 294 { 295 u16 mcam_idx; 296 void *map; 297 298 if (!npc_is_vidx(vidx)) { 299 dev_err(rvu->dev, 300 "%s: vidx(%u) does not map to proper mcam idx\n", 301 __func__, vidx); 302 return -ESRCH; 303 } 304 305 mcam_idx = npc_vidx2idx(vidx); 306 307 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 308 if (!map) { 309 dev_err(rvu->dev, 310 "%s: vidx(%u) does not map to proper mcam idx\n", 311 __func__, vidx); 312 return -ESRCH; 313 } 314 315 map = xa_erase(&npc_priv->xa_idx2vidx_map, mcam_idx); 316 if (!map) { 317 dev_err(rvu->dev, 318 "%s: vidx(%u) is not valid\n", 319 __func__, vidx); 320 return -ESRCH; 321 } 322 323 if (old_midx) 324 *old_midx = mcam_idx; 325 326 return 0; 327 } 328 329 static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) 330 { 331 u16 old_midx; 332 void *map; 333 int rc; 334 335 if (!npc_is_vidx(vidx)) { 336 dev_err(rvu->dev, 337 "%s: vidx(%u) does not map to proper mcam idx\n", 338 __func__, vidx); 339 return -ESRCH; 340 } 341 342 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 343 if (!map) { 344 dev_err(rvu->dev, 345 "%s: vidx(%u) could not be deleted from vidx2idx map\n", 346 __func__, vidx); 347 return -ESRCH; 348 } 349 350 old_midx = xa_to_value(map); 351 352 rc = xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 353 xa_mk_value(new_midx), GFP_KERNEL); 354 if (rc) { 355 dev_err(rvu->dev, 356 "%s: vidx(%u) cannot be added to vidx2idx map\n", 357 __func__, vidx); 358 goto fail1; 359 } 360 361 map = xa_erase(&npc_priv->xa_idx2vidx_map, old_midx); 362 if (!map) { 363 dev_err(rvu->dev, 364 "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 365 __func__, old_midx, vidx); 366 rc = -ESRCH; 367 goto fail2; 368 } 369 370 rc = xa_insert(&npc_priv->xa_idx2vidx_map, new_midx, 371 xa_mk_value(vidx), GFP_KERNEL); 372 if (rc) { 373 dev_err(rvu->dev, 374 "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 375 __func__, new_midx, vidx); 376 goto fail3; 377 } 378 379 return 0; 380 381 fail3: 382 /* Restore vidx at old_midx location */ 383 if (xa_insert(&npc_priv->xa_idx2vidx_map, old_midx, 384 xa_mk_value(vidx), GFP_KERNEL)) 385 dev_err(rvu->dev, 386 "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", 387 __func__, old_midx, vidx); 388 fail2: 389 /* Erase new_midx inserted at vidx */ 390 if (!xa_erase(&npc_priv->xa_vidx2idx_map, vidx)) 391 dev_err(rvu->dev, 392 "%s: Failed to roll back vidx2idx vidx=%u\n", 393 __func__, vidx); 394 395 fail1: 396 /* Restore old_midx at vidx location */ 397 if (xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 398 xa_mk_value(old_midx), GFP_KERNEL)) 399 dev_err(rvu->dev, 400 "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", 401 __func__, old_midx, vidx); 402 403 return rc; 404 } 405 406 static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, 407 u16 *vidx) 408 { 409 int rc, max, min; 410 u32 id; 411 412 /* Virtual index start from maximum mcam index + 1 */ 413 max = npc_priv->bank_depth * 2 * 2 - 1; 414 min = npc_priv->bank_depth * 2; 415 416 rc = xa_alloc(&npc_priv->xa_vidx2idx_map, &id, 417 xa_mk_value(mcam_idx), 418 XA_LIMIT(min, max), GFP_KERNEL); 419 if (rc) { 420 dev_err(rvu->dev, 421 "%s: Failed to add to vidx2idx map (%u)\n", 422 __func__, mcam_idx); 423 goto fail1; 424 } 425 426 rc = xa_insert(&npc_priv->xa_idx2vidx_map, mcam_idx, 427 xa_mk_value(id), GFP_KERNEL); 428 if (rc) { 429 dev_err(rvu->dev, 430 "%s: Failed to add to idx2vidx map (%u)\n", 431 __func__, mcam_idx); 432 goto fail2; 433 } 434 435 if (vidx) 436 *vidx = id; 437 438 return 0; 439 440 fail2: 441 xa_erase(&npc_priv->xa_vidx2idx_map, id); 442 fail1: 443 return rc; 444 } 445 446 static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, 447 const struct npc_kpu_profile_cam *kpucam, 448 int kpm, int entry) 449 { 450 struct npc_kpu_cam cam0 = {0}; 451 struct npc_kpu_cam cam1 = {0}; 452 453 cam1.state = kpucam->state & kpucam->state_mask; 454 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 455 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 456 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 457 458 cam0.state = ~kpucam->state & kpucam->state_mask; 459 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 460 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 461 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 462 463 rvu_write64(rvu, blkaddr, 464 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); 465 rvu_write64(rvu, blkaddr, 466 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); 467 } 468 469 static void 470 npc_config_kpmaction(struct rvu *rvu, int blkaddr, 471 const struct npc_kpu_profile_action *kpuaction, 472 int kpm, int entry, bool pkind) 473 { 474 struct npc_kpm_action0 action0 = {0}; 475 struct npc_kpu_action1 action1 = {0}; 476 u64 reg; 477 478 action1.errlev = kpuaction->errlev; 479 action1.errcode = kpuaction->errcode; 480 action1.dp0_offset = kpuaction->dp0_offset; 481 action1.dp1_offset = kpuaction->dp1_offset; 482 action1.dp2_offset = kpuaction->dp2_offset; 483 484 if (pkind) 485 reg = NPC_AF_PKINDX_ACTION1(entry); 486 else 487 reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); 488 489 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 490 491 action0.byp_count = kpuaction->bypass_count & 0x7; 492 action0.capture_ena = kpuaction->cap_ena & 1; 493 action0.parse_done = kpuaction->parse_done & 1; 494 action0.next_state = kpuaction->next_state & 0xf; 495 action0.capture_lid = kpuaction->lid & 0x7; 496 497 /* Parser functionality will work correctly even though 498 * upper flag bits are silently discarded 499 */ 500 action0.capture_ltype = kpuaction->ltype & 0xf; 501 action0.capture_flags = kpuaction->flags & 0xf; 502 action0.ptr_advance = kpuaction->ptr_advance; 503 504 action0.var_len_offset = kpuaction->offset; 505 action0.var_len_mask = kpuaction->mask; 506 action0.var_len_right = kpuaction->right & 1; 507 action0.var_len_shift = kpuaction->shift & 1; 508 509 if (pkind) 510 reg = NPC_AF_PKINDX_ACTION0(entry); 511 else 512 reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); 513 514 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 515 } 516 517 static void 518 npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, 519 int kpm, int start_entry, 520 const struct npc_kpu_profile *profile) 521 { 522 int num_cam_entries, num_action_entries; 523 int entry, num_entries, max_entries; 524 u64 idx; 525 526 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile); 527 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile); 528 529 if (num_cam_entries != num_action_entries) { 530 dev_err(rvu->dev, 531 "kpm%d: CAM and action entries [%d != %d] not equal\n", 532 kpm, num_cam_entries, num_action_entries); 533 534 WARN(1, "Fatal error\n"); 535 return; 536 } 537 538 max_entries = rvu->hw->npc_kpu_entries / 2; 539 entry = start_entry; 540 /* Program CAM match entries for previous kpm extracted data */ 541 num_entries = min_t(int, num_cam_entries, max_entries); 542 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 543 npc_config_kpmcam(rvu, blkaddr, 544 npc_get_kpu_cam_nth_entry(rvu, profile, idx), 545 kpm, entry); 546 547 entry = start_entry; 548 /* Program this kpm's actions */ 549 num_entries = min_t(int, num_action_entries, max_entries); 550 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 551 npc_config_kpmaction(rvu, blkaddr, 552 npc_get_kpu_action_nth_entry(rvu, profile, idx), 553 kpm, entry, false); 554 } 555 556 static void 557 npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) 558 { 559 u64 entry_mask; 560 561 entry_mask = npc_enable_mask(num_entries); 562 /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ 563 if (!rvu->kpu.custom) 564 entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); 567 if (num_entries <= 64) { 568 /* Disable all the entries in W1, W2 and W3 */ 569 rvu_write64(rvu, blkaddr, 570 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), 571 npc_enable_mask(0)); 572 rvu_write64(rvu, blkaddr, 573 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 574 npc_enable_mask(0)); 575 rvu_write64(rvu, blkaddr, 576 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 577 npc_enable_mask(0)); 578 return; 579 } 580 581 num_entries = num_entries - 64; 582 entry_mask = npc_enable_mask(num_entries); 583 rvu_write64(rvu, blkaddr, 584 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); 585 if (num_entries <= 64) { 586 /* Disable all the entries in W2 and W3 */ 587 rvu_write64(rvu, blkaddr, 588 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 589 npc_enable_mask(0)); 590 rvu_write64(rvu, blkaddr, 591 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 592 npc_enable_mask(0)); 593 return; 594 } 595 596 num_entries = num_entries - 64; 597 entry_mask = npc_enable_mask(num_entries); 598 rvu_write64(rvu, blkaddr, 599 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); 600 if (num_entries <= 64) { 601 /* Disable all the entries in W3 */ 602 rvu_write64(rvu, blkaddr, 603 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 604 npc_enable_mask(0)); 605 return; 606 } 607 608 num_entries = num_entries - 64; 609 entry_mask = npc_enable_mask(num_entries); 610 rvu_write64(rvu, blkaddr, 611 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); 612 } 613 614 #define KPU_OFFSET 8 615 static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) 616 { 617 const struct npc_kpu_profile *profile1, *profile2; 618 int pfl1_num_cam_entries, pfl2_num_cam_entries; 619 int idx, total_cam_entries; 620 621 for (idx = 0; idx < num_kpms; idx++) { 622 profile1 = &rvu->kpu.kpu[idx]; 623 pfl1_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile1); 624 npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); 625 profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; 626 pfl2_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile2); 627 628 npc_program_single_kpm_profile(rvu, blkaddr, idx, 629 pfl1_num_cam_entries, 630 profile2); 631 total_cam_entries = pfl1_num_cam_entries + pfl2_num_cam_entries; 632 npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); 633 rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), 634 pfl1_num_cam_entries); 635 /* Enable the KPUs associated with this KPM */ 636 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); 637 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), 638 0x01); 639 } 640 } 641 642 void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) 643 { 644 struct npc_kpu_profile_action *act; 645 struct rvu_hwinfo *hw = rvu->hw; 646 int num_pkinds, idx; 647 648 /* Disable all KPMs and their entries */ 649 for (idx = 0; idx < hw->npc_kpms; idx++) { 650 rvu_write64(rvu, blkaddr, 651 NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); 652 rvu_write64(rvu, blkaddr, 653 NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); 654 rvu_write64(rvu, blkaddr, 655 NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); 656 rvu_write64(rvu, blkaddr, 657 NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); 658 } 659 660 for (idx = 0; idx < hw->npc_kpus; idx++) 661 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 662 663 /* Load and customize KPU profile. */ 664 npc_load_kpu_profile(rvu); 665 666 /* Configure KPU and KPM mapping for second pass */ 667 rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); 668 669 /* First program IKPU profile i.e PKIND configs. 670 * Check HW max count to avoid configuring junk or 671 * writing to unsupported CSR addresses. 672 */ 673 num_pkinds = rvu->kpu.pkinds; 674 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 675 676 /* Cn20k does not support Custom profile from filesystem */ 677 for (idx = 0; idx < num_pkinds; idx++) { 678 act = npc_get_ikpu_nth_entry(rvu, idx); 679 if (!act) 680 continue; 681 682 npc_config_kpmaction(rvu, blkaddr, act, 683 0, idx, true); 684 } 685 686 /* Program KPM CAM and Action profiles */ 687 npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); 688 } 689 690 struct npc_priv_t *npc_priv_get(void) 691 { 692 return npc_priv; 693 } 694 695 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 696 const struct npc_mcam_kex_extr *mkex_extr, 697 u8 intf) 698 { 699 u8 num_extr = rvu->hw->npc_kex_extr; 700 int extr, lt; 701 u64 val; 702 703 if (is_npc_intf_tx(intf)) 704 return; 705 706 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 707 mkex_extr->keyx_cfg[NIX_INTF_RX]); 708 709 /* Program EXTRACTOR */ 710 for (extr = 0; extr < num_extr; extr++) 711 rvu_write64(rvu, blkaddr, 712 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 713 mkex_extr->intf_extr_lid[intf][extr]); 714 715 /* Program EXTRACTOR_LTYPE */ 716 for (extr = 0; extr < num_extr; extr++) { 717 for (lt = 0; lt < NPC_MAX_LT; lt++) { 718 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 719 CN20K_SET_EXTR_LT(intf, extr, lt, val); 720 } 721 } 722 } 723 724 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 725 const struct npc_mcam_kex_extr *mkex_extr, 726 u8 intf) 727 { 728 u8 num_extr = rvu->hw->npc_kex_extr; 729 int extr, lt; 730 u64 val; 731 732 if (is_npc_intf_rx(intf)) 733 return; 734 735 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 736 mkex_extr->keyx_cfg[NIX_INTF_TX]); 737 738 /* Program EXTRACTOR */ 739 for (extr = 0; extr < num_extr; extr++) 740 rvu_write64(rvu, blkaddr, 741 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 742 mkex_extr->intf_extr_lid[intf][extr]); 743 744 /* Program EXTRACTOR_LTYPE */ 745 for (extr = 0; extr < num_extr; extr++) { 746 for (lt = 0; lt < NPC_MAX_LT; lt++) { 747 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 748 CN20K_SET_EXTR_LT(intf, extr, lt, val); 749 } 750 } 751 } 752 753 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 754 const struct npc_mcam_kex_extr *mkex_extr) 755 { 756 struct rvu_hwinfo *hw = rvu->hw; 757 u8 intf; 758 759 for (intf = 0; intf < hw->npc_intfs; intf++) { 760 npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); 761 npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); 762 } 763 764 /* Programme mkex hash profile */ 765 npc_program_mkex_hash(rvu, blkaddr); 766 } 767 768 void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, 769 const char *mkex_profile) 770 { 771 struct npc_mcam_kex_extr *mcam_kex_extr; 772 struct device *dev = &rvu->pdev->dev; 773 void __iomem *mkex_prfl_addr = NULL; 774 u64 prfl_sz; 775 int ret; 776 777 /* If user not selected mkex profile */ 778 if (rvu->kpu_fwdata_sz || 779 !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) 780 goto program_mkex_extr; 781 782 /* Setting up the mapping for mkex profile image */ 783 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 784 if (ret < 0) 785 goto program_mkex_extr; 786 787 mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; 788 789 while (((s64)prfl_sz > 0) && 790 (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { 791 /* Compare with mkex mod_param name string */ 792 if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && 793 !strncmp(mcam_kex_extr->name, mkex_profile, 794 MKEX_NAME_LEN)) { 795 rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; 796 goto program_mkex_extr; 797 } 798 799 mcam_kex_extr++; 800 prfl_sz -= sizeof(struct npc_mcam_kex_extr); 801 } 802 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 803 rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 804 805 program_mkex_extr: 806 dev_info(rvu->dev, "Using %s mkex profile\n", 807 rvu->kpu.mcam_kex_prfl.mkex_extr->name); 808 /* Program selected mkex profile */ 809 npc_program_mkex_profile(rvu, blkaddr, 810 rvu->kpu.mcam_kex_prfl.mkex_extr); 811 if (mkex_prfl_addr) 812 iounmap(mkex_prfl_addr); 813 } 814 815 int 816 npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, 817 int index, bool enable) 818 { 819 struct npc_mcam *mcam = &rvu->hw->mcam; 820 int mcam_idx = index % mcam->banksize; 821 int bank = index / mcam->banksize; 822 u64 cfg, hw_prio; 823 u8 kw_type; 824 825 if (index < 0 || index >= mcam->total_entries) 826 return -EINVAL; 827 828 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 829 return -EINVAL; 830 831 if (kw_type == NPC_MCAM_KEY_X2) { 832 cfg = rvu_read64(rvu, blkaddr, 833 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 834 bank)); 835 hw_prio = cfg & GENMASK_ULL(30, 24); 836 cfg = enable ? 1 : 0; 837 cfg |= hw_prio; 838 rvu_write64(rvu, blkaddr, 839 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 840 cfg); 841 goto update_en_map; 842 } 843 844 /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks 845 * need to be programmed with the same value. 846 */ 847 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 848 cfg = rvu_read64(rvu, blkaddr, 849 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 850 bank)); 851 hw_prio = cfg & GENMASK_ULL(30, 24); 852 cfg = enable ? 1 : 0; 853 cfg |= hw_prio; 854 rvu_write64(rvu, blkaddr, 855 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 856 cfg); 857 } 858 859 update_en_map: 860 if (enable) 861 set_bit(index, npc_priv->en_map); 862 else 863 clear_bit(index, npc_priv->en_map); 864 865 return 0; 866 } 867 868 static void 869 npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index) 870 { 871 rvu_write64(rvu, blkaddr, 872 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 873 0); 874 rvu_write64(rvu, blkaddr, 875 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 876 0); 877 878 rvu_write64(rvu, blkaddr, 879 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); 880 rvu_write64(rvu, blkaddr, 881 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); 882 883 rvu_write64(rvu, blkaddr, 884 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); 885 rvu_write64(rvu, blkaddr, 886 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); 887 888 rvu_write64(rvu, blkaddr, 889 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); 890 rvu_write64(rvu, blkaddr, 891 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); 892 893 rvu_write64(rvu, blkaddr, 894 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); 895 rvu_write64(rvu, blkaddr, 896 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); 897 898 /* Clear corresponding stats register */ 899 rvu_write64(rvu, blkaddr, 900 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 901 } 902 903 int 904 npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx) 905 { 906 struct npc_mcam *mcam = &rvu->hw->mcam; 907 int bank = npc_get_bank(mcam, mcam_idx); 908 u8 kw_type; 909 int index; 910 911 if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type)) 912 return -EINVAL; 913 914 index = mcam_idx & (mcam->banksize - 1); 915 916 if (kw_type == NPC_MCAM_KEY_X2) { 917 npc_clear_x2_entry(rvu, blkaddr, bank, index); 918 return 0; 919 } 920 921 /* For NPC_MCAM_KEY_X4 keys, both the banks 922 * need to be programmed with the same value. 923 */ 924 for (bank = 0; bank < mcam->banks_per_entry; bank++) 925 npc_clear_x2_entry(rvu, blkaddr, bank, index); 926 927 return 0; 928 } 929 930 static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, 931 u64 *cam0, u64 *cam1) 932 { 933 u64 kw_mask; 934 935 /* The two banks of every MCAM entry are used as a single double-wide 936 * entry that is compared with the search key as follows: 937 * 938 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] 939 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] 940 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] 941 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] 942 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] 943 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] 944 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] 945 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] 946 */ 947 *cam1 = entry->kw[idx]; 948 kw_mask = entry->kw_mask[idx]; 949 *cam1 &= kw_mask; 950 *cam0 = ~*cam1 & kw_mask; 951 } 952 953 /*------------------------------------------------------------------------- 954 *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | 955 * in | | | | || | | 956 * profile| len | | | ||len | type | 957 *------------------------------------------------------------------------- 958 *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | 959 *------------------------------------------------------------------------| 960 *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 961 *------------------------------------------------------------------------| 962 *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 963 *------------------------------------------------------------------------| 964 *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 965 *------------------------------------------------------------------------| 966 *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 967 *------------------------------------------------------------------------| 968 *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 969 *------------------------------------------------------------------------| 970 *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 971 *------------------------------------------------------------------------| 972 * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | 973 *------------------------------------------------------------------------| 974 */ 975 static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, 976 int blkaddr, int index, u8 intf, 977 struct cn20k_mcam_entry *entry, 978 int bank, u8 kw_type, int kw, 979 u8 req_kw_type) 980 { 981 u64 intf_ext = 0, intf_ext_mask = 0; 982 u8 tx_intf_mask = ~intf & 0x3; 983 u8 tx_intf = intf, kex_type; 984 u8 kw_type_mask = ~kw_type; 985 u64 cam0, cam1, kex_cfg; 986 987 if (is_npc_intf_tx(intf)) { 988 /* Last bit must be set and rest don't care 989 * for TX interfaces 990 */ 991 tx_intf_mask = 0x1; 992 tx_intf = intf & tx_intf_mask; 993 tx_intf_mask = ~tx_intf & tx_intf_mask; 994 } 995 996 kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 997 kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; 998 if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && 999 kw_type == NPC_MCAM_KEY_X2) { 1000 kw_type = 0; 1001 kw_type_mask = 0; 1002 } 1003 1004 /* Say, we need to write x2 keyword in an x4 subbank. 1005 * req_kw_type will be x2, and kw_type will be x4. 1006 * So in the case ignore kw bits in mcam. 1007 */ 1008 if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { 1009 kw_type = 0; 1010 kw_type_mask = 0; 1011 } 1012 1013 intf_ext = ((u64)kw_type << 16) | tx_intf; 1014 intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | 1015 tx_intf_mask; 1016 rvu_write64(rvu, blkaddr, 1017 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 1018 intf_ext); 1019 rvu_write64(rvu, blkaddr, 1020 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 1021 intf_ext_mask); 1022 1023 /* Set the match key */ 1024 npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); 1025 rvu_write64(rvu, blkaddr, 1026 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 1027 cam1); 1028 rvu_write64(rvu, blkaddr, 1029 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 1030 cam0); 1031 1032 npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); 1033 rvu_write64(rvu, blkaddr, 1034 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 1035 cam1); 1036 rvu_write64(rvu, blkaddr, 1037 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 1038 cam0); 1039 1040 npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); 1041 rvu_write64(rvu, blkaddr, 1042 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 1043 cam1); 1044 rvu_write64(rvu, blkaddr, 1045 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 1046 cam0); 1047 1048 npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); 1049 rvu_write64(rvu, blkaddr, 1050 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 1051 cam1); 1052 rvu_write64(rvu, blkaddr, 1053 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 1054 cam0); 1055 } 1056 1057 static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, 1058 int blkaddr, int index, u8 intf, 1059 struct cn20k_mcam_entry *entry, 1060 u8 kw_type, u8 req_kw_type) 1061 { 1062 int kw = 0, bank; 1063 1064 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) 1065 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1066 index, intf, 1067 entry, bank, kw_type, 1068 kw, req_kw_type); 1069 } 1070 1071 int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, 1072 u8 intf, struct cn20k_mcam_entry *entry, 1073 bool enable, u8 hw_prio, u8 req_kw_type) 1074 { 1075 struct npc_mcam *mcam = &rvu->hw->mcam; 1076 int mcam_idx = index % mcam->banksize; 1077 int bank = index / mcam->banksize; 1078 u64 bank_cfg = (u64)hw_prio << 24; 1079 int kw = 0; 1080 u8 kw_type; 1081 1082 if (index < 0 || index >= mcam->total_entries) 1083 return -EINVAL; 1084 1085 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1086 return -EINVAL; 1087 1088 /* Disable before mcam entry update */ 1089 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false)) 1090 return -EINVAL; 1091 1092 /* CAM1 takes the comparison value and 1093 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 1094 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 1095 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 1096 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 1097 */ 1098 if (kw_type == NPC_MCAM_KEY_X2) { 1099 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1100 npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx); 1101 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1102 mcam_idx, intf, entry, 1103 bank, kw_type, kw, req_kw_type); 1104 /* Set 'action' */ 1105 rvu_write64(rvu, blkaddr, 1106 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1107 bank, 0), 1108 entry->action); 1109 1110 /* Set 'action2' for inline receive */ 1111 rvu_write64(rvu, blkaddr, 1112 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1113 bank, 2), 1114 entry->action2); 1115 1116 /* Set TAG 'action' */ 1117 rvu_write64(rvu, blkaddr, 1118 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1119 bank, 1), 1120 entry->vtag_action); 1121 1122 /* Set HW priority */ 1123 rvu_write64(rvu, blkaddr, 1124 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1125 bank_cfg); 1126 1127 } else { 1128 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1129 npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx); 1130 npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx); 1131 1132 npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, 1133 mcam_idx, intf, entry, 1134 kw_type, req_kw_type); 1135 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1136 /* Set 'action' */ 1137 rvu_write64(rvu, blkaddr, 1138 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1139 bank, 0), 1140 entry->action); 1141 1142 /* Set TAG 'action' */ 1143 rvu_write64(rvu, blkaddr, 1144 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1145 bank, 1), 1146 entry->vtag_action); 1147 1148 /* Set 'action2' for inline receive */ 1149 rvu_write64(rvu, blkaddr, 1150 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1151 bank, 2), 1152 entry->action2); 1153 1154 /* Set HW priority */ 1155 rvu_write64(rvu, blkaddr, 1156 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1157 bank_cfg); 1158 } 1159 } 1160 1161 /* TODO: */ 1162 /* PF installing VF rule */ 1163 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 1164 return -EINVAL; 1165 1166 return 0; 1167 } 1168 1169 int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) 1170 { 1171 struct npc_mcam *mcam = &rvu->hw->mcam; 1172 u64 cfg, sreg, dreg, soff, doff; 1173 u8 src_kwtype, dest_kwtype; 1174 int bank, i, sb, db; 1175 int dbank, sbank; 1176 1177 if (src >= mcam->total_entries || dest >= mcam->total_entries) 1178 return -EINVAL; 1179 1180 dbank = npc_get_bank(mcam, dest); 1181 sbank = npc_get_bank(mcam, src); 1182 1183 if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype)) 1184 return -EINVAL; 1185 1186 if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype)) 1187 return -EINVAL; 1188 1189 if (src_kwtype != dest_kwtype) 1190 return -EINVAL; 1191 1192 src &= (mcam->banksize - 1); 1193 dest &= (mcam->banksize - 1); 1194 1195 /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ 1196 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1197 sb = sbank + bank; 1198 sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); 1199 db = dbank + bank; 1200 dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); 1201 for (i = 0; i < 10; i++) { 1202 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 1203 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 1204 } 1205 1206 /* Copy action */ 1207 for (i = 0; i < 3; i++) { 1208 soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, 1209 sb, i); 1210 cfg = rvu_read64(rvu, blkaddr, soff); 1211 1212 doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, 1213 i); 1214 rvu_write64(rvu, blkaddr, doff, cfg); 1215 } 1216 1217 /* Copy bank configuration */ 1218 cfg = rvu_read64(rvu, blkaddr, 1219 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); 1220 rvu_write64(rvu, blkaddr, 1221 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); 1222 if (src_kwtype == NPC_MCAM_KEY_X2) 1223 break; 1224 } 1225 1226 return 0; 1227 } 1228 1229 static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, 1230 u64 cam0, u64 cam1) 1231 { 1232 entry->kw[idx] = cam1; 1233 entry->kw_mask[idx] = cam1 ^ cam0; 1234 } 1235 1236 int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, 1237 struct cn20k_mcam_entry *entry, 1238 u8 *intf, u8 *ena, u8 *hw_prio) 1239 { 1240 struct npc_mcam *mcam = &rvu->hw->mcam; 1241 u64 cam0, cam1, bank_cfg, cfg; 1242 int kw = 0, bank; 1243 u8 kw_type; 1244 1245 if (index >= mcam->total_entries) 1246 return -EINVAL; 1247 1248 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1249 return -EINVAL; 1250 1251 bank = npc_get_bank(mcam, index); 1252 index &= (mcam->banksize - 1); 1253 1254 cfg = rvu_read64(rvu, blkaddr, 1255 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0)); 1256 entry->action = cfg; 1257 1258 cfg = rvu_read64(rvu, blkaddr, 1259 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1)); 1260 entry->vtag_action = cfg; 1261 1262 cfg = rvu_read64(rvu, blkaddr, 1263 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2)); 1264 entry->action2 = cfg; 1265 1266 cfg = rvu_read64(rvu, blkaddr, 1267 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, 1268 bank, 1)) & 3; 1269 *intf = cfg; 1270 1271 bank_cfg = rvu_read64(rvu, blkaddr, 1272 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); 1273 *ena = bank_cfg & 0x1; 1274 *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; 1275 if (kw_type == NPC_MCAM_KEY_X2) { 1276 cam1 = rvu_read64(rvu, blkaddr, 1277 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1278 bank, 1279 1)); 1280 cam0 = rvu_read64(rvu, blkaddr, 1281 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1282 bank, 1283 0)); 1284 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1285 1286 cam1 = rvu_read64(rvu, blkaddr, 1287 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1288 bank, 1289 1)); 1290 cam0 = rvu_read64(rvu, blkaddr, 1291 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1292 bank, 1293 0)); 1294 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1295 1296 cam1 = rvu_read64(rvu, blkaddr, 1297 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1298 bank, 1299 1)); 1300 cam0 = rvu_read64(rvu, blkaddr, 1301 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1302 bank, 1303 0)); 1304 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1305 1306 cam1 = rvu_read64(rvu, blkaddr, 1307 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1308 bank, 1309 1)); 1310 cam0 = rvu_read64(rvu, blkaddr, 1311 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1312 bank, 1313 0)); 1314 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1315 return 0; 1316 } 1317 1318 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { 1319 cam1 = rvu_read64(rvu, blkaddr, 1320 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1321 bank, 1322 1)); 1323 cam0 = rvu_read64(rvu, blkaddr, 1324 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1325 bank, 1326 0)); 1327 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1328 1329 cam1 = rvu_read64(rvu, blkaddr, 1330 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1331 bank, 1332 1)); 1333 cam0 = rvu_read64(rvu, blkaddr, 1334 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1335 bank, 1336 0)); 1337 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1338 1339 cam1 = rvu_read64(rvu, blkaddr, 1340 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1341 bank, 1342 1)); 1343 cam0 = rvu_read64(rvu, blkaddr, 1344 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1345 bank, 1346 0)); 1347 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1348 1349 cam1 = rvu_read64(rvu, blkaddr, 1350 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1351 bank, 1352 1)); 1353 cam0 = rvu_read64(rvu, blkaddr, 1354 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1355 bank, 1356 0)); 1357 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1358 } 1359 1360 return 0; 1361 } 1362 1363 int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, 1364 struct npc_cn20k_mcam_write_entry_req *req, 1365 struct msg_rsp *rsp) 1366 { 1367 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1368 struct npc_mcam *mcam = &rvu->hw->mcam; 1369 u16 pcifunc = req->hdr.pcifunc; 1370 int blkaddr, rc; 1371 u8 nix_intf; 1372 1373 req->entry = npc_cn20k_vidx2idx(req->entry); 1374 1375 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1376 if (blkaddr < 0) 1377 return NPC_MCAM_INVALID_REQ; 1378 1379 mutex_lock(&mcam->lock); 1380 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1381 if (rc) 1382 goto exit; 1383 1384 if (!is_npc_interface_valid(rvu, req->intf)) { 1385 rc = NPC_MCAM_INVALID_REQ; 1386 goto exit; 1387 } 1388 1389 if (is_npc_intf_tx(req->intf)) 1390 nix_intf = pfvf->nix_tx_intf; 1391 else 1392 nix_intf = pfvf->nix_rx_intf; 1393 1394 /* For AF installed rules, the nix_intf should be set to target NIX */ 1395 if (is_pffunc_af(req->hdr.pcifunc)) 1396 nix_intf = req->intf; 1397 1398 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, 1399 &req->entry_data, req->enable_entry, 1400 req->hw_prio, req->req_kw_type); 1401 1402 exit: 1403 mutex_unlock(&mcam->lock); 1404 return rc; 1405 } 1406 1407 int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, 1408 struct npc_mcam_read_entry_req *req, 1409 struct npc_cn20k_mcam_read_entry_rsp *rsp) 1410 { 1411 struct npc_mcam *mcam = &rvu->hw->mcam; 1412 u16 pcifunc = req->hdr.pcifunc; 1413 int blkaddr, rc; 1414 1415 req->entry = npc_cn20k_vidx2idx(req->entry); 1416 1417 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1418 if (blkaddr < 0) 1419 return NPC_MCAM_INVALID_REQ; 1420 1421 mutex_lock(&mcam->lock); 1422 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1423 if (rc) 1424 goto fail; 1425 1426 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, 1427 &rsp->entry_data, &rsp->intf, 1428 &rsp->enable, &rsp->hw_prio); 1429 fail: 1430 mutex_unlock(&mcam->lock); 1431 return rc; 1432 } 1433 1434 int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, 1435 struct npc_cn20k_mcam_alloc_and_write_entry_req *req, 1436 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 1437 { 1438 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1439 struct npc_mcam_free_entry_req free_req = { 0 }; 1440 struct npc_mcam_alloc_entry_req entry_req; 1441 struct npc_mcam_alloc_entry_rsp entry_rsp; 1442 struct npc_mcam *mcam = &rvu->hw->mcam; 1443 u16 entry = NPC_MCAM_ENTRY_INVALID; 1444 struct msg_rsp free_rsp; 1445 int blkaddr, rc, err; 1446 u8 nix_intf; 1447 1448 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1449 if (blkaddr < 0) 1450 return NPC_MCAM_INVALID_REQ; 1451 1452 if (!is_npc_interface_valid(rvu, req->intf)) 1453 return NPC_MCAM_INVALID_REQ; 1454 1455 /* Try to allocate a MCAM entry */ 1456 entry_req.hdr.pcifunc = req->hdr.pcifunc; 1457 entry_req.contig = true; 1458 entry_req.ref_prio = req->ref_prio; 1459 entry_req.ref_entry = req->ref_entry; 1460 entry_req.count = 1; 1461 entry_req.virt = req->virt; 1462 1463 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 1464 &entry_req, &entry_rsp); 1465 if (rc) 1466 return rc; 1467 1468 if (!entry_rsp.count) 1469 return NPC_MCAM_ALLOC_FAILED; 1470 1471 /* entry_req.count is 1, so single entry is allocated */ 1472 entry = npc_cn20k_vidx2idx(entry_rsp.entry); 1473 1474 mutex_lock(&mcam->lock); 1475 1476 if (is_npc_intf_tx(req->intf)) 1477 nix_intf = pfvf->nix_tx_intf; 1478 else 1479 nix_intf = pfvf->nix_rx_intf; 1480 1481 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, 1482 &req->entry_data, req->enable_entry, 1483 req->hw_prio, req->req_kw_type); 1484 1485 mutex_unlock(&mcam->lock); 1486 1487 if (rc) { 1488 free_req.hdr.pcifunc = req->hdr.pcifunc; 1489 free_req.entry = entry_rsp.entry; 1490 err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 1491 if (err) 1492 dev_err(rvu->dev, 1493 "%s: Error to free mcam idx %u\n", 1494 __func__, entry_rsp.entry); 1495 return rc; 1496 } 1497 1498 rsp->entry = entry_rsp.entry; 1499 return 0; 1500 } 1501 1502 static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) 1503 { 1504 if (is_lbk_vf(rvu, pcifunc)) 1505 return NIXLF_PROMISC_ENTRY; 1506 1507 return NIXLF_UCAST_ENTRY; 1508 } 1509 1510 int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, 1511 struct msg_req *req, 1512 struct npc_cn20k_mcam_read_base_rule_rsp *rsp) 1513 { 1514 struct npc_mcam *mcam = &rvu->hw->mcam; 1515 int index, blkaddr, nixlf, rc = 0; 1516 u16 pcifunc = req->hdr.pcifunc; 1517 u8 intf, enable, hw_prio; 1518 struct rvu_pfvf *pfvf; 1519 int rl_type; 1520 1521 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1522 if (blkaddr < 0) 1523 return NPC_MCAM_INVALID_REQ; 1524 1525 /* Return the channel number in case of PF */ 1526 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 1527 pfvf = rvu_get_pfvf(rvu, pcifunc); 1528 rsp->entry.kw[0] = pfvf->rx_chan_base; 1529 rsp->entry.kw_mask[0] = 0xFFFULL; 1530 goto out; 1531 } 1532 1533 /* Find the pkt steering rule installed by PF to this VF */ 1534 mutex_lock(&mcam->lock); 1535 for (index = 0; index < mcam->bmap_entries; index++) { 1536 if (mcam->entry2target_pffunc[index] == pcifunc) 1537 goto read_entry; 1538 } 1539 1540 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 1541 if (rc < 0) { 1542 mutex_unlock(&mcam->lock); 1543 goto out; 1544 } 1545 1546 rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); 1547 1548 /* Read the default ucast entry if there is no pkt steering rule */ 1549 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); 1550 if (index < 0) { 1551 mutex_unlock(&mcam->lock); 1552 goto out; 1553 } 1554 1555 read_entry: 1556 /* Read the mcam entry */ 1557 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index, 1558 &rsp->entry, &intf, 1559 &enable, &hw_prio); 1560 mutex_unlock(&mcam->lock); 1561 out: 1562 return rc; 1563 } 1564 1565 static u8 npc_map2cn20k_flag(u8 flag) 1566 { 1567 switch (flag) { 1568 case NPC_F_LC_U_IP_FRAG: 1569 return NPC_CN20K_F_LC_L_IP_FRAG; 1570 1571 case NPC_F_LC_U_IP6_FRAG: 1572 return NPC_CN20K_F_LC_L_IP6_FRAG; 1573 1574 case NPC_F_LC_L_6TO4: 1575 return NPC_CN20K_F_LC_L_6TO4; 1576 1577 case NPC_F_LC_L_MPLS_IN_IP: 1578 return NPC_CN20K_F_LC_U_MPLS_IN_IP; 1579 1580 case NPC_F_LC_L_IP6_TUN_IP6: 1581 return NPC_CN20K_F_LC_U_IP6_TUN_IP6; 1582 1583 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1584 return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; 1585 1586 default: 1587 break; 1588 } 1589 1590 WARN(1, "%s: Invalid flag=%u\n", __func__, flag); 1591 return 0xff; 1592 } 1593 1594 static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act) 1595 { 1596 u8 ltype, val; 1597 1598 if (act->lid != NPC_LID_LC) 1599 return; 1600 1601 ltype = act->ltype; 1602 if (ltype != NPC_LT_LC_IP && 1603 ltype != NPC_LT_LC_IP6 && 1604 ltype != NPC_LT_LC_IP_OPT && 1605 ltype != NPC_LT_LC_IP6_EXT) 1606 return; 1607 1608 switch (act->flags) { 1609 case NPC_F_LC_U_IP_FRAG: 1610 case NPC_F_LC_U_IP6_FRAG: 1611 case NPC_F_LC_L_6TO4: 1612 case NPC_F_LC_L_MPLS_IN_IP: 1613 case NPC_F_LC_L_IP6_TUN_IP6: 1614 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1615 val = npc_map2cn20k_flag(act->flags); 1616 if (val != 0xFF) 1617 act->flags = val; 1618 break; 1619 default: 1620 break; 1621 } 1622 } 1623 1624 void 1625 npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, 1626 struct npc_kpu_profile_adapter *pfl) 1627 { 1628 struct npc_kpu_profile_action *action; 1629 int entries; 1630 1631 for (int i = 0; i < pfl->kpus; i++) { 1632 action = pfl->kpu[i].action; 1633 entries = pfl->kpu[i].action_entries; 1634 1635 for (int j = 0; j < entries; j++) 1636 npc_cn20k_translate_action_flags(&action[j]); 1637 } 1638 } 1639 1640 int npc_cn20k_apply_custom_kpu(struct rvu *rvu, 1641 struct npc_kpu_profile_adapter *profile) 1642 { 1643 const struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1644 size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1645 struct npc_kpu_profile_action *action; 1646 struct npc_kpu_profile_cam *cam; 1647 struct npc_kpu_fwdata *fw_kpu; 1648 size_t offset = 0; 1649 u16 kpu, entry; 1650 int entries; 1651 1652 hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1653 1654 if (rvu->kpu_fwdata_sz < hdr_sz) { 1655 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1656 return -EINVAL; 1657 } 1658 1659 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1660 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1661 fw->signature); 1662 return -EINVAL; 1663 } 1664 1665 /* Verify if the using known profile structure */ 1666 if (NPC_KPU_VER_MAJ(profile->version) > 1667 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1668 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1669 NPC_KPU_VER_MAJ(profile->version), 1670 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1671 return -EINVAL; 1672 } 1673 1674 /* Verify if profile is aligned with the required kernel changes */ 1675 if (NPC_KPU_VER_MIN(profile->version) < 1676 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1677 dev_warn(rvu->dev, 1678 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1679 NPC_KPU_VER_MAJ(profile->version), 1680 NPC_KPU_VER_MIN(profile->version), 1681 NPC_KPU_VER_PATCH(profile->version), 1682 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1683 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1684 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1685 return -EINVAL; 1686 } 1687 1688 /* Verify if profile fits the HW */ 1689 if (fw->kpus > rvu->hw->npc_kpus) { 1690 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus, 1691 rvu->hw->npc_kpus); 1692 return -EINVAL; 1693 } 1694 1695 /* Check if there is enough memory */ 1696 if (fw->kpus > profile->kpus) { 1697 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus, 1698 profile->kpus); 1699 return -EINVAL; 1700 } 1701 1702 profile->mcam_kex_prfl.mkex_extr = &fw->mkex; 1703 if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { 1704 dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", 1705 profile->mcam_kex_prfl.mkex_extr->mkex_sign); 1706 return -EINVAL; 1707 } 1708 1709 profile->custom = 1; 1710 profile->name = fw->name; 1711 profile->version = le64_to_cpu(fw->version); 1712 profile->lt_def = &fw->lt_def; 1713 1714 for (kpu = 0; kpu < fw->kpus; kpu++) { 1715 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1716 if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) 1717 dev_warn(rvu->dev, 1718 "Too many custom entries on KPU%d: %d > %d\n", 1719 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1720 entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1721 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1722 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1723 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1724 offset += fw_kpu->entries * sizeof(*action); 1725 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1726 dev_warn(rvu->dev, 1727 "Profile size mismatch on KPU%i parsing.\n", 1728 kpu + 1); 1729 return -EINVAL; 1730 } 1731 1732 for (entry = 0; entry < entries; entry++) { 1733 profile->kpu[kpu].cam[entry] = cam[entry]; 1734 profile->kpu[kpu].action[entry] = action[entry]; 1735 npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]); 1736 } 1737 } 1738 1739 return 0; 1740 } 1741 1742 int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) 1743 { 1744 struct npc_subbank *sb; 1745 int bank_off, sb_id; 1746 1747 /* mcam_idx should be less than (2 * bank depth) */ 1748 if (mcam_idx >= npc_priv->bank_depth * 2) { 1749 dev_err(rvu->dev, "%s: bad params\n", 1750 __func__); 1751 return -EINVAL; 1752 } 1753 1754 /* find mcam offset per bank */ 1755 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1756 1757 /* Find subbank id */ 1758 sb_id = bank_off / npc_priv->subbank_depth; 1759 1760 /* Check if subbank id is more than maximum 1761 * number of subbanks available 1762 */ 1763 if (sb_id >= npc_priv->num_subbanks) { 1764 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1765 __func__, sb_id); 1766 return -EINVAL; 1767 } 1768 1769 sb = &npc_priv->sb[sb_id]; 1770 1771 *key_type = sb->key_type; 1772 1773 return 0; 1774 } 1775 1776 static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, 1777 u16 sub_off, u16 *mcam_idx) 1778 { 1779 int off, bot; 1780 1781 /* for x4 section, maximum allowed subbank index = 1782 * subsection depth - 1 1783 */ 1784 if (sb->key_type == NPC_MCAM_KEY_X4 && 1785 sub_off >= npc_priv->subbank_depth) { 1786 dev_err(rvu->dev, 1787 "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", 1788 __func__, sb->idx, sub_off); 1789 return -EINVAL; 1790 } 1791 1792 /* for x2 section, maximum allowed subbank index = 1793 * 2 * subsection depth - 1 1794 */ 1795 if (sb->key_type == NPC_MCAM_KEY_X2 && 1796 sub_off >= npc_priv->subbank_depth * 2) { 1797 dev_err(rvu->dev, 1798 "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", 1799 __func__, sb->idx, sub_off); 1800 return -EINVAL; 1801 } 1802 1803 /* Find subbank offset from respective subbank (w.r.t bank) */ 1804 off = sub_off & (npc_priv->subbank_depth - 1); 1805 1806 /* if subsection idx is in bank1, add bank depth, 1807 * which is part of sb->b1b 1808 */ 1809 bot = sub_off >= npc_priv->subbank_depth ? sb->b1b : sb->b0b; 1810 1811 *mcam_idx = bot + off; 1812 return 0; 1813 } 1814 1815 int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, 1816 struct npc_subbank **sb, 1817 int *sb_off) 1818 { 1819 int bank_off, sb_id; 1820 1821 /* mcam_idx should be less than (2 * bank depth) */ 1822 if (mcam_idx >= npc_priv->bank_depth * 2) { 1823 dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", 1824 __func__, mcam_idx); 1825 return -EINVAL; 1826 } 1827 1828 /* find mcam offset per bank */ 1829 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1830 1831 /* Find subbank id */ 1832 sb_id = bank_off / npc_priv->subbank_depth; 1833 1834 /* Check if subbank id is more than maximum 1835 * number of subbanks available 1836 */ 1837 if (sb_id >= npc_priv->num_subbanks) { 1838 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1839 __func__, sb_id); 1840 return -EINVAL; 1841 } 1842 1843 *sb = &npc_priv->sb[sb_id]; 1844 1845 /* Subbank offset per bank */ 1846 *sb_off = bank_off % npc_priv->subbank_depth; 1847 1848 /* Index in a subbank should add subbank depth 1849 * if it is in bank1 1850 */ 1851 if (mcam_idx >= npc_priv->bank_depth) 1852 *sb_off += npc_priv->subbank_depth; 1853 1854 return 0; 1855 } 1856 1857 static int __npc_subbank_contig_alloc(struct rvu *rvu, 1858 struct npc_subbank *sb, 1859 int key_type, int sidx, 1860 int eidx, int prio, 1861 int count, int t, int b, 1862 unsigned long *bmap, 1863 u16 *save) 1864 { 1865 int k, offset, delta = 0; 1866 int cnt = 0, sbd; 1867 1868 sbd = npc_priv->subbank_depth; 1869 1870 if (sidx >= npc_priv->bank_depth) 1871 delta = sbd; 1872 1873 switch (prio) { 1874 case NPC_MCAM_LOWER_PRIO: 1875 case NPC_MCAM_ANY_PRIO: 1876 /* Find an area of size 'count' from sidx to eidx */ 1877 offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, 1878 count, 0); 1879 1880 if (offset >= sbd) { 1881 dev_err(rvu->dev, 1882 "%s: Could not find contiguous(%d) entries\n", 1883 __func__, count); 1884 return -EFAULT; 1885 } 1886 1887 dev_dbg(rvu->dev, 1888 "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", 1889 __func__, sidx, eidx, t, b, offset, 1890 count, delta); 1891 1892 for (cnt = 0; cnt < count; cnt++) 1893 save[cnt] = offset + cnt + delta; 1894 1895 break; 1896 1897 case NPC_MCAM_HIGHER_PRIO: 1898 /* Find an area of 'count' from eidx to sidx */ 1899 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1900 /* If an intermediate slot is not free, 1901 * reset the counter (cnt) to zero as 1902 * request is for contiguous. 1903 */ 1904 if (test_bit(k, bmap)) { 1905 cnt = 0; 1906 continue; 1907 } 1908 1909 save[cnt++] = k + delta; 1910 } 1911 break; 1912 } 1913 1914 /* Found 'count' number of free slots */ 1915 if (cnt == count) 1916 return 0; 1917 1918 dev_dbg(rvu->dev, 1919 "%s: Could not find contiguous(%d) entries in subbank=%u\n", 1920 __func__, count, sb->idx); 1921 return -EFAULT; 1922 } 1923 1924 static int __npc_subbank_non_contig_alloc(struct rvu *rvu, 1925 struct npc_subbank *sb, 1926 int key_type, int sidx, 1927 int eidx, int prio, 1928 int t, int b, 1929 unsigned long *bmap, 1930 int count, u16 *save, 1931 bool max_alloc, int *alloc_cnt) 1932 { 1933 unsigned long index; 1934 int cnt = 0, delta; 1935 int k, sbd; 1936 1937 sbd = npc_priv->subbank_depth; 1938 delta = sidx >= npc_priv->bank_depth ? sbd : 0; 1939 1940 switch (prio) { 1941 /* Find an area of size 'count' from sidx to eidx */ 1942 case NPC_MCAM_LOWER_PRIO: 1943 case NPC_MCAM_ANY_PRIO: 1944 index = find_next_zero_bit(bmap, sbd, sidx - b); 1945 if (index >= sbd) { 1946 dev_err(rvu->dev, 1947 "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", 1948 __func__, count, 1949 bitmap_weight(bmap, sbd), 1950 sb->idx); 1951 break; 1952 } 1953 1954 for (k = index; cnt < count && k <= (eidx - b); k++) { 1955 /* Skip used slots */ 1956 if (test_bit(k, bmap)) 1957 continue; 1958 1959 save[cnt++] = k + delta; 1960 } 1961 break; 1962 1963 /* Find an area of 'count' from eidx to sidx */ 1964 case NPC_MCAM_HIGHER_PRIO: 1965 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1966 /* Skip used slots */ 1967 if (test_bit(k, bmap)) 1968 continue; 1969 1970 save[cnt++] = k + delta; 1971 } 1972 break; 1973 } 1974 1975 /* Update allocated 'cnt' to alloc_cnt */ 1976 *alloc_cnt = cnt; 1977 1978 /* Successfully allocated requested count slots */ 1979 if (cnt == count) 1980 return 0; 1981 1982 /* Allocation successful for cnt < count */ 1983 if (max_alloc && cnt > 0) 1984 return 0; 1985 1986 dev_dbg(rvu->dev, 1987 "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", 1988 __func__, count, sb->idx, cnt, max_alloc); 1989 1990 return -EFAULT; 1991 } 1992 1993 static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, 1994 int sb_off, unsigned long **bmap, 1995 int *off) 1996 { 1997 int sbd; 1998 1999 sbd = npc_priv->subbank_depth; 2000 2001 *off = sb_off & (sbd - 1); 2002 *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; 2003 } 2004 2005 /* set/clear bitmap */ 2006 static bool __npc_subbank_mark_slot(struct rvu *rvu, 2007 struct npc_subbank *sb, 2008 int sb_off, bool set) 2009 { 2010 unsigned long *bmap; 2011 int off; 2012 2013 /* if sb_off >= subbank.depth, then slots are in 2014 * bank1 2015 */ 2016 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2017 2018 dev_dbg(rvu->dev, 2019 "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", 2020 __func__, set, sb_off, sb->idx, off); 2021 2022 if (set) { 2023 /* Slot is already used */ 2024 if (test_bit(off, bmap)) 2025 return false; 2026 2027 sb->free_cnt--; 2028 set_bit(off, bmap); 2029 return true; 2030 } 2031 2032 /* Slot is already free */ 2033 if (!test_bit(off, bmap)) 2034 return false; 2035 2036 sb->free_cnt++; 2037 clear_bit(off, bmap); 2038 return true; 2039 } 2040 2041 static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) 2042 { 2043 int rc, blkaddr; 2044 2045 sb->flags = NPC_SUBBANK_FLAG_FREE; 2046 sb->key_type = 0; 2047 2048 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2049 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2050 2051 if (!xa_erase(&npc_priv->xa_sb_used, sb->arr_idx)) { 2052 dev_err(rvu->dev, 2053 "%s: Error to delete from xa_sb_used array\n", 2054 __func__); 2055 return -EFAULT; 2056 } 2057 2058 rc = xa_insert(&npc_priv->xa_sb_free, sb->arr_idx, 2059 xa_mk_value(sb->idx), GFP_KERNEL); 2060 if (rc) { 2061 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2062 xa_mk_value(sb->idx), GFP_KERNEL); 2063 if (rc) 2064 dev_err(rvu->dev, 2065 "%s: Failed to roll back sb(%u) arr_idx=%d\n", 2066 __func__, sb->idx, sb->arr_idx); 2067 2068 dev_err(rvu->dev, 2069 "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", 2070 __func__, sb->idx, sb->arr_idx); 2071 return rc; 2072 } 2073 2074 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2075 rvu_write64(rvu, blkaddr, 2076 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2077 NPC_MCAM_KEY_X2); 2078 2079 return rc; 2080 } 2081 2082 static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, 2083 int key_type) 2084 { 2085 int rc; 2086 2087 sb->flags = NPC_SUBBANK_FLAG_USED; 2088 sb->key_type = key_type; 2089 if (key_type == NPC_MCAM_KEY_X4) 2090 sb->free_cnt = npc_priv->subbank_depth; 2091 else 2092 sb->free_cnt = 2 * npc_priv->subbank_depth; 2093 2094 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2095 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2096 2097 if (!xa_erase(&npc_priv->xa_sb_free, sb->arr_idx)) { 2098 dev_err(rvu->dev, 2099 "%s: Error to delete from xa_sb_free array\n", 2100 __func__); 2101 return -EFAULT; 2102 } 2103 2104 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2105 xa_mk_value(sb->idx), GFP_KERNEL); 2106 if (rc) 2107 dev_err(rvu->dev, 2108 "%s: Error to add to xa_sb_used array\n", __func__); 2109 2110 return rc; 2111 } 2112 2113 static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, 2114 u16 sb_off) 2115 { 2116 bool deleted = false; 2117 unsigned long *bmap; 2118 int rc, off; 2119 2120 deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); 2121 if (!deleted) 2122 goto done; 2123 2124 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2125 2126 /* Check whether we can mark whole subbank as free */ 2127 if (sb->key_type == NPC_MCAM_KEY_X4) { 2128 if (sb->free_cnt < npc_priv->subbank_depth) 2129 goto done; 2130 } else { 2131 if (sb->free_cnt < 2 * npc_priv->subbank_depth) 2132 goto done; 2133 } 2134 2135 /* All slots in subbank are unused. Mark the subbank as free 2136 * and add to free pool 2137 */ 2138 rc = __npc_subbank_mark_free(rvu, sb); 2139 if (rc) 2140 dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); 2141 2142 done: 2143 return deleted; 2144 } 2145 2146 static int 2147 npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) 2148 { 2149 bool deleted; 2150 2151 mutex_lock(&sb->lock); 2152 deleted = __npc_subbank_free(rvu, sb, sb_off); 2153 mutex_unlock(&sb->lock); 2154 2155 return deleted ? 0 : -EFAULT; 2156 } 2157 2158 static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2159 int key_type, int ref, int limit, int prio, 2160 bool contig, int count, u16 *mcam_idx, 2161 int idx_sz, bool max_alloc, int *alloc_cnt) 2162 { 2163 int cnt, t, b, i, blkaddr; 2164 bool new_sub_bank = false; 2165 unsigned long *bmap; 2166 u16 *save = NULL; 2167 int sidx, eidx; 2168 bool diffbank; 2169 int bw, bfree; 2170 int rc = 0; 2171 bool ret; 2172 2173 /* Check if enough space is there to return requested number of 2174 * mcam indexes in case of contiguous allocation 2175 */ 2176 if (!max_alloc && count > idx_sz) { 2177 dev_err(rvu->dev, 2178 "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", 2179 __func__, count, idx_sz, sb->idx); 2180 return -ENOSPC; 2181 } 2182 2183 /* Allocation on multiple subbank is not supported by this function. 2184 * it means that ref and limit should be on same subbank. 2185 * 2186 * ref and limit values should be validated w.r.t prio as below. 2187 * say ref = 100, limit = 200, 2188 * if NPC_MCAM_LOWER_PRIO, allocate index 100 2189 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. 2190 * if NPC_MCAM_ANY_PRIO, allocate index 100 2191 * 2192 * say ref = 200, limit = 100 2193 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. 2194 * if NPC_MCAM_HIGHER_PRIO, allocate index 200 2195 * if NPC_MCAM_ANY_PRIO, allocate index 100 2196 * 2197 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction 2198 * on "ref" and "limit" values. ie, ref > limit and limit > ref 2199 * are valid cases. 2200 */ 2201 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 2202 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 2203 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 2204 __func__, ref, limit); 2205 return -EINVAL; 2206 } 2207 2208 /* x4 indexes are from 0 to bank size as it combines two x2 banks */ 2209 if (key_type == NPC_MCAM_KEY_X4 && 2210 (ref >= npc_priv->bank_depth || limit >= npc_priv->bank_depth)) { 2211 dev_err(rvu->dev, 2212 "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", 2213 __func__, ref, limit); 2214 return -EINVAL; 2215 } 2216 2217 /* This function is called either bank0 or bank1 portion of a subbank. 2218 * so ref and limit should be on same bank. 2219 */ 2220 diffbank = !!((ref & npc_priv->bank_depth) ^ 2221 (limit & npc_priv->bank_depth)); 2222 if (diffbank) { 2223 dev_err(rvu->dev, 2224 "%s: request ref and limit should be from same bank\n", 2225 __func__); 2226 return -EINVAL; 2227 } 2228 2229 sidx = min_t(int, limit, ref); 2230 eidx = max_t(int, limit, ref); 2231 2232 /* Find total number of slots available; both used and free */ 2233 cnt = eidx - sidx + 1; 2234 if (contig && cnt < count) { 2235 dev_err(rvu->dev, 2236 "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", 2237 __func__, ref, limit, count); 2238 return -EINVAL; 2239 } 2240 2241 /* If subbank is free, check if requested number of indexes is less than 2242 * or equal to mcam entries available in the subbank if contig. 2243 */ 2244 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2245 if (contig && count > npc_priv->subbank_depth) { 2246 dev_err(rvu->dev, "%s: Less number of entries\n", 2247 __func__); 2248 return -ENOSPC; 2249 } 2250 2251 new_sub_bank = true; 2252 goto process; 2253 } 2254 2255 /* Flag should be set for all used subbanks */ 2256 WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), 2257 "Used flag is not set(%#x)\n", sb->flags); 2258 2259 /* If subbank key type does not match with requested key_type, 2260 * return error 2261 */ 2262 if (sb->key_type != key_type) { 2263 dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); 2264 return -EINVAL; 2265 } 2266 2267 process: 2268 /* if ref or limit >= npc_priv->bank_depth, index are in bank1. 2269 * else bank0. 2270 */ 2271 if (ref >= npc_priv->bank_depth) { 2272 bmap = sb->b1map; 2273 t = sb->b1t; 2274 b = sb->b1b; 2275 } else { 2276 bmap = sb->b0map; 2277 t = sb->b0t; 2278 b = sb->b0b; 2279 } 2280 2281 /* Calculate free slots */ 2282 bw = bitmap_weight(bmap, npc_priv->subbank_depth); 2283 bfree = npc_priv->subbank_depth - bw; 2284 2285 if (!bfree) { 2286 dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); 2287 return -ENOSPC; 2288 } 2289 2290 /* If request is for contiguous , then max we can allocate is 2291 * equal to subbank_depth 2292 */ 2293 if (contig && bfree < count) { 2294 dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); 2295 return -ENOSPC; 2296 } 2297 2298 /* 'save' array stores available indexes temporarily before 2299 * marking it as allocated 2300 */ 2301 save = kcalloc(count, sizeof(u16), GFP_KERNEL); 2302 if (!save) { 2303 rc = -ENOMEM; 2304 goto err1; 2305 } 2306 2307 if (contig) { 2308 rc = __npc_subbank_contig_alloc(rvu, sb, key_type, 2309 sidx, eidx, prio, 2310 count, t, b, 2311 bmap, save); 2312 /* contiguous allocation success means that 2313 * requested number of free slots got 2314 * allocated 2315 */ 2316 if (!rc) 2317 *alloc_cnt = count; 2318 2319 } else { 2320 rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, 2321 sidx, eidx, prio, 2322 t, b, bmap, 2323 count, save, 2324 max_alloc, alloc_cnt); 2325 } 2326 2327 if (rc) 2328 goto err1; 2329 2330 /* Mark new subbank bank as used */ 2331 if (new_sub_bank) { 2332 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2333 if (blkaddr < 0) { 2334 dev_err(rvu->dev, 2335 "%s: NPC block not implemented\n", __func__); 2336 rc = -EFAULT; 2337 goto err1; 2338 } 2339 2340 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2341 if (rc) { 2342 dev_err(rvu->dev, 2343 "%s: Error to mark subbank as used\n", 2344 __func__); 2345 goto err2; 2346 } 2347 2348 /* Configure section type to key_type */ 2349 rvu_write64(rvu, blkaddr, 2350 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2351 key_type); 2352 } 2353 2354 for (i = 0; i < *alloc_cnt; i++) { 2355 rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], 2356 &mcam_idx[i]); 2357 if (rc) { 2358 dev_err(rvu->dev, 2359 "%s: Error to find mcam idx for %u\n", 2360 __func__, save[i]); 2361 /* TODO: handle err case gracefully */ 2362 goto err3; 2363 } 2364 2365 /* Mark all slots as used */ 2366 ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); 2367 if (!ret) { 2368 dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", 2369 __func__, mcam_idx[i]); 2370 rc = -EFAULT; 2371 goto err3; 2372 } 2373 } 2374 kfree(save); 2375 return 0; 2376 2377 err3: 2378 for (int j = 0; j < i; j++) 2379 __npc_subbank_mark_slot(rvu, sb, save[j], false); 2380 err2: 2381 if (new_sub_bank) 2382 __npc_subbank_mark_free(rvu, sb); 2383 err1: 2384 kfree(save); 2385 *alloc_cnt = 0; 2386 return rc; 2387 } 2388 2389 static int 2390 npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2391 int key_type, int ref, int limit, int prio, 2392 bool contig, int count, u16 *mcam_idx, 2393 int idx_sz, bool max_alloc, int *alloc_cnt) 2394 { 2395 int rc; 2396 2397 mutex_lock(&sb->lock); 2398 rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, 2399 contig, count, mcam_idx, idx_sz, 2400 max_alloc, alloc_cnt); 2401 mutex_unlock(&sb->lock); 2402 2403 return rc; 2404 } 2405 2406 static int 2407 npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) 2408 { 2409 int pcifunc, idx; 2410 void *map; 2411 2412 map = xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2413 if (!map) { 2414 dev_err(rvu->dev, 2415 "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", 2416 __func__, mcam_idx); 2417 return -EFAULT; 2418 } 2419 2420 pcifunc = xa_to_value(map); 2421 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2422 if (!map) { 2423 dev_err(rvu->dev, 2424 "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", 2425 __func__, pcifunc, mcam_idx); 2426 return -ESRCH; 2427 } 2428 2429 idx = xa_to_value(map); 2430 2431 map = xa_erase(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2432 if (!map) { 2433 dev_err(rvu->dev, 2434 "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", 2435 __func__, mcam_idx); 2436 return -EFAULT; 2437 } 2438 return 0; 2439 } 2440 2441 static int 2442 npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) 2443 { 2444 int rc, idx; 2445 void *map; 2446 2447 dev_dbg(rvu->dev, 2448 "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", 2449 __func__, mcam_idx, pcifunc); 2450 2451 rc = xa_insert(&npc_priv->xa_idx2pf_map, mcam_idx, 2452 xa_mk_value(pcifunc), GFP_KERNEL); 2453 2454 if (rc) { 2455 map = xa_load(&npc_priv->xa_idx2pf_map, mcam_idx); 2456 dev_err(rvu->dev, 2457 "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", 2458 __func__, mcam_idx, xa_to_value(map)); 2459 return -EFAULT; 2460 } 2461 2462 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2463 if (!map) { 2464 dev_err(rvu->dev, 2465 "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", 2466 __func__, pcifunc, mcam_idx); 2467 return -ESRCH; 2468 } 2469 2470 idx = xa_to_value(map); 2471 2472 rc = xa_insert(&npc_priv->xa_pf2idx_map[idx], mcam_idx, 2473 xa_mk_value(pcifunc), GFP_KERNEL); 2474 2475 if (rc) { 2476 map = xa_load(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2477 xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2478 dev_err(rvu->dev, 2479 "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", 2480 __func__, mcam_idx, xa_to_value(map), idx); 2481 2482 return -EFAULT; 2483 } 2484 2485 return 0; 2486 } 2487 2488 static bool 2489 npc_subbank_suits(struct npc_subbank *sb, int key_type) 2490 { 2491 mutex_lock(&sb->lock); 2492 2493 if (!sb->key_type) { 2494 mutex_unlock(&sb->lock); 2495 return true; 2496 } 2497 2498 if (sb->key_type == key_type) { 2499 mutex_unlock(&sb->lock); 2500 return true; 2501 } 2502 2503 mutex_unlock(&sb->lock); 2504 return false; 2505 } 2506 2507 #define SB_ALIGN_UP(val) (((val) + npc_priv->subbank_depth) & \ 2508 ~((npc_priv->subbank_depth) - 1)) 2509 #define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv->subbank_depth) 2510 2511 static void npc_subbank_iter_down(struct rvu *rvu, 2512 int ref, int limit, 2513 int *cur_ref, int *cur_limit, 2514 bool *start, bool *stop) 2515 { 2516 int align; 2517 2518 *stop = false; 2519 2520 /* ALIGN_DOWN the limit to current subbank boundary bottom index */ 2521 if (*start) { 2522 *start = false; 2523 *cur_ref = ref; 2524 align = SB_ALIGN_DOWN(ref); 2525 if (align < limit) { 2526 *stop = true; 2527 *cur_limit = limit; 2528 return; 2529 } 2530 *cur_limit = align; 2531 return; 2532 } 2533 2534 *cur_ref = *cur_limit - 1; 2535 align = *cur_ref - npc_priv->subbank_depth + 1; 2536 if (align <= limit) { 2537 *stop = true; 2538 *cur_limit = limit; 2539 return; 2540 } 2541 2542 *cur_limit = align; 2543 } 2544 2545 static void npc_subbank_iter_up(struct rvu *rvu, 2546 int ref, int limit, 2547 int *cur_ref, int *cur_limit, 2548 bool *start, bool *stop) 2549 { 2550 int align; 2551 2552 *stop = false; 2553 2554 /* ALIGN_UP the limit to current subbank boundary top index */ 2555 if (*start) { 2556 *start = false; 2557 *cur_ref = ref; 2558 2559 /* Find next lower prio subbank's bottom index */ 2560 align = SB_ALIGN_UP(ref); 2561 2562 /* Crosses limit ? */ 2563 if (align - 1 > limit) { 2564 *stop = true; 2565 *cur_limit = limit; 2566 return; 2567 } 2568 2569 /* Current subbank's top index */ 2570 *cur_limit = align - 1; 2571 return; 2572 } 2573 2574 *cur_ref = *cur_limit + 1; 2575 align = *cur_ref + npc_priv->subbank_depth - 1; 2576 2577 if (align >= limit) { 2578 *stop = true; 2579 *cur_limit = limit; 2580 return; 2581 } 2582 2583 *cur_limit = align; 2584 } 2585 2586 static int 2587 npc_subbank_iter(struct rvu *rvu, int key_type, 2588 int ref, int limit, int prio, 2589 int *cur_ref, int *cur_limit, 2590 bool *start, bool *stop) 2591 { 2592 if (prio != NPC_MCAM_HIGHER_PRIO) 2593 npc_subbank_iter_up(rvu, ref, limit, 2594 cur_ref, cur_limit, 2595 start, stop); 2596 else 2597 npc_subbank_iter_down(rvu, ref, limit, 2598 cur_ref, cur_limit, 2599 start, stop); 2600 2601 /* limit and ref should < bank_depth for x4 */ 2602 if (key_type == NPC_MCAM_KEY_X4) { 2603 if (*cur_ref >= npc_priv->bank_depth) 2604 return -EINVAL; 2605 2606 if (*cur_limit >= npc_priv->bank_depth) 2607 return -EINVAL; 2608 } 2609 /* limit and ref should < 2 * bank_depth, for x2 */ 2610 if (*cur_ref >= 2 * npc_priv->bank_depth) 2611 return -EINVAL; 2612 2613 if (*cur_limit >= 2 * npc_priv->bank_depth) 2614 return -EINVAL; 2615 2616 return 0; 2617 } 2618 2619 static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, 2620 bool maps_del) 2621 { 2622 struct npc_subbank *sb; 2623 u16 vidx, midx; 2624 int sb_off, i; 2625 bool ret; 2626 int rc; 2627 2628 /* Check if we can dealloc indexes properly ? */ 2629 for (i = 0; i < count; i++) { 2630 rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), 2631 &sb, &sb_off); 2632 if (rc) { 2633 dev_err(rvu->dev, 2634 "Failed to free mcam idx=%u\n", mcam_idx[i]); 2635 return rc; 2636 } 2637 } 2638 2639 for (i = 0; i < count; i++) { 2640 if (npc_is_vidx(mcam_idx[i])) { 2641 vidx = mcam_idx[i]; 2642 midx = npc_vidx2idx(vidx); 2643 } else { 2644 midx = mcam_idx[i]; 2645 vidx = npc_idx2vidx(midx); 2646 } 2647 2648 if (midx >= npc_priv->bank_depth * npc_priv->num_banks) { 2649 dev_err(rvu->dev, 2650 "%s: Invalid mcam_idx=%u cannot be deleted\n", 2651 __func__, mcam_idx[i]); 2652 return -EINVAL; 2653 } 2654 2655 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 2656 &sb, &sb_off); 2657 if (rc) { 2658 dev_err(rvu->dev, 2659 "%s: Failed to find subbank info for vidx=%u\n", 2660 __func__, vidx); 2661 return rc; 2662 } 2663 2664 ret = npc_subbank_free(rvu, sb, sb_off); 2665 if (ret) { 2666 dev_err(rvu->dev, 2667 "%s: Failed to find subbank info for vidx=%u\n", 2668 __func__, vidx); 2669 return -EINVAL; 2670 } 2671 2672 if (!maps_del) 2673 continue; 2674 2675 rc = npc_del_from_pf_maps(rvu, midx); 2676 if (rc) 2677 return rc; 2678 2679 /* If there is no vidx mapping; continue */ 2680 if (vidx == midx) 2681 continue; 2682 2683 rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); 2684 if (rc) 2685 return rc; 2686 } 2687 2688 return 0; 2689 } 2690 2691 static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, 2692 int ref, int limit, int prio, 2693 bool contig, int count, 2694 u16 *mcam_idx) 2695 { 2696 struct npc_subbank *sb; 2697 unsigned long *bmap; 2698 int sb_off, off, rc; 2699 int cnt = 0; 2700 bool bitset; 2701 2702 if (prio != NPC_MCAM_HIGHER_PRIO) { 2703 while (ref <= limit) { 2704 /* Calculate subbank and subbank index */ 2705 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2706 &sb, &sb_off); 2707 if (rc) 2708 goto err; 2709 2710 /* If subbank is not suitable for requested key type 2711 * restart search from next subbank 2712 */ 2713 if (!npc_subbank_suits(sb, key_type)) { 2714 ref = SB_ALIGN_UP(ref); 2715 if (contig) { 2716 rc = npc_idx_free(rvu, mcam_idx, 2717 cnt, false); 2718 if (rc) 2719 return rc; 2720 cnt = 0; 2721 } 2722 continue; 2723 } 2724 2725 mutex_lock(&sb->lock); 2726 2727 /* If subbank is free; mark it as used */ 2728 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2729 rc = __npc_subbank_mark_used(rvu, sb, 2730 key_type); 2731 if (rc) { 2732 mutex_unlock(&sb->lock); 2733 dev_err(rvu->dev, 2734 "%s:Error to add to use array\n", 2735 __func__); 2736 goto err; 2737 } 2738 } 2739 2740 /* Find correct bmap */ 2741 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2742 2743 /* if bit is already set, reset 'cnt' */ 2744 bitset = test_bit(off, bmap); 2745 if (bitset) { 2746 mutex_unlock(&sb->lock); 2747 if (contig) { 2748 rc = npc_idx_free(rvu, mcam_idx, 2749 cnt, false); 2750 if (rc) 2751 return rc; 2752 cnt = 0; 2753 } 2754 2755 ref++; 2756 continue; 2757 } 2758 2759 set_bit(off, bmap); 2760 sb->free_cnt--; 2761 mcam_idx[cnt++] = ref; 2762 mutex_unlock(&sb->lock); 2763 2764 if (cnt == count) 2765 return 0; 2766 ref++; 2767 } 2768 2769 /* Could not allocate request count slots */ 2770 goto err; 2771 } 2772 while (ref >= limit) { 2773 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2774 &sb, &sb_off); 2775 if (rc) 2776 goto err; 2777 2778 if (!npc_subbank_suits(sb, key_type)) { 2779 ref = SB_ALIGN_DOWN(ref) - 1; 2780 if (contig) { 2781 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2782 if (rc) 2783 return rc; 2784 2785 cnt = 0; 2786 } 2787 continue; 2788 } 2789 2790 mutex_lock(&sb->lock); 2791 2792 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2793 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2794 if (rc) { 2795 mutex_unlock(&sb->lock); 2796 dev_err(rvu->dev, 2797 "%s:Error to add to use array\n", 2798 __func__); 2799 goto err; 2800 } 2801 } 2802 2803 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2804 bitset = test_bit(off, bmap); 2805 if (bitset) { 2806 mutex_unlock(&sb->lock); 2807 if (contig) { 2808 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2809 if (rc) 2810 return rc; 2811 2812 cnt = 0; 2813 } 2814 ref--; 2815 continue; 2816 } 2817 2818 mcam_idx[cnt++] = ref; 2819 sb->free_cnt--; 2820 set_bit(off, bmap); 2821 mutex_unlock(&sb->lock); 2822 2823 if (cnt == count) 2824 return 0; 2825 ref--; 2826 } 2827 2828 err: 2829 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2830 if (rc) 2831 dev_err(rvu->dev, 2832 "%s: Error happened while freeing cnt=%u indexes\n", 2833 __func__, cnt); 2834 2835 return -ENOSPC; 2836 } 2837 2838 static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, 2839 int key_type) 2840 { 2841 int cnt, spd; 2842 2843 spd = npc_priv->subbank_depth; 2844 mutex_lock(&sb->lock); 2845 2846 if (sb->flags & NPC_SUBBANK_FLAG_FREE) 2847 cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; 2848 else 2849 cnt = sb->free_cnt; 2850 2851 mutex_unlock(&sb->lock); 2852 return cnt; 2853 } 2854 2855 static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, 2856 int ref, int limit, int prio, 2857 bool contig, int count, 2858 u16 *mcam_idx) 2859 { 2860 struct npc_subbank *sb1, *sb2; 2861 bool max_alloc, start, stop; 2862 int r, l, sb_idx1, sb_idx2; 2863 int tot = 0, rc; 2864 int alloc_cnt; 2865 2866 max_alloc = !contig; 2867 2868 start = true; 2869 stop = false; 2870 2871 /* Loop until we cross the ref/limit boundary */ 2872 while (!stop) { 2873 rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, 2874 &r, &l, &start, &stop); 2875 2876 dev_dbg(rvu->dev, 2877 "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", 2878 __func__, ref, limit, r, l, 2879 start, stop, tot, count, rc); 2880 2881 if (rc) 2882 goto err; 2883 2884 /* Find subbank and subbank index for ref */ 2885 rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, 2886 &sb_idx1); 2887 if (rc) 2888 goto err; 2889 2890 dev_dbg(rvu->dev, 2891 "%s: ref subbank=%d off=%d\n", 2892 __func__, sb1->idx, sb_idx1); 2893 2894 /* Skip subbank if it is not available for the keytype */ 2895 if (!npc_subbank_suits(sb1, key_type)) { 2896 dev_dbg(rvu->dev, 2897 "%s: not suitable sb=%d key_type=%d\n", 2898 __func__, sb1->idx, key_type); 2899 continue; 2900 } 2901 2902 /* Find subbank and subbank index for limit */ 2903 rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, 2904 &sb_idx2); 2905 if (rc) 2906 goto err; 2907 2908 dev_dbg(rvu->dev, 2909 "%s: limit subbank=%d off=%d\n", 2910 __func__, sb_idx1, sb_idx2); 2911 2912 /* subbank of ref and limit should be same */ 2913 if (sb1 != sb2) { 2914 dev_err(rvu->dev, 2915 "%s: l(%d) and r(%d) are not in same subbank\n", 2916 __func__, r, l); 2917 goto err; 2918 } 2919 2920 if (contig && 2921 npc_subbank_free_cnt(rvu, sb1, key_type) < count) { 2922 dev_dbg(rvu->dev, "%s: less count =%d\n", 2923 __func__, 2924 npc_subbank_free_cnt(rvu, sb1, key_type)); 2925 continue; 2926 } 2927 2928 /* Try in one bank of a subbank */ 2929 alloc_cnt = 0; 2930 rc = npc_subbank_alloc(rvu, sb1, key_type, 2931 r, l, prio, contig, 2932 count - tot, mcam_idx + tot, 2933 count - tot, max_alloc, 2934 &alloc_cnt); 2935 2936 tot += alloc_cnt; 2937 2938 dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", 2939 __func__, tot, alloc_cnt); 2940 2941 if (!rc && count == tot) 2942 return 0; 2943 } 2944 err: 2945 dev_dbg(rvu->dev, "%s: Error to allocate\n", 2946 __func__); 2947 2948 /* non contiguous allocation fails. We need to do clean up */ 2949 if (max_alloc) { 2950 rc = npc_idx_free(rvu, mcam_idx, tot, false); 2951 if (rc) 2952 dev_err(rvu->dev, 2953 "%s: failed to free %u indexes\n", 2954 __func__, tot); 2955 } 2956 2957 return -EFAULT; 2958 } 2959 2960 /* Minimize allocation from bottom and top subbanks for noref allocations. 2961 * Default allocations are ref based, and will be allocated from top 2962 * subbanks (least priority subbanks). Since default allocation is at very 2963 * early stage of kernel netdev probes, this subbanks will be moved to 2964 * used subbanks list. This will pave a way for noref allocation from these 2965 * used subbanks. Skip allocation for these top and bottom, and try free 2966 * bank next. If none slot is available, come back and search in these 2967 * subbanks. 2968 */ 2969 2970 static int npc_subbank_restricted_idxs[2]; 2971 static bool restrict_valid = true; 2972 2973 static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) 2974 { 2975 int i; 2976 2977 if (!restrict_valid) 2978 return false; 2979 2980 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { 2981 if (index == npc_subbank_restricted_idxs[i]) 2982 return true; 2983 } 2984 2985 return false; 2986 } 2987 2988 static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, 2989 int count, u16 *mcam_idx) 2990 { 2991 struct npc_subbank *sb; 2992 unsigned long index; 2993 int tot = 0, rc; 2994 bool max_alloc; 2995 int alloc_cnt; 2996 int idx, i; 2997 void *val; 2998 2999 max_alloc = !contig; 3000 3001 /* Check used subbanks for free slots */ 3002 xa_for_each(&npc_priv->xa_sb_used, index, val) { 3003 idx = xa_to_value(val); 3004 3005 /* Minimize allocation from restricted subbanks 3006 * in noref allocations. 3007 */ 3008 if (npc_subbank_restrict_usage(rvu, idx)) 3009 continue; 3010 3011 sb = &npc_priv->sb[idx]; 3012 3013 /* Skip if not suitable subbank */ 3014 if (!npc_subbank_suits(sb, key_type)) 3015 continue; 3016 3017 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3018 continue; 3019 3020 /* try in bank 0. Try passing ref and limit equal to 3021 * subbank boundaries 3022 */ 3023 alloc_cnt = 0; 3024 rc = npc_subbank_alloc(rvu, sb, key_type, 3025 sb->b0b, sb->b0t, 0, 3026 contig, count - tot, 3027 mcam_idx + tot, 3028 count - tot, 3029 max_alloc, &alloc_cnt); 3030 3031 /* Non contiguous allocation may allocate less than 3032 * requested 'count'. 3033 */ 3034 tot += alloc_cnt; 3035 3036 dev_dbg(rvu->dev, 3037 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3038 __func__, alloc_cnt, sb->idx, tot, count); 3039 3040 /* Successfully allocated */ 3041 if (!rc && count == tot) 3042 return 0; 3043 3044 /* x4 entries can be allocated from bank 0 only */ 3045 if (key_type == NPC_MCAM_KEY_X4) 3046 continue; 3047 3048 /* try in bank 1 for x2 */ 3049 alloc_cnt = 0; 3050 rc = npc_subbank_alloc(rvu, sb, key_type, 3051 sb->b1b, sb->b1t, 0, 3052 contig, count - tot, 3053 mcam_idx + tot, 3054 count - tot, max_alloc, 3055 &alloc_cnt); 3056 3057 tot += alloc_cnt; 3058 3059 dev_dbg(rvu->dev, 3060 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3061 __func__, alloc_cnt, sb->idx, tot, count); 3062 3063 if (!rc && count == tot) 3064 return 0; 3065 } 3066 3067 /* Allocate in free subbanks */ 3068 xa_for_each(&npc_priv->xa_sb_free, index, val) { 3069 idx = xa_to_value(val); 3070 sb = &npc_priv->sb[idx]; 3071 3072 /* Minimize allocation from restricted subbanks 3073 * in noref allocations. 3074 */ 3075 if (npc_subbank_restrict_usage(rvu, idx)) 3076 continue; 3077 3078 if (!npc_subbank_suits(sb, key_type)) 3079 continue; 3080 3081 /* try in bank 0 */ 3082 alloc_cnt = 0; 3083 rc = npc_subbank_alloc(rvu, sb, key_type, 3084 sb->b0b, sb->b0t, 0, 3085 contig, count - tot, 3086 mcam_idx + tot, 3087 count - tot, 3088 max_alloc, &alloc_cnt); 3089 3090 tot += alloc_cnt; 3091 3092 dev_dbg(rvu->dev, 3093 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3094 __func__, alloc_cnt, sb->idx, tot, count); 3095 3096 /* Successfully allocated */ 3097 if (!rc && count == tot) 3098 return 0; 3099 3100 /* x4 entries can be allocated from bank 0 only */ 3101 if (key_type == NPC_MCAM_KEY_X4) 3102 continue; 3103 3104 /* try in bank 1 for x2 */ 3105 alloc_cnt = 0; 3106 rc = npc_subbank_alloc(rvu, sb, 3107 key_type, sb->b1b, sb->b1t, 0, 3108 contig, count - tot, 3109 mcam_idx + tot, count - tot, 3110 max_alloc, &alloc_cnt); 3111 3112 tot += alloc_cnt; 3113 3114 dev_dbg(rvu->dev, 3115 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3116 __func__, alloc_cnt, sb->idx, tot, count); 3117 3118 if (!rc && count == tot) 3119 return 0; 3120 } 3121 3122 /* Allocate from restricted subbanks */ 3123 for (i = 0; restrict_valid && 3124 (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { 3125 idx = npc_subbank_restricted_idxs[i]; 3126 sb = &npc_priv->sb[idx]; 3127 3128 /* Skip if not suitable subbank */ 3129 if (!npc_subbank_suits(sb, key_type)) 3130 continue; 3131 3132 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3133 continue; 3134 3135 /* try in bank 0. Try passing ref and limit equal to 3136 * subbank boundaries 3137 */ 3138 alloc_cnt = 0; 3139 rc = npc_subbank_alloc(rvu, sb, key_type, 3140 sb->b0b, sb->b0t, 0, 3141 contig, count - tot, 3142 mcam_idx + tot, 3143 count - tot, 3144 max_alloc, &alloc_cnt); 3145 3146 /* Non contiguous allocation may allocate less than 3147 * requested 'count'. 3148 */ 3149 tot += alloc_cnt; 3150 3151 dev_dbg(rvu->dev, 3152 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3153 __func__, alloc_cnt, sb->idx, tot, count); 3154 3155 /* Successfully allocated */ 3156 if (!rc && count == tot) 3157 return 0; 3158 3159 /* x4 entries can be allocated from bank 0 only */ 3160 if (key_type == NPC_MCAM_KEY_X4) 3161 continue; 3162 3163 /* try in bank 1 for x2 */ 3164 alloc_cnt = 0; 3165 rc = npc_subbank_alloc(rvu, sb, key_type, 3166 sb->b1b, sb->b1t, 0, 3167 contig, count - tot, 3168 mcam_idx + tot, 3169 count - tot, max_alloc, 3170 &alloc_cnt); 3171 3172 tot += alloc_cnt; 3173 3174 dev_dbg(rvu->dev, 3175 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3176 __func__, alloc_cnt, sb->idx, tot, count); 3177 3178 if (!rc && count == tot) 3179 return 0; 3180 } 3181 3182 /* non contiguous allocation fails. We need to do clean up */ 3183 if (max_alloc) 3184 npc_idx_free(rvu, mcam_idx, tot, false); 3185 3186 dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", 3187 __func__); 3188 3189 return -EFAULT; 3190 } 3191 3192 int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) 3193 { 3194 return npc_idx_free(rvu, mcam_idx, count, true); 3195 } 3196 3197 int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, 3198 int prio, u16 *mcam_idx, int ref, int limit, 3199 bool contig, int count, bool virt) 3200 { 3201 bool defrag_candidate = false; 3202 int i, eidx, rc, bd; 3203 bool ref_valid; 3204 u16 vidx; 3205 3206 bd = npc_priv->bank_depth; 3207 3208 /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 3209 * Here user wants to allocate 0th entry 3210 */ 3211 if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && 3212 count == 1) { 3213 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3214 prio, contig, count, mcam_idx); 3215 3216 if (rc) 3217 return rc; 3218 goto add2map; 3219 } 3220 3221 ref_valid = !!(limit || ref); 3222 defrag_candidate = !ref_valid && !contig && virt; 3223 if (!ref_valid) { 3224 if (contig && count > npc_priv->subbank_depth) 3225 goto try_noref_multi_subbank; 3226 3227 rc = npc_subbank_noref_alloc(rvu, key_type, contig, 3228 count, mcam_idx); 3229 if (!rc) 3230 goto add2map; 3231 3232 try_noref_multi_subbank: 3233 eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; 3234 3235 if (prio == NPC_MCAM_HIGHER_PRIO) 3236 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3237 eidx, 0, 3238 NPC_MCAM_HIGHER_PRIO, 3239 contig, count, 3240 mcam_idx); 3241 else 3242 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3243 0, eidx, 3244 NPC_MCAM_LOWER_PRIO, 3245 contig, count, 3246 mcam_idx); 3247 3248 if (!rc) 3249 goto add2map; 3250 3251 return rc; 3252 } 3253 3254 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 3255 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 3256 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3257 __func__, ref, limit); 3258 return -EINVAL; 3259 } 3260 3261 if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || 3262 (key_type == NPC_MCAM_KEY_X2 && 3263 (ref >= 2 * bd || limit >= 2 * bd))) { 3264 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3265 __func__, ref, limit); 3266 return -EINVAL; 3267 } 3268 3269 if (contig && count > npc_priv->subbank_depth) 3270 goto try_ref_multi_subbank; 3271 3272 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3273 prio, contig, count, mcam_idx); 3274 if (!rc) 3275 goto add2map; 3276 3277 try_ref_multi_subbank: 3278 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3279 ref, limit, prio, 3280 contig, count, mcam_idx); 3281 if (!rc) 3282 goto add2map; 3283 3284 return rc; 3285 3286 add2map: 3287 for (i = 0; i < count; i++) { 3288 rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); 3289 if (rc) 3290 goto err; 3291 3292 if (!defrag_candidate) 3293 continue; 3294 3295 rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); 3296 if (rc) { 3297 npc_del_from_pf_maps(rvu, mcam_idx[i]); 3298 goto err; 3299 } 3300 3301 /* Return vidx to caller */ 3302 mcam_idx[i] = vidx; 3303 } 3304 3305 return 0; 3306 err: 3307 for (int j = 0; j < i; j++) { 3308 npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); 3309 3310 if (!defrag_candidate) 3311 continue; 3312 3313 npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); 3314 } 3315 3316 return rc; 3317 3318 } 3319 3320 void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, 3321 int *x4_free, int *sb_free) 3322 { 3323 struct npc_subbank *sb; 3324 int i; 3325 3326 /* Reset all stats to zero */ 3327 *x2_free = 0; 3328 *x4_free = 0; 3329 *sb_free = 0; 3330 3331 for (i = 0; i < npc_priv->num_subbanks; i++) { 3332 sb = &npc_priv->sb[i]; 3333 mutex_lock(&sb->lock); 3334 3335 /* Count number of free subbanks */ 3336 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 3337 (*sb_free)++; 3338 goto next; 3339 } 3340 3341 /* Sumup x4 free count */ 3342 if (sb->key_type == NPC_MCAM_KEY_X4) { 3343 (*x4_free) += sb->free_cnt; 3344 goto next; 3345 } 3346 3347 /* Sumup x2 free counts */ 3348 (*x2_free) += sb->free_cnt; 3349 next: 3350 mutex_unlock(&sb->lock); 3351 } 3352 } 3353 3354 int 3355 rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, 3356 struct msg_req *req, 3357 struct npc_cn20k_get_fcnt_rsp *rsp) 3358 { 3359 npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, 3360 &rsp->free_x4, &rsp->free_subbanks); 3361 return 0; 3362 } 3363 3364 int 3365 rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, 3366 struct msg_req *req, 3367 struct npc_cn20k_get_kex_cfg_rsp *rsp) 3368 { 3369 int extr, lt; 3370 3371 rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); 3372 rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); 3373 3374 /* Get EXTRACTOR LID */ 3375 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3376 rsp->intf_extr_lid[NIX_INTF_RX][extr] = 3377 CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); 3378 rsp->intf_extr_lid[NIX_INTF_TX][extr] = 3379 CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); 3380 } 3381 3382 /* Get EXTRACTOR LTYPE */ 3383 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3384 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3385 rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = 3386 CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); 3387 rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = 3388 CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); 3389 } 3390 } 3391 3392 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3393 return 0; 3394 } 3395 3396 static u32 *subbank_srch_order; 3397 3398 static void npc_populate_restricted_idxs(int num_subbanks) 3399 { 3400 npc_subbank_restricted_idxs[0] = num_subbanks - 1; 3401 npc_subbank_restricted_idxs[1] = 0; 3402 } 3403 3404 static int npc_create_srch_order(int cnt) 3405 { 3406 int val = 0; 3407 3408 subbank_srch_order = kcalloc(cnt, sizeof(u32), 3409 GFP_KERNEL); 3410 if (!subbank_srch_order) 3411 return -ENOMEM; 3412 3413 /* cnt(subbank depth) is always a power of 2. There is a check in 3414 * npc_priv_init() to check the same. 3415 */ 3416 for (int i = 0; i < cnt; i += 2) { 3417 subbank_srch_order[i] = cnt / 2 - val - 1; 3418 subbank_srch_order[i + 1] = cnt / 2 + 1 + val; 3419 val++; 3420 } 3421 3422 subbank_srch_order[cnt - 1] = cnt / 2; 3423 return 0; 3424 } 3425 3426 static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) 3427 { 3428 mutex_init(&sb->lock); 3429 3430 sb->b0b = idx * npc_priv->subbank_depth; 3431 sb->b0t = sb->b0b + npc_priv->subbank_depth - 1; 3432 3433 sb->b1b = npc_priv->bank_depth + idx * npc_priv->subbank_depth; 3434 sb->b1t = sb->b1b + npc_priv->subbank_depth - 1; 3435 3436 sb->flags = NPC_SUBBANK_FLAG_FREE; 3437 sb->idx = idx; 3438 sb->arr_idx = subbank_srch_order[idx]; 3439 3440 dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", 3441 __func__, sb->idx, sb->arr_idx); 3442 3443 /* Keep first and last subbank at end of free array; so that 3444 * it will be used at last 3445 */ 3446 xa_store(&npc_priv->xa_sb_free, sb->arr_idx, 3447 xa_mk_value(sb->idx), GFP_KERNEL); 3448 } 3449 3450 static int npc_pcifunc_map_create(struct rvu *rvu) 3451 { 3452 int pf, vf, numvfs; 3453 int cnt = 0; 3454 u16 pcifunc; 3455 u64 cfg; 3456 3457 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { 3458 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3459 numvfs = (cfg >> 12) & 0xFF; 3460 3461 /* Skip not enabled PFs */ 3462 if (!(cfg & BIT_ULL(20))) 3463 goto chk_vfs; 3464 3465 /* If Admin function, check on VFs */ 3466 if (cfg & BIT_ULL(21)) 3467 goto chk_vfs; 3468 3469 pcifunc = pf << 9; 3470 3471 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3472 xa_mk_value(cnt), GFP_KERNEL); 3473 3474 cnt++; 3475 3476 chk_vfs: 3477 for (vf = 0; vf < numvfs; vf++) { 3478 pcifunc = (pf << 9) | (vf + 1); 3479 3480 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3481 xa_mk_value(cnt), GFP_KERNEL); 3482 cnt++; 3483 } 3484 } 3485 3486 return cnt; 3487 } 3488 3489 struct npc_defrag_node { 3490 u8 idx; 3491 u8 key_type; 3492 bool valid; 3493 bool refs; 3494 u16 free_cnt; 3495 u16 vidx_cnt; 3496 u16 *vidx; 3497 struct list_head list; 3498 }; 3499 3500 static bool npc_defrag_skip_restricted_sb(int sb_id) 3501 { 3502 int i; 3503 3504 if (!restrict_valid) 3505 return false; 3506 3507 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) 3508 if (sb_id == npc_subbank_restricted_idxs[i]) 3509 return true; 3510 return false; 3511 } 3512 3513 /* Find subbank with minimum number of virtual indexes */ 3514 static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) 3515 { 3516 struct npc_defrag_node *node, *tnode = NULL; 3517 int min = INT_MAX; 3518 3519 list_for_each_entry(node, lh, list) { 3520 if (!node->valid) 3521 continue; 3522 3523 /* if subbank has ref allocated mcam indexes, that subbank 3524 * is not a good candidate to move out indexes. 3525 */ 3526 if (node->refs) 3527 continue; 3528 3529 if (min > node->vidx_cnt) { 3530 min = node->vidx_cnt; 3531 tnode = node; 3532 } 3533 } 3534 3535 return tnode; 3536 } 3537 3538 /* Find subbank with maximum number of free spaces */ 3539 static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) 3540 { 3541 struct npc_defrag_node *node, *tnode = NULL; 3542 int max = INT_MIN; 3543 3544 list_for_each_entry(node, lh, list) { 3545 if (!node->valid) 3546 continue; 3547 3548 if (max < node->free_cnt) { 3549 max = node->free_cnt; 3550 tnode = node; 3551 } 3552 } 3553 3554 return tnode; 3555 } 3556 3557 static int npc_defrag_alloc_free_slots(struct rvu *rvu, 3558 struct npc_defrag_node *f, 3559 int cnt, u16 *save) 3560 { 3561 int alloc_cnt1, alloc_cnt2; 3562 struct npc_subbank *sb; 3563 int rc, sb_off, i, err; 3564 bool deleted; 3565 3566 sb = &npc_priv->sb[f->idx]; 3567 3568 alloc_cnt1 = 0; 3569 alloc_cnt2 = 0; 3570 3571 rc = __npc_subbank_alloc(rvu, sb, 3572 NPC_MCAM_KEY_X2, sb->b0b, 3573 sb->b0t, 3574 NPC_MCAM_LOWER_PRIO, 3575 false, cnt, save, cnt, true, 3576 &alloc_cnt1); 3577 3578 if (alloc_cnt1 < cnt) { 3579 rc = __npc_subbank_alloc(rvu, sb, 3580 NPC_MCAM_KEY_X2, sb->b1b, 3581 sb->b1t, 3582 NPC_MCAM_LOWER_PRIO, 3583 false, cnt - alloc_cnt1, 3584 save + alloc_cnt1, 3585 cnt - alloc_cnt1, 3586 true, &alloc_cnt2); 3587 } 3588 3589 if (alloc_cnt1 + alloc_cnt2 != cnt) { 3590 dev_err(rvu->dev, 3591 "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", 3592 __func__, cnt, alloc_cnt1, alloc_cnt2); 3593 rc = -ENOSPC; 3594 goto fail_free_alloc; 3595 } 3596 3597 return 0; 3598 3599 fail_free_alloc: 3600 for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { 3601 err = npc_mcam_idx_2_subbank_idx(rvu, save[i], 3602 &sb, &sb_off); 3603 if (err) { 3604 dev_err(rvu->dev, 3605 "%s: Error to find subbank for mcam idx=%u\n", 3606 __func__, save[i]); 3607 break; 3608 } 3609 3610 deleted = __npc_subbank_free(rvu, sb, sb_off); 3611 if (!deleted) { 3612 dev_err(rvu->dev, 3613 "%s: Error to free mcam idx=%u\n", 3614 __func__, save[i]); 3615 break; 3616 } 3617 } 3618 3619 return rc; 3620 } 3621 3622 static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, 3623 u16 new_midx, u16 vidx) 3624 { 3625 struct npc_defrag_show_node *node; 3626 3627 node = kcalloc(1, sizeof(*node), GFP_KERNEL); 3628 if (!node) 3629 return -ENOMEM; 3630 3631 node->old_midx = old_midx; 3632 node->new_midx = new_midx; 3633 node->vidx = vidx; 3634 INIT_LIST_HEAD(&node->list); 3635 3636 mutex_lock(&npc_priv->lock); 3637 list_add_tail(&node->list, &npc_priv->defrag_lh); 3638 mutex_unlock(&npc_priv->lock); 3639 3640 return 0; 3641 } 3642 3643 static 3644 int npc_defrag_move_vdx_to_free(struct rvu *rvu, 3645 struct npc_defrag_node *f, 3646 struct npc_defrag_node *v, 3647 int cnt, u16 *save) 3648 { 3649 u16 new_midx, old_midx, vidx, target_pf; 3650 struct npc_mcam *mcam = &rvu->hw->mcam; 3651 struct rvu_npc_mcam_rule *rule, *tmp; 3652 int i, vidx_cnt, rc, sb_off; 3653 struct npc_subbank *sb; 3654 bool deleted; 3655 u16 pcifunc; 3656 int blkaddr; 3657 void *map; 3658 u8 bank; 3659 u16 midx; 3660 u64 stats; 3661 3662 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3663 3664 vidx_cnt = v->vidx_cnt; 3665 for (i = 0; i < cnt; i++) { 3666 vidx = v->vidx[vidx_cnt - i - 1]; 3667 old_midx = npc_vidx2idx(vidx); 3668 new_midx = save[cnt - i - 1]; 3669 3670 dev_dbg(rvu->dev, 3671 "%s: Moving %u ---> %u (vidx=%u)\n", 3672 __func__, 3673 old_midx, new_midx, vidx); 3674 3675 rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); 3676 if (rc) 3677 dev_err(rvu->dev, 3678 "%s: Error happened to add to show list vidx=%u\n", 3679 __func__, vidx); 3680 3681 /* Modify vidx to point to new mcam idx */ 3682 rc = npc_vidx_maps_modify(rvu, vidx, new_midx); 3683 if (rc) 3684 return rc; 3685 3686 midx = old_midx % mcam->banksize; 3687 bank = old_midx / mcam->banksize; 3688 stats = rvu_read64(rvu, blkaddr, 3689 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, 3690 bank)); 3691 3692 /* If bug happened during copy/enable mcam, then there is a bug in allocation 3693 * algorithm itself. There is no point in rewinding and returning, as it 3694 * will face further issue. Return error after printing error 3695 */ 3696 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) { 3697 dev_err(rvu->dev, 3698 "%s: Error happened while disabling old_mid=%u\n", 3699 __func__, old_midx); 3700 return -EFAULT; 3701 } 3702 3703 if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) { 3704 dev_err(rvu->dev, 3705 "%s: Error happened while copying old_midx=%u new_midx=%u\n", 3706 __func__, old_midx, new_midx); 3707 return -EFAULT; 3708 } 3709 3710 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) { 3711 dev_err(rvu->dev, 3712 "%s: Error happened while enabling new_mid=%u\n", 3713 __func__, new_midx); 3714 return -EFAULT; 3715 } 3716 3717 midx = new_midx % mcam->banksize; 3718 bank = new_midx / mcam->banksize; 3719 rvu_write64(rvu, blkaddr, 3720 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), 3721 stats); 3722 3723 /* Free the old mcam idx */ 3724 rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, 3725 &sb, &sb_off); 3726 if (rc) { 3727 dev_err(rvu->dev, 3728 "%s: Unable to calculate subbank off for mcamidx=%u\n", 3729 __func__, old_midx); 3730 return rc; 3731 } 3732 3733 deleted = __npc_subbank_free(rvu, sb, sb_off); 3734 if (!deleted) { 3735 dev_err(rvu->dev, 3736 "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", 3737 __func__, old_midx, sb->idx, sb_off); 3738 return -EFAULT; 3739 } 3740 3741 /* save pcifunc */ 3742 map = xa_load(&npc_priv->xa_idx2pf_map, old_midx); 3743 pcifunc = xa_to_value(map); 3744 3745 /* delete from pf maps */ 3746 rc = npc_del_from_pf_maps(rvu, old_midx); 3747 if (rc) { 3748 dev_err(rvu->dev, 3749 "%s: Failed to delete pf maps for mcamidx=%u\n", 3750 __func__, old_midx); 3751 return rc; 3752 } 3753 3754 /* add new mcam_idx to pf map */ 3755 rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); 3756 if (rc) { 3757 dev_err(rvu->dev, 3758 "%s: Failed to add pf maps for mcamidx=%u\n", 3759 __func__, new_midx); 3760 return rc; 3761 } 3762 3763 /* Remove from mcam maps */ 3764 mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; 3765 mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; 3766 npc_mcam_clear_bit(mcam, old_midx); 3767 3768 mcam->entry2pfvf_map[new_midx] = pcifunc; 3769 /* Counter is not preserved */ 3770 mcam->entry2cntr_map[new_midx] = new_midx; 3771 target_pf = mcam->entry2target_pffunc[old_midx]; 3772 mcam->entry2target_pffunc[new_midx] = target_pf; 3773 mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP; 3774 3775 npc_mcam_set_bit(mcam, new_midx); 3776 3777 /* Note: list order is not functionally required for mcam_rules */ 3778 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 3779 if (rule->entry != old_midx) 3780 continue; 3781 3782 rule->entry = new_midx; 3783 break; 3784 } 3785 3786 /* Mark as invalid */ 3787 v->vidx[vidx_cnt - i - 1] = -1; 3788 save[cnt - i - 1] = -1; 3789 3790 f->free_cnt--; 3791 v->vidx_cnt--; 3792 } 3793 3794 return 0; 3795 } 3796 3797 static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) 3798 { 3799 struct npc_defrag_node *v = NULL; 3800 struct npc_defrag_node *f = NULL; 3801 int rc = 0, cnt; 3802 u16 *save; 3803 3804 while (1) { 3805 /* Find subbank with minimum vidx */ 3806 if (!v) { 3807 v = npc_subbank_min_vidx(lh); 3808 if (!v) 3809 break; 3810 } 3811 3812 /* Find subbank with maximum free slots */ 3813 if (!f) { 3814 f = npc_subbank_max_free(lh); 3815 if (!f) 3816 break; 3817 } 3818 3819 if (!v->vidx_cnt) { 3820 list_del_init(&v->list); 3821 v = NULL; 3822 continue; 3823 } 3824 3825 if (!f->free_cnt) { 3826 list_del_init(&f->list); 3827 f = NULL; 3828 continue; 3829 } 3830 3831 /* If both subbanks are same, choose vidx and 3832 * search for free list again 3833 */ 3834 if (f == v) { 3835 list_del_init(&f->list); 3836 f = NULL; 3837 continue; 3838 } 3839 3840 /* Calculate minimum free slots needs to be allocated */ 3841 cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : 3842 f->free_cnt; 3843 3844 dev_dbg(rvu->dev, 3845 "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", 3846 __func__, cnt, f->free_cnt, f->idx, 3847 v->vidx_cnt, v->idx); 3848 3849 /* Allocate an array to store newly allocated 3850 * free slots (mcam indexes) 3851 */ 3852 save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); 3853 if (!save) { 3854 rc = -ENOMEM; 3855 goto err; 3856 } 3857 3858 /* Alloc free slots for existing vidx */ 3859 rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); 3860 if (rc) { 3861 kfree(save); 3862 goto err; 3863 } 3864 3865 /* Move vidx to free slots; update pf_map and vidx maps, 3866 * and free existing vidx mcam slots 3867 */ 3868 rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); 3869 if (rc) { 3870 kfree(save); 3871 goto err; 3872 } 3873 3874 kfree(save); 3875 3876 if (!f->free_cnt) { 3877 list_del_init(&f->list); 3878 f = NULL; 3879 } 3880 3881 if (!v->vidx_cnt) { 3882 list_del_init(&v->list); 3883 v = NULL; 3884 } 3885 } 3886 3887 err: 3888 /* Whole defragmentation process is done within locks. if there 3889 * is an error, it would be hard to roll back as index remove/add 3890 * can fail again if it failed before. This would mean that there 3891 * is bug in the index management algorithm. 3892 * Return from here than rolling back. 3893 */ 3894 return rc; 3895 } 3896 3897 static void npc_defrag_list_clear(void) 3898 { 3899 struct npc_defrag_show_node *node, *next; 3900 3901 mutex_lock(&npc_priv->lock); 3902 list_for_each_entry_safe(node, next, &npc_priv->defrag_lh, list) { 3903 list_del_init(&node->list); 3904 kfree(node); 3905 } 3906 3907 mutex_unlock(&npc_priv->lock); 3908 } 3909 3910 static void npc_lock_all_subbank(void) 3911 { 3912 int i; 3913 3914 for (i = 0; i < npc_priv->num_subbanks; i++) 3915 mutex_lock(&npc_priv->sb[i].lock); 3916 } 3917 3918 static void npc_unlock_all_subbank(void) 3919 { 3920 int i; 3921 3922 for (i = npc_priv->num_subbanks - 1; i >= 0; i--) 3923 mutex_unlock(&npc_priv->sb[i].lock); 3924 } 3925 3926 int npc_cn20k_search_order_set(struct rvu *rvu, 3927 u64 narr[MAX_NUM_SUB_BANKS], int cnt) 3928 { 3929 struct npc_mcam *mcam = &rvu->hw->mcam; 3930 int rsrc[2][MAX_NUM_SUB_BANKS] = { }; 3931 u8 save[MAX_NUM_SUB_BANKS] = { }; 3932 struct npc_subbank *sb; 3933 struct xarray *xa; 3934 int prio, rc, err; 3935 int sb_idx; 3936 enum { 3937 FREE = 0, 3938 USED = 1, 3939 }; 3940 3941 if (cnt != npc_priv->num_subbanks) { 3942 dev_err(rvu->dev, "Number of entries(%u) != %u\n", 3943 cnt, npc_priv->num_subbanks); 3944 return -EINVAL; 3945 } 3946 3947 mutex_lock(&mcam->lock); 3948 npc_lock_all_subbank(); 3949 3950 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 3951 sb = &npc_priv->sb[sb_idx]; 3952 save[sb->idx] = sb->arr_idx; 3953 } 3954 3955 for (prio = 0; prio < cnt; prio++) { 3956 sb_idx = narr[prio]; 3957 sb = &npc_priv->sb[sb_idx]; 3958 3959 if (sb->flags & NPC_SUBBANK_FLAG_USED) 3960 xa = &npc_priv->xa_sb_used; 3961 else 3962 xa = &npc_priv->xa_sb_free; 3963 3964 rc = xa_err(xa_store(xa, prio, 3965 xa_mk_value(sb_idx), GFP_KERNEL)); 3966 if (rc) { 3967 dev_err(rvu->dev, 3968 "Setting arr_idx=%d for sb=%d failed\n", 3969 sb->arr_idx, sb_idx); 3970 goto fail; 3971 } 3972 3973 if (sb->flags & NPC_SUBBANK_FLAG_USED) { 3974 rsrc[USED][sb->arr_idx] -= 1; 3975 rsrc[USED][prio] += 1; 3976 } else { 3977 rsrc[FREE][sb->arr_idx] -= 1; 3978 rsrc[FREE][prio] += 1; 3979 } 3980 3981 sb->arr_idx = prio; 3982 } 3983 3984 for (prio = 0; prio < cnt; prio++) { 3985 if (rsrc[FREE][prio] == -1) 3986 xa_erase(&npc_priv->xa_sb_free, prio); 3987 3988 if (rsrc[USED][prio] == -1) 3989 xa_erase(&npc_priv->xa_sb_used, prio); 3990 } 3991 3992 for (int i = 0; i < cnt; i++) 3993 subbank_srch_order[i] = (u32)narr[i]; 3994 3995 restrict_valid = false; 3996 3997 npc_unlock_all_subbank(); 3998 mutex_unlock(&mcam->lock); 3999 4000 return 0; 4001 4002 fail: 4003 for (prio = 0; prio < cnt; prio++) { 4004 if (rsrc[FREE][prio] == 1) 4005 xa_erase(&npc_priv->xa_sb_free, prio); 4006 4007 if (rsrc[USED][prio] == 1) 4008 xa_erase(&npc_priv->xa_sb_used, prio); 4009 } 4010 4011 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 4012 sb = &npc_priv->sb[sb_idx]; 4013 sb->arr_idx = save[sb_idx]; 4014 4015 if (sb->flags & NPC_SUBBANK_FLAG_USED) 4016 xa = &npc_priv->xa_sb_used; 4017 else 4018 xa = &npc_priv->xa_sb_free; 4019 4020 /* Since the entry already exists, xa_store() replaces 4021 * the value without a kmalloc(), making failure highly unlikely. 4022 */ 4023 err = xa_err(xa_store(xa, sb->arr_idx, 4024 xa_mk_value(sb->idx), GFP_KERNEL)); 4025 WARN(!!err, "Failed to rollback sb=%u idx=%u\n", 4026 sb->idx, sb->arr_idx); 4027 } 4028 4029 npc_unlock_all_subbank(); 4030 mutex_unlock(&mcam->lock); 4031 4032 return rc; 4033 } 4034 4035 const u32 *npc_cn20k_search_order_get(bool *restricted_order, u32 *sz) 4036 { 4037 *restricted_order = restrict_valid; 4038 *sz = npc_priv->num_subbanks; 4039 return subbank_srch_order; 4040 } 4041 4042 /* Only non-ref non-contigous mcam indexes 4043 * are picked for defrag process 4044 */ 4045 int npc_cn20k_defrag(struct rvu *rvu) 4046 { 4047 struct npc_mcam *mcam = &rvu->hw->mcam; 4048 struct npc_defrag_node *node, *tnode; 4049 struct list_head x4lh, x2lh, *lh; 4050 int rc = 0, i, sb_off, tot; 4051 struct npc_subbank *sb; 4052 unsigned long index; 4053 void *map; 4054 u16 midx; 4055 4056 /* Free previous show list */ 4057 npc_defrag_list_clear(); 4058 4059 INIT_LIST_HEAD(&x4lh); 4060 INIT_LIST_HEAD(&x2lh); 4061 4062 node = kcalloc(npc_priv->num_subbanks, sizeof(*node), GFP_KERNEL); 4063 if (!node) 4064 return -ENOMEM; 4065 4066 /* Lock mcam */ 4067 mutex_lock(&mcam->lock); 4068 npc_lock_all_subbank(); 4069 4070 /* Fill in node with subbank properties */ 4071 for (i = 0; i < npc_priv->num_subbanks; i++) { 4072 sb = &npc_priv->sb[i]; 4073 4074 node[i].idx = i; 4075 node[i].key_type = sb->key_type; 4076 node[i].free_cnt = sb->free_cnt; 4077 node[i].vidx = kcalloc(npc_priv->subbank_depth * 2, 4078 sizeof(*node[i].vidx), 4079 GFP_KERNEL); 4080 if (!node[i].vidx) { 4081 rc = -ENOMEM; 4082 goto free_vidx; 4083 } 4084 4085 /* If subbank is empty, dont include it in defrag 4086 * process 4087 */ 4088 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 4089 node[i].valid = false; 4090 continue; 4091 } 4092 4093 if (npc_defrag_skip_restricted_sb(i)) { 4094 node[i].valid = false; 4095 continue; 4096 } 4097 4098 node[i].valid = true; 4099 INIT_LIST_HEAD(&node[i].list); 4100 4101 /* Add node to x2 or x4 list */ 4102 lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; 4103 list_add_tail(&node[i].list, lh); 4104 } 4105 4106 /* Filling vidx[] array with all vidx in that subbank */ 4107 xa_for_each_start(&npc_priv->xa_vidx2idx_map, index, map, 4108 npc_priv->bank_depth * 2) { 4109 midx = xa_to_value(map); 4110 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 4111 &sb, &sb_off); 4112 if (rc) { 4113 dev_err(rvu->dev, 4114 "%s: Error to get mcam_idx for vidx=%lu\n", 4115 __func__, index); 4116 goto free_vidx; 4117 } 4118 4119 tnode = &node[sb->idx]; 4120 tnode->vidx[tnode->vidx_cnt] = index; 4121 tnode->vidx_cnt++; 4122 } 4123 4124 /* Mark all subbank which has ref allocation */ 4125 for (i = 0; i < npc_priv->num_subbanks; i++) { 4126 tnode = &node[i]; 4127 4128 if (!tnode->valid) 4129 continue; 4130 4131 tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? 4132 npc_priv->subbank_depth * 2 : npc_priv->subbank_depth; 4133 4134 if (node[i].vidx_cnt != tot - tnode->free_cnt) 4135 tnode->refs = true; 4136 } 4137 4138 rc = npc_defrag_process(rvu, &x2lh); 4139 if (rc) 4140 goto free_vidx; 4141 4142 rc = npc_defrag_process(rvu, &x4lh); 4143 if (rc) 4144 goto free_vidx; 4145 4146 free_vidx: 4147 npc_unlock_all_subbank(); 4148 mutex_unlock(&mcam->lock); 4149 for (i = 0; i < npc_priv->num_subbanks; i++) 4150 kfree(node[i].vidx); 4151 kfree(node); 4152 return rc; 4153 } 4154 4155 int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, 4156 struct msg_rsp *rsp) 4157 { 4158 return npc_cn20k_defrag(rvu); 4159 } 4160 4161 int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, 4162 u16 *mcast, u16 *promisc, u16 *ucast) 4163 { 4164 u16 *ptr[4] = {promisc, mcast, bcast, ucast}; 4165 unsigned long idx; 4166 bool set = false; 4167 void *val; 4168 int i, j; 4169 4170 for (i = 0; i < ARRAY_SIZE(ptr); i++) { 4171 if (!ptr[i]) 4172 continue; 4173 4174 *ptr[i] = USHRT_MAX; 4175 } 4176 4177 if (!npc_priv) 4178 return 0; 4179 4180 if (is_lbk_vf(rvu, pcifunc)) { 4181 if (!ptr[0]) 4182 return -EINVAL; 4183 4184 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4185 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4186 if (!val) { 4187 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4188 __func__, 4189 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4190 pcifunc); 4191 4192 return -ESRCH; 4193 } 4194 4195 *ptr[0] = xa_to_value(val); 4196 return 0; 4197 } 4198 4199 if (is_vf(pcifunc)) { 4200 if (!ptr[3]) 4201 return -EINVAL; 4202 4203 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4204 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4205 if (!val) { 4206 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4207 __func__, 4208 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4209 pcifunc); 4210 4211 return -ESRCH; 4212 } 4213 4214 *ptr[3] = xa_to_value(val); 4215 return 0; 4216 } 4217 4218 for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, 4219 j++) { 4220 if (!ptr[j]) 4221 continue; 4222 4223 idx = NPC_DFT_RULE_ID_MK(pcifunc, i); 4224 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4225 if (!val) { 4226 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4227 __func__, 4228 npc_dft_rule_name[i], pcifunc); 4229 4230 continue; 4231 } 4232 4233 *ptr[j] = xa_to_value(val); 4234 set = true; 4235 } 4236 4237 return set ? 0 : -ESRCH; 4238 } 4239 4240 int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, 4241 struct npc_get_pfl_info_rsp *rsp) 4242 { 4243 if (!is_cn20k(rvu->pdev)) { 4244 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4245 return -EOPNOTSUPP; 4246 } 4247 4248 rsp->kw_type = npc_priv->kw; 4249 rsp->x4_slots = npc_priv->bank_depth; 4250 return 0; 4251 } 4252 4253 int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, 4254 struct npc_get_num_kws_req *req, 4255 struct npc_get_num_kws_rsp *rsp) 4256 { 4257 u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4258 u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4259 struct rvu_npc_mcam_rule dummy = { 0 }; 4260 struct mcam_entry_mdata mdata = { }; 4261 struct npc_install_flow_req *fl; 4262 int i, cnt = 0, blkaddr; 4263 4264 if (!is_cn20k(rvu->pdev)) { 4265 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4266 return -EOPNOTSUPP; 4267 } 4268 4269 fl = &req->fl; 4270 4271 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4272 if (blkaddr < 0) { 4273 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4274 return NPC_MCAM_INVALID_REQ; 4275 } 4276 4277 mdata.kw = kw; 4278 mdata.kw_mask = kw_mask; 4279 4280 npc_update_flow(rvu, &mdata, fl->features, &fl->packet, 4281 &fl->mask, &dummy, fl->intf, blkaddr); 4282 4283 /* Find the most significant word valid. Traverse from 4284 * MSB to LSB, check if cam0 or cam1 is set 4285 */ 4286 for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { 4287 if (kw[i] || kw_mask[i]) { 4288 cnt = i + 1; 4289 break; 4290 } 4291 } 4292 4293 rsp->kws = cnt; 4294 4295 return 0; 4296 } 4297 4298 int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, 4299 struct npc_get_dft_rl_idxs_rsp *rsp) 4300 { 4301 u16 bcast, mcast, promisc, ucast; 4302 u16 pcifunc; 4303 int rc; 4304 4305 if (!is_cn20k(rvu->pdev)) { 4306 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4307 return -EOPNOTSUPP; 4308 } 4309 4310 pcifunc = req->hdr.pcifunc; 4311 4312 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 4313 &promisc, &ucast); 4314 if (rc) 4315 return rc; 4316 4317 rsp->bcast = bcast; 4318 rsp->mcast = mcast; 4319 rsp->promisc = promisc; 4320 rsp->ucast = ucast; 4321 return 0; 4322 } 4323 4324 bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) 4325 { 4326 return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || 4327 is_lbk_vf(rvu, pcifunc); 4328 } 4329 4330 void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) 4331 { 4332 struct npc_mcam *mcam = &rvu->hw->mcam; 4333 u16 ptr[4] = {[0 ... 3] = USHRT_MAX}; 4334 struct rvu_npc_mcam_rule *rule, *tmp; 4335 unsigned long index; 4336 int blkaddr, rc, i; 4337 void *map; 4338 4339 if (!npc_priv) 4340 return; 4341 4342 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4343 dev_dbg(rvu->dev, 4344 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4345 __func__, pcifunc); 4346 return; 4347 } 4348 4349 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 4350 &ptr[2], &ptr[3]); 4351 if (rc) 4352 return; 4353 4354 /* LBK */ 4355 if (is_lbk_vf(rvu, pcifunc)) { 4356 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4357 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4358 if (!map) 4359 dev_dbg(rvu->dev, 4360 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4361 __func__, 4362 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4363 pcifunc); 4364 4365 goto free_rules; 4366 } 4367 4368 /* VF */ 4369 if (is_vf(pcifunc)) { 4370 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4371 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4372 if (!map) 4373 dev_dbg(rvu->dev, 4374 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4375 __func__, 4376 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4377 pcifunc); 4378 4379 goto free_rules; 4380 } 4381 4382 /* PF */ 4383 for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { 4384 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4385 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4386 if (!map) 4387 dev_dbg(rvu->dev, 4388 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4389 __func__, npc_dft_rule_name[i], 4390 pcifunc); 4391 } 4392 4393 free_rules: 4394 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4395 if (blkaddr < 0) 4396 return; 4397 for (int i = 0; i < 4; i++) { 4398 if (ptr[i] == USHRT_MAX) 4399 continue; 4400 4401 mutex_lock(&mcam->lock); 4402 npc_mcam_clear_bit(mcam, ptr[i]); 4403 mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP; 4404 npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false); 4405 mcam->entry2target_pffunc[ptr[i]] = 0x0; 4406 mutex_unlock(&mcam->lock); 4407 4408 rc = npc_cn20k_idx_free(rvu, &ptr[i], 1); 4409 if (rc) { 4410 /* Non recoverable error. Let us WARN and return. Keep system alive to 4411 * enable debugging 4412 */ 4413 WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n", 4414 __func__, pcifunc, ptr[i]); 4415 return; 4416 } 4417 } 4418 4419 mutex_lock(&mcam->lock); 4420 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 4421 for (int i = 0; i < 4; i++) { 4422 if (ptr[i] != rule->entry) 4423 continue; 4424 4425 list_del(&rule->list); 4426 kfree(rule); 4427 break; 4428 } 4429 } 4430 mutex_unlock(&mcam->lock); 4431 } 4432 4433 int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) 4434 { 4435 struct npc_mcam_free_entry_req free_req = { 0 }; 4436 u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; 4437 struct npc_mcam_alloc_entry_req req = { 0 }; 4438 struct npc_mcam_alloc_entry_rsp rsp = { 0 }; 4439 int ret, eidx, i, k, pf, cnt; 4440 struct rvu_pfvf *pfvf; 4441 unsigned long index; 4442 struct msg_rsp free_rsp; 4443 u16 b, m, p, u; 4444 4445 if (!npc_priv) 4446 return 0; 4447 4448 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4449 dev_dbg(rvu->dev, 4450 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4451 __func__, pcifunc); 4452 return 0; 4453 } 4454 4455 /* Check if default rules are already alloced for this pcifunc */ 4456 ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); 4457 if (!ret) { 4458 dev_dbg(rvu->dev, 4459 "%s: default rules are already installed (pcifunc=%#x)\n", 4460 __func__, pcifunc); 4461 dev_dbg(rvu->dev, 4462 "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", 4463 __func__, b, m, p, u); 4464 return 0; 4465 } 4466 4467 /* Set ref index as lowest priority index */ 4468 eidx = 2 * npc_priv->bank_depth - 1; 4469 4470 /* Install only UCAST for VF */ 4471 cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); 4472 4473 /* For VF pcifunc, allocate default mcam indexes by taking 4474 * ref as PF's ucast index. 4475 */ 4476 if (is_vf(pcifunc)) { 4477 pf = rvu_get_pf(rvu->pdev, pcifunc); 4478 pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; 4479 4480 /* Get PF's ucast entry index */ 4481 ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, 4482 NULL, NULL, &pf_ucast); 4483 4484 /* There is no PF rules installed; and VF installation comes 4485 * first. PF may come later. 4486 * TODO: Install PF rules before installing VF rules. 4487 */ 4488 4489 /* Set PF's ucast as ref entry */ 4490 if (!ret) 4491 eidx = pf_ucast; 4492 } 4493 4494 pfvf = rvu_get_pfvf(rvu, pcifunc); 4495 pfvf->hw_prio = NPC_DFT_RULE_PRIO; 4496 4497 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4498 req.kw_type = NPC_MCAM_KEY_X4; 4499 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4500 } else { 4501 req.kw_type = NPC_MCAM_KEY_X2; 4502 req.ref_entry = eidx; 4503 } 4504 4505 req.contig = false; 4506 req.ref_prio = NPC_MCAM_HIGHER_PRIO; 4507 req.count = cnt; 4508 req.hdr.pcifunc = pcifunc; 4509 4510 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4511 4512 /* successfully allocated index */ 4513 if (!ret) { 4514 /* Copy indexes to local array */ 4515 for (i = 0; i < cnt; i++) 4516 mcam_idx[i] = rsp.entry_list[i]; 4517 4518 goto chk_sanity; 4519 } 4520 4521 /* If there is no slots available and request is for PF, 4522 * return error. 4523 */ 4524 if (!is_vf(pcifunc)) { 4525 dev_err(rvu->dev, 4526 "%s: Default index allocation failed for pcifunc=%#x\n", 4527 __func__, pcifunc); 4528 return ret; 4529 } 4530 4531 /* We could not find an index with higher priority index for VF. 4532 * Find rule with lower priority index and set hardware priority 4533 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) 4534 */ 4535 req.contig = false; 4536 req.count = cnt; 4537 req.hdr.pcifunc = pcifunc; 4538 req.ref_prio = NPC_MCAM_LOWER_PRIO; 4539 4540 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4541 req.kw_type = NPC_MCAM_KEY_X4; 4542 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4543 } else { 4544 req.kw_type = NPC_MCAM_KEY_X2; 4545 req.ref_entry = eidx; 4546 } 4547 4548 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4549 if (ret) { 4550 dev_err(rvu->dev, 4551 "%s: Default index allocation failed for pcifunc=%#x\n", 4552 __func__, pcifunc); 4553 return ret; 4554 } 4555 4556 /* Copy indexes to local array */ 4557 for (i = 0; i < cnt; i++) 4558 mcam_idx[i] = rsp.entry_list[i]; 4559 4560 pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; 4561 4562 chk_sanity: 4563 /* LBK */ 4564 if (is_lbk_vf(rvu, pcifunc)) { 4565 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4566 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4567 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4568 if (ret) { 4569 dev_err(rvu->dev, 4570 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4571 __func__, 4572 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4573 pcifunc); 4574 goto err; 4575 } 4576 4577 goto done; 4578 } 4579 4580 /* VF */ 4581 if (is_vf(pcifunc)) { 4582 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4583 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4584 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4585 if (ret) { 4586 dev_err(rvu->dev, 4587 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4588 __func__, 4589 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4590 pcifunc); 4591 goto err; 4592 } 4593 4594 goto done; 4595 } 4596 4597 /* PF */ 4598 for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && 4599 k < cnt; i++, k++) { 4600 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4601 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4602 xa_mk_value(mcam_idx[k]), GFP_KERNEL); 4603 if (ret) { 4604 dev_err(rvu->dev, 4605 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4606 __func__, npc_dft_rule_name[i], 4607 pcifunc); 4608 for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { 4609 index = NPC_DFT_RULE_ID_MK(pcifunc, p); 4610 xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4611 } 4612 goto err; 4613 } 4614 } 4615 4616 done: 4617 return 0; 4618 4619 err: 4620 free_req.hdr.pcifunc = pcifunc; 4621 free_req.all = 1; 4622 ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 4623 if (ret) 4624 dev_err(rvu->dev, 4625 "%s: Error deleting default entries (pcifunc=%#x\n", 4626 __func__, pcifunc); 4627 4628 return -EFAULT; 4629 } 4630 4631 static int npc_priv_init(struct rvu *rvu) 4632 { 4633 struct npc_mcam *mcam = &rvu->hw->mcam; 4634 int blkaddr, num_banks, bank_depth; 4635 int num_subbanks, subbank_depth; 4636 u64 npc_const1, npc_const2 = 0; 4637 struct npc_subbank *sb; 4638 u64 cfg; 4639 int i; 4640 4641 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4642 if (blkaddr < 0) { 4643 dev_err(rvu->dev, "%s: NPC block not implemented\n", 4644 __func__); 4645 return -ENODEV; 4646 } 4647 4648 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 4649 if (npc_const1 & BIT_ULL(63)) 4650 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 4651 4652 num_banks = mcam->banks; 4653 if (!num_banks || num_banks > MAX_NUM_BANKS) { 4654 dev_err(rvu->dev, 4655 "Number of banks(%u) is invalid\n", num_banks); 4656 return -EINVAL; 4657 } 4658 4659 num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); 4660 if (!num_subbanks || num_subbanks > MAX_NUM_SUB_BANKS) { 4661 dev_err(rvu->dev, 4662 "Number of subbanks is invalid %u\n", 4663 num_subbanks); 4664 return -EFAULT; 4665 } 4666 4667 if (num_subbanks & (num_subbanks - 1)) { 4668 dev_err(rvu->dev, 4669 "subbanks cnt(%u) should be a power of 2\n", 4670 num_subbanks); 4671 return -EINVAL; 4672 } 4673 4674 bank_depth = mcam->banksize; 4675 if (!bank_depth || bank_depth % num_subbanks) { 4676 dev_err(rvu->dev, 4677 "Bank depth(%u) should be a multiple of num_subbanks(%u)\n", 4678 bank_depth, num_subbanks); 4679 return -EINVAL; 4680 } 4681 4682 subbank_depth = bank_depth / num_subbanks; 4683 if (!subbank_depth || subbank_depth > MAX_SUBBANK_DEPTH) { 4684 dev_err(rvu->dev, 4685 "Invalid subbank depth %u\n", 4686 subbank_depth); 4687 return -EINVAL; 4688 } 4689 4690 npc_priv = kcalloc(1, sizeof(*npc_priv), GFP_KERNEL); 4691 if (!npc_priv) 4692 return -ENOMEM; 4693 4694 npc_priv->num_banks = num_banks; 4695 npc_priv->num_subbanks = num_subbanks; 4696 npc_priv->bank_depth = bank_depth; 4697 npc_priv->subbank_depth = subbank_depth; 4698 4699 /* Get kex configured key size */ 4700 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); 4701 npc_priv->kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); 4702 4703 dev_info(rvu->dev, 4704 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", 4705 num_banks, bank_depth, num_subbanks, subbank_depth, 4706 npc_kw_name[npc_priv->kw]); 4707 4708 npc_priv->sb = kcalloc(num_subbanks, sizeof(struct npc_subbank), 4709 GFP_KERNEL); 4710 if (!npc_priv->sb) 4711 goto fail1; 4712 4713 xa_init_flags(&npc_priv->xa_sb_used, XA_FLAGS_ALLOC); 4714 xa_init_flags(&npc_priv->xa_sb_free, XA_FLAGS_ALLOC); 4715 xa_init_flags(&npc_priv->xa_idx2pf_map, XA_FLAGS_ALLOC); 4716 xa_init_flags(&npc_priv->xa_pf_map, XA_FLAGS_ALLOC); 4717 xa_init_flags(&npc_priv->xa_pf2dfl_rmap, XA_FLAGS_ALLOC); 4718 xa_init_flags(&npc_priv->xa_idx2vidx_map, XA_FLAGS_ALLOC); 4719 xa_init_flags(&npc_priv->xa_vidx2idx_map, XA_FLAGS_ALLOC); 4720 4721 if (npc_create_srch_order(num_subbanks)) 4722 goto fail2; 4723 4724 npc_populate_restricted_idxs(num_subbanks); 4725 4726 /* Initialize subbanks */ 4727 for (i = 0, sb = npc_priv->sb; i < num_subbanks; i++, sb++) 4728 npc_subbank_init(rvu, sb, i); 4729 4730 /* Get number of pcifuncs in the system */ 4731 npc_priv->pf_cnt = npc_pcifunc_map_create(rvu); 4732 npc_priv->xa_pf2idx_map = kcalloc(npc_priv->pf_cnt, 4733 sizeof(struct xarray), 4734 GFP_KERNEL); 4735 if (!npc_priv->xa_pf2idx_map) 4736 goto fail3; 4737 4738 for (i = 0; i < npc_priv->pf_cnt; i++) 4739 xa_init_flags(&npc_priv->xa_pf2idx_map[i], XA_FLAGS_ALLOC); 4740 4741 INIT_LIST_HEAD(&npc_priv->defrag_lh); 4742 mutex_init(&npc_priv->lock); 4743 4744 return 0; 4745 4746 fail3: 4747 kfree(subbank_srch_order); 4748 subbank_srch_order = NULL; 4749 4750 fail2: 4751 xa_destroy(&npc_priv->xa_sb_used); 4752 xa_destroy(&npc_priv->xa_sb_free); 4753 xa_destroy(&npc_priv->xa_idx2pf_map); 4754 xa_destroy(&npc_priv->xa_pf_map); 4755 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4756 xa_destroy(&npc_priv->xa_idx2vidx_map); 4757 xa_destroy(&npc_priv->xa_vidx2idx_map); 4758 kfree(npc_priv->sb); 4759 npc_priv->sb = NULL; 4760 fail1: 4761 kfree(npc_priv); 4762 npc_priv = NULL; 4763 return -ENOMEM; 4764 } 4765 4766 void npc_cn20k_deinit(struct rvu *rvu) 4767 { 4768 int i; 4769 4770 if (!npc_priv) 4771 return; 4772 4773 xa_destroy(&npc_priv->xa_sb_used); 4774 xa_destroy(&npc_priv->xa_sb_free); 4775 xa_destroy(&npc_priv->xa_idx2pf_map); 4776 xa_destroy(&npc_priv->xa_pf_map); 4777 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4778 xa_destroy(&npc_priv->xa_idx2vidx_map); 4779 xa_destroy(&npc_priv->xa_vidx2idx_map); 4780 4781 for (i = 0; i < npc_priv->pf_cnt; i++) 4782 xa_destroy(&npc_priv->xa_pf2idx_map[i]); 4783 4784 kfree(npc_priv->xa_pf2idx_map); 4785 /* No need to destroy mutex lock as it is 4786 * part of subbank structure 4787 */ 4788 kfree(npc_priv->sb); 4789 kfree(subbank_srch_order); 4790 bitmap_clear(npc_priv->en_map, 0, MAX_NUM_BANKS * MAX_NUM_SUB_BANKS * 4791 MAX_SUBBANK_DEPTH); 4792 npc_defrag_list_clear(); 4793 kfree(npc_priv); 4794 npc_priv = NULL; 4795 } 4796 4797 static int npc_setup_mcam_section(struct rvu *rvu, int key_type) 4798 { 4799 int blkaddr, sec; 4800 4801 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4802 if (blkaddr < 0) { 4803 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4804 return -ENODEV; 4805 } 4806 4807 for (sec = 0; sec < npc_priv->num_subbanks; sec++) 4808 rvu_write64(rvu, blkaddr, 4809 NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); 4810 4811 return 0; 4812 } 4813 4814 int npc_cn20k_init(struct rvu *rvu) 4815 { 4816 int err; 4817 4818 err = npc_priv_init(rvu); 4819 if (err) { 4820 dev_err(rvu->dev, "%s: Error to init\n", 4821 __func__); 4822 return err; 4823 } 4824 4825 err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); 4826 if (err) { 4827 dev_err(rvu->dev, "%s: mcam section configuration failure\n", 4828 __func__); 4829 goto fail; 4830 } 4831 4832 return 0; 4833 4834 fail: 4835 npc_cn20k_deinit(rvu); 4836 return err; 4837 } 4838