1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2026 Marvell. 5 * 6 */ 7 #include <linux/xarray.h> 8 #include <linux/bitfield.h> 9 10 #include "rvu.h" 11 #include "npc.h" 12 #include "npc_profile.h" 13 #include "rvu_npc_hash.h" 14 #include "rvu_npc.h" 15 #include "cn20k/npc.h" 16 #include "cn20k/reg.h" 17 #include "rvu_npc_fs.h" 18 19 static struct npc_priv_t npc_priv = { 20 .num_banks = MAX_NUM_BANKS, 21 }; 22 23 static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { 24 [NPC_MCAM_KEY_DYN] = "DYNAMIC", 25 [NPC_MCAM_KEY_X2] = "X2", 26 [NPC_MCAM_KEY_X4] = "X4", 27 }; 28 29 static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { 30 [NPC_DFT_RULE_PROMISC_ID] = "Promisc", 31 [NPC_DFT_RULE_MCAST_ID] = "Mcast", 32 [NPC_DFT_RULE_BCAST_ID] = "Bcast", 33 [NPC_DFT_RULE_UCAST_ID] = "Ucast", 34 }; 35 36 #define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ 37 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 38 ((key_ofs) & 0x3F)) 39 40 #define NPC_DFT_RULE_ID_MK(pcifunc, id) \ 41 ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) 42 43 #define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ 44 FIELD_GET(GENMASK_ULL(15, 0), rid) 45 46 #define NPC_DFT_RULE_ID_2_ID(rid) \ 47 FIELD_GET(GENMASK_ULL(31, 16), rid) 48 49 #define NPC_DFT_RULE_PRIO 127 50 51 static const char cn20k_def_pfl_name[] = "default"; 52 53 static struct npc_mcam_kex_extr npc_mkex_extr_default = { 54 .mkex_sign = MKEX_CN20K_SIGN, 55 .name = "default", 56 .kpu_version = NPC_KPU_PROFILE_VER, 57 .keyx_cfg = { 58 /* nibble: LA..LE (ltype only) + Error code + Channel */ 59 [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | 60 NPC_PARSE_NIBBLE_INTF_RX | 61 NPC_CN20K_PARSE_NIBBLE_ERRCODE, 62 63 /* nibble: LA..LE (ltype only) */ 64 [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | 65 NPC_CN20K_PARSE_NIBBLE_INTF_TX, 66 }, 67 .intf_extr_lid = { 68 /* Default RX MCAM KEX profile */ 69 [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 70 NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, 71 [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 72 NPC_LID_LC, NPC_LID_LD }, 73 }, 74 .intf_extr_lt = { 75 /* Default RX MCAM KEX profile */ 76 [NIX_INTF_RX] = { 77 [0] = { 78 /* Layer A: Ethernet: */ 79 [NPC_LT_LA_ETHER] = 80 /* DMAC: 6 bytes, KW1[63:15] */ 81 KEX_EXTR_CFG(0x05, 0x0, 0x1, 82 NPC_KEXOF_DMAC + 1), 83 [NPC_LT_LA_CPT_HDR] = 84 /* DMAC: 6 bytes, KW1[63:15] */ 85 KEX_EXTR_CFG(0x05, 0x0, 0x1, 86 NPC_KEXOF_DMAC + 1), 87 }, 88 [1] = { 89 /* Layer A: Ethernet: */ 90 [NPC_LT_LA_ETHER] = 91 /* Ethertype: 2 bytes, KW0[63:48] */ 92 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 93 [NPC_LT_LA_CPT_HDR] = 94 /* Ethertype: 2 bytes, KW0[63:48] */ 95 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 96 }, 97 [2] = { 98 /* Layer B: Single VLAN (CTAG) */ 99 [NPC_LT_LB_CTAG] = 100 /* CTAG VLAN: 2 bytes, KW1[15:0] */ 101 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 102 /* Layer B: Stacked VLAN (STAG|QinQ) */ 103 [NPC_LT_LB_STAG_QINQ] = 104 /* Outer VLAN: 2 bytes, KW1[15:0] */ 105 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 106 [NPC_LT_LB_FDSA] = 107 /* SWITCH PORT: 1 byte, KW1[7:0] */ 108 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), 109 }, 110 [3] = { 111 [NPC_LT_LB_CTAG] = 112 /* Ethertype: 2 bytes, KW0[63:48] */ 113 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 114 [NPC_LT_LB_STAG_QINQ] = 115 /* Ethertype: 2 bytes, KW0[63:48] */ 116 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), 117 [NPC_LT_LB_FDSA] = 118 /* Ethertype: 2 bytes, KW0[63:48] */ 119 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 120 }, 121 [4] = { 122 /* Layer C: IPv4 */ 123 [NPC_LT_LC_IP] = 124 /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ 125 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), 126 /* Layer C: IPv6 */ 127 [NPC_LT_LC_IP6] = 128 /* Everything up to SADDR: 8 bytes, KW3[7:0], 129 * KW2[63:8] 130 */ 131 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), 132 }, 133 [5] = { 134 [NPC_LT_LC_IP] = 135 /* TOS: 1 byte, KW2[7:0] */ 136 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), 137 }, 138 [6] = { 139 /* Layer D:UDP */ 140 [NPC_LT_LD_UDP] = 141 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 142 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 143 /* Layer D:TCP */ 144 [NPC_LT_LD_TCP] = 145 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 146 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 147 }, 148 }, 149 /* Default TX MCAM KEX profile */ 150 [NIX_INTF_TX] = { 151 [0] = { 152 /* Layer A: NIX_INST_HDR_S + Ethernet */ 153 /* NIX appends 8 bytes of NIX_INST_HDR_S at the 154 * start of each TX packet supplied to NPC. 155 */ 156 [NPC_LT_LA_IH_NIX_ETHER] = 157 /* PF_FUNC: 2B , KW0 [47:32] */ 158 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 159 /* Layer A: HiGig2: */ 160 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 161 /* PF_FUNC: 2B , KW0 [47:32] */ 162 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 163 }, 164 [1] = { 165 [NPC_LT_LA_IH_NIX_ETHER] = 166 /* SQ_ID 3 bytes, KW1[63:16] */ 167 KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), 168 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 169 /* VID: 2 bytes, KW1[31:16] */ 170 KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), 171 }, 172 [2] = { 173 /* Layer B: Single VLAN (CTAG) */ 174 [NPC_LT_LB_CTAG] = 175 /* CTAG VLAN[2..3] KW0[63:48] */ 176 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 177 /* Layer B: Stacked VLAN (STAG|QinQ) */ 178 [NPC_LT_LB_STAG_QINQ] = 179 /* Outer VLAN: 2 bytes, KW0[63:48] */ 180 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 181 }, 182 [3] = { 183 [NPC_LT_LB_CTAG] = 184 /* CTAG VLAN[2..3] KW1[15:0] */ 185 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), 186 [NPC_LT_LB_STAG_QINQ] = 187 /* Outer VLAN: 2 Bytes, KW1[15:0] */ 188 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), 189 }, 190 [4] = { 191 /* Layer C: IPv4 */ 192 [NPC_LT_LC_IP] = 193 /* SIP+DIP: 8 bytes, KW2[63:0] */ 194 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), 195 /* Layer C: IPv6 */ 196 [NPC_LT_LC_IP6] = 197 /* Everything up to SADDR: 8 bytes, KW2[63:0] */ 198 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), 199 }, 200 [5] = { 201 /* Layer D:UDP */ 202 [NPC_LT_LD_UDP] = 203 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 204 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 205 /* Layer D:TCP */ 206 [NPC_LT_LD_TCP] = 207 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 208 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 209 }, 210 }, 211 }, 212 }; 213 214 struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) 215 { 216 return &npc_mkex_extr_default; 217 } 218 219 static u16 npc_idx2vidx(u16 idx) 220 { 221 unsigned long index; 222 void *map; 223 u16 vidx; 224 int val; 225 226 vidx = idx; 227 index = idx; 228 229 map = xa_load(&npc_priv.xa_idx2vidx_map, index); 230 if (!map) 231 goto done; 232 233 val = xa_to_value(map); 234 if (val == -1) 235 goto done; 236 237 vidx = val; 238 239 done: 240 return vidx; 241 } 242 243 static bool npc_is_vidx(u16 vidx) 244 { 245 return vidx >= npc_priv.bank_depth * 2; 246 } 247 248 static u16 npc_vidx2idx(u16 vidx) 249 { 250 unsigned long sentinel = (unsigned long)-1; 251 unsigned long index; 252 void *map; 253 int val; 254 u16 idx; 255 256 idx = vidx; 257 index = vidx; 258 259 map = xa_load(&npc_priv.xa_vidx2idx_map, index); 260 if (!map) 261 goto done; 262 263 val = xa_to_value(map); 264 if (val == sentinel) 265 goto done; 266 267 idx = val; 268 269 done: 270 return idx; 271 } 272 273 u16 npc_cn20k_vidx2idx(u16 idx) 274 { 275 if (!npc_priv.init_done) 276 return idx; 277 278 if (!npc_is_vidx(idx)) 279 return idx; 280 281 return npc_vidx2idx(idx); 282 } 283 284 u16 npc_cn20k_idx2vidx(u16 idx) 285 { 286 if (!npc_priv.init_done) 287 return idx; 288 289 if (npc_is_vidx(idx)) 290 return idx; 291 292 return npc_idx2vidx(idx); 293 } 294 295 static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) 296 { 297 u16 mcam_idx; 298 void *map; 299 300 if (!npc_is_vidx(vidx)) { 301 dev_err(rvu->dev, 302 "%s: vidx(%u) does not map to proper mcam idx\n", 303 __func__, vidx); 304 return -ESRCH; 305 } 306 307 mcam_idx = npc_vidx2idx(vidx); 308 309 map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); 310 if (!map) { 311 dev_err(rvu->dev, 312 "%s: vidx(%u) does not map to proper mcam idx\n", 313 __func__, vidx); 314 return -ESRCH; 315 } 316 317 map = xa_erase(&npc_priv.xa_idx2vidx_map, mcam_idx); 318 if (!map) { 319 dev_err(rvu->dev, 320 "%s: vidx(%u) is not valid\n", 321 __func__, vidx); 322 return -ESRCH; 323 } 324 325 if (old_midx) 326 *old_midx = mcam_idx; 327 328 return 0; 329 } 330 331 static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) 332 { 333 u16 old_midx; 334 void *map; 335 int rc; 336 337 if (!npc_is_vidx(vidx)) { 338 dev_err(rvu->dev, 339 "%s: vidx(%u) does not map to proper mcam idx\n", 340 __func__, vidx); 341 return -ESRCH; 342 } 343 344 map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); 345 if (!map) { 346 dev_err(rvu->dev, 347 "%s: vidx(%u) could not be deleted from vidx2idx map\n", 348 __func__, vidx); 349 return -ESRCH; 350 } 351 352 old_midx = xa_to_value(map); 353 354 rc = xa_insert(&npc_priv.xa_vidx2idx_map, vidx, 355 xa_mk_value(new_midx), GFP_KERNEL); 356 if (rc) { 357 dev_err(rvu->dev, 358 "%s: vidx(%u) cannot be added to vidx2idx map\n", 359 __func__, vidx); 360 goto fail1; 361 } 362 363 map = xa_erase(&npc_priv.xa_idx2vidx_map, old_midx); 364 if (!map) { 365 dev_err(rvu->dev, 366 "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 367 __func__, old_midx, vidx); 368 rc = -ESRCH; 369 goto fail2; 370 } 371 372 rc = xa_insert(&npc_priv.xa_idx2vidx_map, new_midx, 373 xa_mk_value(vidx), GFP_KERNEL); 374 if (rc) { 375 dev_err(rvu->dev, 376 "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 377 __func__, new_midx, vidx); 378 goto fail3; 379 } 380 381 return 0; 382 383 fail3: 384 /* Restore vidx at old_midx location */ 385 if (xa_insert(&npc_priv.xa_idx2vidx_map, old_midx, 386 xa_mk_value(vidx), GFP_KERNEL)) 387 dev_err(rvu->dev, 388 "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", 389 __func__, old_midx, vidx); 390 fail2: 391 /* Erase new_midx inserted at vidx */ 392 if (!xa_erase(&npc_priv.xa_vidx2idx_map, vidx)) 393 dev_err(rvu->dev, 394 "%s: Failed to roll back vidx2idx vidx=%u\n", 395 __func__, vidx); 396 397 fail1: 398 /* Restore old_midx at vidx location */ 399 if (xa_insert(&npc_priv.xa_vidx2idx_map, vidx, 400 xa_mk_value(old_midx), GFP_KERNEL)) 401 dev_err(rvu->dev, 402 "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", 403 __func__, old_midx, vidx); 404 405 return rc; 406 } 407 408 static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, 409 u16 *vidx) 410 { 411 int rc, max, min; 412 u32 id; 413 414 /* Virtual index start from maximum mcam index + 1 */ 415 max = npc_priv.bank_depth * 2 * 2 - 1; 416 min = npc_priv.bank_depth * 2; 417 418 rc = xa_alloc(&npc_priv.xa_vidx2idx_map, &id, 419 xa_mk_value(mcam_idx), 420 XA_LIMIT(min, max), GFP_KERNEL); 421 if (rc) { 422 dev_err(rvu->dev, 423 "%s: Failed to add to vidx2idx map (%u)\n", 424 __func__, mcam_idx); 425 goto fail1; 426 } 427 428 rc = xa_insert(&npc_priv.xa_idx2vidx_map, mcam_idx, 429 xa_mk_value(id), GFP_KERNEL); 430 if (rc) { 431 dev_err(rvu->dev, 432 "%s: Failed to add to idx2vidx map (%u)\n", 433 __func__, mcam_idx); 434 goto fail2; 435 } 436 437 if (vidx) 438 *vidx = id; 439 440 return 0; 441 442 fail2: 443 xa_erase(&npc_priv.xa_vidx2idx_map, id); 444 fail1: 445 return rc; 446 } 447 448 static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, 449 const struct npc_kpu_profile_cam *kpucam, 450 int kpm, int entry) 451 { 452 struct npc_kpu_cam cam0 = {0}; 453 struct npc_kpu_cam cam1 = {0}; 454 455 cam1.state = kpucam->state & kpucam->state_mask; 456 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 457 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 458 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 459 460 cam0.state = ~kpucam->state & kpucam->state_mask; 461 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 462 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 463 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 464 465 rvu_write64(rvu, blkaddr, 466 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); 467 rvu_write64(rvu, blkaddr, 468 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); 469 } 470 471 static void 472 npc_config_kpmaction(struct rvu *rvu, int blkaddr, 473 const struct npc_kpu_profile_action *kpuaction, 474 int kpm, int entry, bool pkind) 475 { 476 struct npc_kpm_action0 action0 = {0}; 477 struct npc_kpu_action1 action1 = {0}; 478 u64 reg; 479 480 action1.errlev = kpuaction->errlev; 481 action1.errcode = kpuaction->errcode; 482 action1.dp0_offset = kpuaction->dp0_offset; 483 action1.dp1_offset = kpuaction->dp1_offset; 484 action1.dp2_offset = kpuaction->dp2_offset; 485 486 if (pkind) 487 reg = NPC_AF_PKINDX_ACTION1(entry); 488 else 489 reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); 490 491 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 492 493 action0.byp_count = kpuaction->bypass_count & 0x7; 494 action0.capture_ena = kpuaction->cap_ena & 1; 495 action0.parse_done = kpuaction->parse_done & 1; 496 action0.next_state = kpuaction->next_state & 0xf; 497 action0.capture_lid = kpuaction->lid & 0x7; 498 499 /* Parser functionality will work correctly even though 500 * upper flag bits are silently discarded 501 */ 502 action0.capture_ltype = kpuaction->ltype & 0xf; 503 action0.capture_flags = kpuaction->flags & 0xf; 504 action0.ptr_advance = kpuaction->ptr_advance; 505 506 action0.var_len_offset = kpuaction->offset; 507 action0.var_len_mask = kpuaction->mask; 508 action0.var_len_right = kpuaction->right & 1; 509 action0.var_len_shift = kpuaction->shift & 1; 510 511 if (pkind) 512 reg = NPC_AF_PKINDX_ACTION0(entry); 513 else 514 reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); 515 516 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 517 } 518 519 static void 520 npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, 521 int kpm, int start_entry, 522 const struct npc_kpu_profile *profile) 523 { 524 int entry, num_entries, max_entries; 525 u64 idx; 526 527 if (profile->cam_entries != profile->action_entries) { 528 dev_err(rvu->dev, 529 "kpm%d: CAM and action entries [%d != %d] not equal\n", 530 kpm, profile->cam_entries, profile->action_entries); 531 532 WARN(1, "Fatal error\n"); 533 return; 534 } 535 536 max_entries = rvu->hw->npc_kpu_entries / 2; 537 entry = start_entry; 538 /* Program CAM match entries for previous kpm extracted data */ 539 num_entries = min_t(int, profile->cam_entries, max_entries); 540 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 541 npc_config_kpmcam(rvu, blkaddr, &profile->cam[idx], 542 kpm, entry); 543 544 entry = start_entry; 545 /* Program this kpm's actions */ 546 num_entries = min_t(int, profile->action_entries, max_entries); 547 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 548 npc_config_kpmaction(rvu, blkaddr, &profile->action[idx], 549 kpm, entry, false); 550 } 551 552 static void 553 npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) 554 { 555 u64 entry_mask; 556 557 entry_mask = npc_enable_mask(num_entries); 558 /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ 559 if (!rvu->kpu.custom) 560 entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); 561 rvu_write64(rvu, blkaddr, 562 NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); 563 if (num_entries <= 64) { 564 /* Disable all the entries in W1, W2 and W3 */ 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), 567 npc_enable_mask(0)); 568 rvu_write64(rvu, blkaddr, 569 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 570 npc_enable_mask(0)); 571 rvu_write64(rvu, blkaddr, 572 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 573 npc_enable_mask(0)); 574 return; 575 } 576 577 num_entries = num_entries - 64; 578 entry_mask = npc_enable_mask(num_entries); 579 rvu_write64(rvu, blkaddr, 580 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); 581 if (num_entries <= 64) { 582 /* Disable all the entries in W2 and W3 */ 583 rvu_write64(rvu, blkaddr, 584 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 585 npc_enable_mask(0)); 586 rvu_write64(rvu, blkaddr, 587 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 588 npc_enable_mask(0)); 589 return; 590 } 591 592 num_entries = num_entries - 64; 593 entry_mask = npc_enable_mask(num_entries); 594 rvu_write64(rvu, blkaddr, 595 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); 596 if (num_entries <= 64) { 597 /* Disable all the entries in W3 */ 598 rvu_write64(rvu, blkaddr, 599 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 600 npc_enable_mask(0)); 601 return; 602 } 603 604 num_entries = num_entries - 64; 605 entry_mask = npc_enable_mask(num_entries); 606 rvu_write64(rvu, blkaddr, 607 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); 608 } 609 610 #define KPU_OFFSET 8 611 static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) 612 { 613 const struct npc_kpu_profile *profile1, *profile2; 614 int idx, total_cam_entries; 615 616 for (idx = 0; idx < num_kpms; idx++) { 617 profile1 = &rvu->kpu.kpu[idx]; 618 npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); 619 profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; 620 npc_program_single_kpm_profile(rvu, blkaddr, idx, 621 profile1->cam_entries, 622 profile2); 623 total_cam_entries = profile1->cam_entries + 624 profile2->cam_entries; 625 npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); 626 rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), 627 profile1->cam_entries); 628 /* Enable the KPUs associated with this KPM */ 629 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); 630 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), 631 0x01); 632 } 633 } 634 635 void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) 636 { 637 struct rvu_hwinfo *hw = rvu->hw; 638 int num_pkinds, idx; 639 640 /* Disable all KPMs and their entries */ 641 for (idx = 0; idx < hw->npc_kpms; idx++) { 642 rvu_write64(rvu, blkaddr, 643 NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); 644 rvu_write64(rvu, blkaddr, 645 NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); 646 rvu_write64(rvu, blkaddr, 647 NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); 648 rvu_write64(rvu, blkaddr, 649 NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); 650 } 651 652 for (idx = 0; idx < hw->npc_kpus; idx++) 653 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 654 655 /* Load and customize KPU profile. */ 656 npc_load_kpu_profile(rvu); 657 658 /* Configure KPU and KPM mapping for second pass */ 659 rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); 660 661 /* First program IKPU profile i.e PKIND configs. 662 * Check HW max count to avoid configuring junk or 663 * writing to unsupported CSR addresses. 664 */ 665 num_pkinds = rvu->kpu.pkinds; 666 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 667 668 for (idx = 0; idx < num_pkinds; idx++) 669 npc_config_kpmaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 670 0, idx, true); 671 672 /* Program KPM CAM and Action profiles */ 673 npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); 674 } 675 676 struct npc_priv_t *npc_priv_get(void) 677 { 678 return &npc_priv; 679 } 680 681 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 682 struct npc_mcam_kex_extr *mkex_extr, 683 u8 intf) 684 { 685 u8 num_extr = rvu->hw->npc_kex_extr; 686 int extr, lt; 687 u64 val; 688 689 if (is_npc_intf_tx(intf)) 690 return; 691 692 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 693 mkex_extr->keyx_cfg[NIX_INTF_RX]); 694 695 /* Program EXTRACTOR */ 696 for (extr = 0; extr < num_extr; extr++) 697 rvu_write64(rvu, blkaddr, 698 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 699 mkex_extr->intf_extr_lid[intf][extr]); 700 701 /* Program EXTRACTOR_LTYPE */ 702 for (extr = 0; extr < num_extr; extr++) { 703 for (lt = 0; lt < NPC_MAX_LT; lt++) { 704 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 705 CN20K_SET_EXTR_LT(intf, extr, lt, val); 706 } 707 } 708 } 709 710 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 711 struct npc_mcam_kex_extr *mkex_extr, 712 u8 intf) 713 { 714 u8 num_extr = rvu->hw->npc_kex_extr; 715 int extr, lt; 716 u64 val; 717 718 if (is_npc_intf_rx(intf)) 719 return; 720 721 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 722 mkex_extr->keyx_cfg[NIX_INTF_TX]); 723 724 /* Program EXTRACTOR */ 725 for (extr = 0; extr < num_extr; extr++) 726 rvu_write64(rvu, blkaddr, 727 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 728 mkex_extr->intf_extr_lid[intf][extr]); 729 730 /* Program EXTRACTOR_LTYPE */ 731 for (extr = 0; extr < num_extr; extr++) { 732 for (lt = 0; lt < NPC_MAX_LT; lt++) { 733 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 734 CN20K_SET_EXTR_LT(intf, extr, lt, val); 735 } 736 } 737 } 738 739 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 740 struct npc_mcam_kex_extr *mkex_extr) 741 { 742 struct rvu_hwinfo *hw = rvu->hw; 743 u8 intf; 744 745 for (intf = 0; intf < hw->npc_intfs; intf++) { 746 npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); 747 npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); 748 } 749 750 /* Programme mkex hash profile */ 751 npc_program_mkex_hash(rvu, blkaddr); 752 } 753 754 void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, 755 const char *mkex_profile) 756 { 757 struct npc_mcam_kex_extr *mcam_kex_extr; 758 struct device *dev = &rvu->pdev->dev; 759 void __iomem *mkex_prfl_addr = NULL; 760 u64 prfl_sz; 761 int ret; 762 763 /* If user not selected mkex profile */ 764 if (rvu->kpu_fwdata_sz || 765 !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) 766 goto program_mkex_extr; 767 768 /* Setting up the mapping for mkex profile image */ 769 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 770 if (ret < 0) 771 goto program_mkex_extr; 772 773 mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; 774 775 while (((s64)prfl_sz > 0) && 776 (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { 777 /* Compare with mkex mod_param name string */ 778 if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && 779 !strncmp(mcam_kex_extr->name, mkex_profile, 780 MKEX_NAME_LEN)) { 781 rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; 782 goto program_mkex_extr; 783 } 784 785 mcam_kex_extr++; 786 prfl_sz -= sizeof(struct npc_mcam_kex_extr); 787 } 788 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 789 rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 790 791 program_mkex_extr: 792 dev_info(rvu->dev, "Using %s mkex profile\n", 793 rvu->kpu.mcam_kex_prfl.mkex_extr->name); 794 /* Program selected mkex profile */ 795 npc_program_mkex_profile(rvu, blkaddr, 796 rvu->kpu.mcam_kex_prfl.mkex_extr); 797 if (mkex_prfl_addr) 798 iounmap(mkex_prfl_addr); 799 } 800 801 int 802 npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, 803 int index, bool enable) 804 { 805 struct npc_mcam *mcam = &rvu->hw->mcam; 806 int mcam_idx = index % mcam->banksize; 807 int bank = index / mcam->banksize; 808 u64 cfg, hw_prio; 809 u8 kw_type; 810 811 if (index < 0 || index >= mcam->total_entries) 812 return -EINVAL; 813 814 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 815 return -EINVAL; 816 817 if (kw_type == NPC_MCAM_KEY_X2) { 818 cfg = rvu_read64(rvu, blkaddr, 819 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 820 bank)); 821 hw_prio = cfg & GENMASK_ULL(30, 24); 822 cfg = enable ? 1 : 0; 823 cfg |= hw_prio; 824 rvu_write64(rvu, blkaddr, 825 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 826 cfg); 827 return 0; 828 } 829 830 /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks 831 * need to be programmed with the same value. 832 */ 833 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 834 cfg = rvu_read64(rvu, blkaddr, 835 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 836 bank)); 837 hw_prio = cfg & GENMASK_ULL(30, 24); 838 cfg = enable ? 1 : 0; 839 cfg |= hw_prio; 840 rvu_write64(rvu, blkaddr, 841 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 842 cfg); 843 } 844 845 return 0; 846 } 847 848 static void 849 npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index) 850 { 851 rvu_write64(rvu, blkaddr, 852 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 853 0); 854 rvu_write64(rvu, blkaddr, 855 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 856 0); 857 858 rvu_write64(rvu, blkaddr, 859 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); 860 rvu_write64(rvu, blkaddr, 861 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); 862 863 rvu_write64(rvu, blkaddr, 864 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); 865 rvu_write64(rvu, blkaddr, 866 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); 867 868 rvu_write64(rvu, blkaddr, 869 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); 870 rvu_write64(rvu, blkaddr, 871 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); 872 873 rvu_write64(rvu, blkaddr, 874 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); 875 rvu_write64(rvu, blkaddr, 876 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); 877 878 /* Clear corresponding stats register */ 879 rvu_write64(rvu, blkaddr, 880 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 881 } 882 883 int 884 npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx) 885 { 886 struct npc_mcam *mcam = &rvu->hw->mcam; 887 int bank = npc_get_bank(mcam, mcam_idx); 888 u8 kw_type; 889 int index; 890 891 if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type)) 892 return -EINVAL; 893 894 index = mcam_idx & (mcam->banksize - 1); 895 896 if (kw_type == NPC_MCAM_KEY_X2) { 897 npc_clear_x2_entry(rvu, blkaddr, bank, index); 898 return 0; 899 } 900 901 /* For NPC_MCAM_KEY_X4 keys, both the banks 902 * need to be programmed with the same value. 903 */ 904 for (bank = 0; bank < mcam->banks_per_entry; bank++) 905 npc_clear_x2_entry(rvu, blkaddr, bank, index); 906 907 return 0; 908 } 909 910 static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, 911 u64 *cam0, u64 *cam1) 912 { 913 u64 kw_mask; 914 915 /* The two banks of every MCAM entry are used as a single double-wide 916 * entry that is compared with the search key as follows: 917 * 918 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] 919 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] 920 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] 921 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] 922 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] 923 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] 924 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] 925 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] 926 */ 927 *cam1 = entry->kw[idx]; 928 kw_mask = entry->kw_mask[idx]; 929 *cam1 &= kw_mask; 930 *cam0 = ~*cam1 & kw_mask; 931 } 932 933 /*------------------------------------------------------------------------- 934 *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | 935 * in | | | | || | | 936 * profile| len | | | ||len | type | 937 *------------------------------------------------------------------------- 938 *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | 939 *------------------------------------------------------------------------| 940 *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 941 *------------------------------------------------------------------------| 942 *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 943 *------------------------------------------------------------------------| 944 *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 945 *------------------------------------------------------------------------| 946 *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 947 *------------------------------------------------------------------------| 948 *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 949 *------------------------------------------------------------------------| 950 *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 951 *------------------------------------------------------------------------| 952 * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | 953 *------------------------------------------------------------------------| 954 */ 955 static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, 956 int blkaddr, int index, u8 intf, 957 struct cn20k_mcam_entry *entry, 958 int bank, u8 kw_type, int kw, 959 u8 req_kw_type) 960 { 961 u64 intf_ext = 0, intf_ext_mask = 0; 962 u8 tx_intf_mask = ~intf & 0x3; 963 u8 tx_intf = intf, kex_type; 964 u8 kw_type_mask = ~kw_type; 965 u64 cam0, cam1, kex_cfg; 966 967 if (is_npc_intf_tx(intf)) { 968 /* Last bit must be set and rest don't care 969 * for TX interfaces 970 */ 971 tx_intf_mask = 0x1; 972 tx_intf = intf & tx_intf_mask; 973 tx_intf_mask = ~tx_intf & tx_intf_mask; 974 } 975 976 kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 977 kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; 978 if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && 979 kw_type == NPC_MCAM_KEY_X2) { 980 kw_type = 0; 981 kw_type_mask = 0; 982 } 983 984 /* Say, we need to write x2 keyword in an x4 subbank. 985 * req_kw_type will be x2, and kw_type will be x4. 986 * So in the case ignore kw bits in mcam. 987 */ 988 if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { 989 kw_type = 0; 990 kw_type_mask = 0; 991 } 992 993 intf_ext = ((u64)kw_type << 16) | tx_intf; 994 intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | 995 tx_intf_mask; 996 rvu_write64(rvu, blkaddr, 997 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 998 intf_ext); 999 rvu_write64(rvu, blkaddr, 1000 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 1001 intf_ext_mask); 1002 1003 /* Set the match key */ 1004 npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); 1005 rvu_write64(rvu, blkaddr, 1006 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 1007 cam1); 1008 rvu_write64(rvu, blkaddr, 1009 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 1010 cam0); 1011 1012 npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); 1013 rvu_write64(rvu, blkaddr, 1014 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 1015 cam1); 1016 rvu_write64(rvu, blkaddr, 1017 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 1018 cam0); 1019 1020 npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); 1021 rvu_write64(rvu, blkaddr, 1022 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 1023 cam1); 1024 rvu_write64(rvu, blkaddr, 1025 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 1026 cam0); 1027 1028 npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); 1029 rvu_write64(rvu, blkaddr, 1030 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 1031 cam1); 1032 rvu_write64(rvu, blkaddr, 1033 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 1034 cam0); 1035 } 1036 1037 static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, 1038 int blkaddr, int index, u8 intf, 1039 struct cn20k_mcam_entry *entry, 1040 u8 kw_type, u8 req_kw_type) 1041 { 1042 int kw = 0, bank; 1043 1044 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) 1045 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1046 index, intf, 1047 entry, bank, kw_type, 1048 kw, req_kw_type); 1049 } 1050 1051 int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, 1052 u8 intf, struct cn20k_mcam_entry *entry, 1053 bool enable, u8 hw_prio, u8 req_kw_type) 1054 { 1055 struct npc_mcam *mcam = &rvu->hw->mcam; 1056 int mcam_idx = index % mcam->banksize; 1057 int bank = index / mcam->banksize; 1058 u64 bank_cfg = (u64)hw_prio << 24; 1059 int kw = 0; 1060 u8 kw_type; 1061 1062 if (index < 0 || index >= mcam->total_entries) 1063 return -EINVAL; 1064 1065 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1066 return -EINVAL; 1067 1068 /* Disable before mcam entry update */ 1069 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false)) 1070 return -EINVAL; 1071 1072 /* CAM1 takes the comparison value and 1073 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 1074 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 1075 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 1076 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 1077 */ 1078 if (kw_type == NPC_MCAM_KEY_X2) { 1079 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1080 npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx); 1081 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1082 mcam_idx, intf, entry, 1083 bank, kw_type, kw, req_kw_type); 1084 /* Set 'action' */ 1085 rvu_write64(rvu, blkaddr, 1086 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1087 bank, 0), 1088 entry->action); 1089 1090 /* Set 'action2' for inline receive */ 1091 rvu_write64(rvu, blkaddr, 1092 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1093 bank, 2), 1094 entry->action2); 1095 1096 /* Set TAG 'action' */ 1097 rvu_write64(rvu, blkaddr, 1098 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1099 bank, 1), 1100 entry->vtag_action); 1101 1102 /* Set HW priority */ 1103 rvu_write64(rvu, blkaddr, 1104 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1105 bank_cfg); 1106 1107 } else { 1108 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1109 npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx); 1110 npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx); 1111 1112 npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, 1113 mcam_idx, intf, entry, 1114 kw_type, req_kw_type); 1115 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1116 /* Set 'action' */ 1117 rvu_write64(rvu, blkaddr, 1118 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1119 bank, 0), 1120 entry->action); 1121 1122 /* Set TAG 'action' */ 1123 rvu_write64(rvu, blkaddr, 1124 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1125 bank, 1), 1126 entry->vtag_action); 1127 1128 /* Set 'action2' for inline receive */ 1129 rvu_write64(rvu, blkaddr, 1130 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1131 bank, 2), 1132 entry->action2); 1133 1134 /* Set HW priority */ 1135 rvu_write64(rvu, blkaddr, 1136 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1137 bank_cfg); 1138 } 1139 } 1140 1141 /* TODO: */ 1142 /* PF installing VF rule */ 1143 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 1144 return -EINVAL; 1145 1146 return 0; 1147 } 1148 1149 int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) 1150 { 1151 struct npc_mcam *mcam = &rvu->hw->mcam; 1152 u64 cfg, sreg, dreg, soff, doff; 1153 u8 src_kwtype, dest_kwtype; 1154 int bank, i, sb, db; 1155 int dbank, sbank; 1156 1157 if (src >= mcam->total_entries || dest >= mcam->total_entries) 1158 return -EINVAL; 1159 1160 dbank = npc_get_bank(mcam, dest); 1161 sbank = npc_get_bank(mcam, src); 1162 1163 if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype)) 1164 return -EINVAL; 1165 1166 if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype)) 1167 return -EINVAL; 1168 1169 if (src_kwtype != dest_kwtype) 1170 return -EINVAL; 1171 1172 src &= (mcam->banksize - 1); 1173 dest &= (mcam->banksize - 1); 1174 1175 /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ 1176 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1177 sb = sbank + bank; 1178 sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); 1179 db = dbank + bank; 1180 dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); 1181 for (i = 0; i < 10; i++) { 1182 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 1183 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 1184 } 1185 1186 /* Copy action */ 1187 for (i = 0; i < 3; i++) { 1188 soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, 1189 sb, i); 1190 cfg = rvu_read64(rvu, blkaddr, soff); 1191 1192 doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, 1193 i); 1194 rvu_write64(rvu, blkaddr, doff, cfg); 1195 } 1196 1197 /* Copy bank configuration */ 1198 cfg = rvu_read64(rvu, blkaddr, 1199 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); 1200 rvu_write64(rvu, blkaddr, 1201 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); 1202 if (src_kwtype == NPC_MCAM_KEY_X2) 1203 break; 1204 } 1205 1206 return 0; 1207 } 1208 1209 static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, 1210 u64 cam0, u64 cam1) 1211 { 1212 entry->kw[idx] = cam1; 1213 entry->kw_mask[idx] = cam1 ^ cam0; 1214 } 1215 1216 int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, 1217 struct cn20k_mcam_entry *entry, 1218 u8 *intf, u8 *ena, u8 *hw_prio) 1219 { 1220 struct npc_mcam *mcam = &rvu->hw->mcam; 1221 u64 cam0, cam1, bank_cfg, cfg; 1222 int kw = 0, bank; 1223 u8 kw_type; 1224 1225 if (index >= mcam->total_entries) 1226 return -EINVAL; 1227 1228 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1229 return -EINVAL; 1230 1231 bank = npc_get_bank(mcam, index); 1232 index &= (mcam->banksize - 1); 1233 1234 cfg = rvu_read64(rvu, blkaddr, 1235 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0)); 1236 entry->action = cfg; 1237 1238 cfg = rvu_read64(rvu, blkaddr, 1239 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1)); 1240 entry->vtag_action = cfg; 1241 1242 cfg = rvu_read64(rvu, blkaddr, 1243 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2)); 1244 entry->action2 = cfg; 1245 1246 cfg = rvu_read64(rvu, blkaddr, 1247 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, 1248 bank, 1)) & 3; 1249 *intf = cfg; 1250 1251 bank_cfg = rvu_read64(rvu, blkaddr, 1252 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); 1253 *ena = bank_cfg & 0x1; 1254 *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; 1255 if (kw_type == NPC_MCAM_KEY_X2) { 1256 cam1 = rvu_read64(rvu, blkaddr, 1257 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1258 bank, 1259 1)); 1260 cam0 = rvu_read64(rvu, blkaddr, 1261 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1262 bank, 1263 0)); 1264 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1265 1266 cam1 = rvu_read64(rvu, blkaddr, 1267 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1268 bank, 1269 1)); 1270 cam0 = rvu_read64(rvu, blkaddr, 1271 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1272 bank, 1273 0)); 1274 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1275 1276 cam1 = rvu_read64(rvu, blkaddr, 1277 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1278 bank, 1279 1)); 1280 cam0 = rvu_read64(rvu, blkaddr, 1281 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1282 bank, 1283 0)); 1284 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1285 1286 cam1 = rvu_read64(rvu, blkaddr, 1287 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1288 bank, 1289 1)); 1290 cam0 = rvu_read64(rvu, blkaddr, 1291 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1292 bank, 1293 0)); 1294 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1295 return 0; 1296 } 1297 1298 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { 1299 cam1 = rvu_read64(rvu, blkaddr, 1300 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1301 bank, 1302 1)); 1303 cam0 = rvu_read64(rvu, blkaddr, 1304 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1305 bank, 1306 0)); 1307 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1308 1309 cam1 = rvu_read64(rvu, blkaddr, 1310 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1311 bank, 1312 1)); 1313 cam0 = rvu_read64(rvu, blkaddr, 1314 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1315 bank, 1316 0)); 1317 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1318 1319 cam1 = rvu_read64(rvu, blkaddr, 1320 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1321 bank, 1322 1)); 1323 cam0 = rvu_read64(rvu, blkaddr, 1324 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1325 bank, 1326 0)); 1327 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1328 1329 cam1 = rvu_read64(rvu, blkaddr, 1330 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1331 bank, 1332 1)); 1333 cam0 = rvu_read64(rvu, blkaddr, 1334 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1335 bank, 1336 0)); 1337 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1338 } 1339 1340 return 0; 1341 } 1342 1343 int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, 1344 struct npc_cn20k_mcam_write_entry_req *req, 1345 struct msg_rsp *rsp) 1346 { 1347 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1348 struct npc_mcam *mcam = &rvu->hw->mcam; 1349 u16 pcifunc = req->hdr.pcifunc; 1350 int blkaddr, rc; 1351 u8 nix_intf; 1352 1353 req->entry = npc_cn20k_vidx2idx(req->entry); 1354 1355 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1356 if (blkaddr < 0) 1357 return NPC_MCAM_INVALID_REQ; 1358 1359 mutex_lock(&mcam->lock); 1360 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1361 if (rc) 1362 goto exit; 1363 1364 if (!is_npc_interface_valid(rvu, req->intf)) { 1365 rc = NPC_MCAM_INVALID_REQ; 1366 goto exit; 1367 } 1368 1369 if (is_npc_intf_tx(req->intf)) 1370 nix_intf = pfvf->nix_tx_intf; 1371 else 1372 nix_intf = pfvf->nix_rx_intf; 1373 1374 /* For AF installed rules, the nix_intf should be set to target NIX */ 1375 if (is_pffunc_af(req->hdr.pcifunc)) 1376 nix_intf = req->intf; 1377 1378 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, 1379 &req->entry_data, req->enable_entry, 1380 req->hw_prio, req->req_kw_type); 1381 1382 exit: 1383 mutex_unlock(&mcam->lock); 1384 return rc; 1385 } 1386 1387 int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, 1388 struct npc_mcam_read_entry_req *req, 1389 struct npc_cn20k_mcam_read_entry_rsp *rsp) 1390 { 1391 struct npc_mcam *mcam = &rvu->hw->mcam; 1392 u16 pcifunc = req->hdr.pcifunc; 1393 int blkaddr, rc; 1394 1395 req->entry = npc_cn20k_vidx2idx(req->entry); 1396 1397 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1398 if (blkaddr < 0) 1399 return NPC_MCAM_INVALID_REQ; 1400 1401 mutex_lock(&mcam->lock); 1402 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1403 if (rc) 1404 goto fail; 1405 1406 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, 1407 &rsp->entry_data, &rsp->intf, 1408 &rsp->enable, &rsp->hw_prio); 1409 fail: 1410 mutex_unlock(&mcam->lock); 1411 return rc; 1412 } 1413 1414 int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, 1415 struct npc_cn20k_mcam_alloc_and_write_entry_req *req, 1416 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 1417 { 1418 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1419 struct npc_mcam_free_entry_req free_req = { 0 }; 1420 struct npc_mcam_alloc_entry_req entry_req; 1421 struct npc_mcam_alloc_entry_rsp entry_rsp; 1422 struct npc_mcam *mcam = &rvu->hw->mcam; 1423 u16 entry = NPC_MCAM_ENTRY_INVALID; 1424 struct msg_rsp free_rsp; 1425 int blkaddr, rc, err; 1426 u8 nix_intf; 1427 1428 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1429 if (blkaddr < 0) 1430 return NPC_MCAM_INVALID_REQ; 1431 1432 if (!is_npc_interface_valid(rvu, req->intf)) 1433 return NPC_MCAM_INVALID_REQ; 1434 1435 /* Try to allocate a MCAM entry */ 1436 entry_req.hdr.pcifunc = req->hdr.pcifunc; 1437 entry_req.contig = true; 1438 entry_req.ref_prio = req->ref_prio; 1439 entry_req.ref_entry = req->ref_entry; 1440 entry_req.count = 1; 1441 entry_req.virt = req->virt; 1442 1443 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 1444 &entry_req, &entry_rsp); 1445 if (rc) 1446 return rc; 1447 1448 if (!entry_rsp.count) 1449 return NPC_MCAM_ALLOC_FAILED; 1450 1451 /* entry_req.count is 1, so single entry is allocated */ 1452 entry = npc_cn20k_vidx2idx(entry_rsp.entry); 1453 1454 mutex_lock(&mcam->lock); 1455 1456 if (is_npc_intf_tx(req->intf)) 1457 nix_intf = pfvf->nix_tx_intf; 1458 else 1459 nix_intf = pfvf->nix_rx_intf; 1460 1461 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, 1462 &req->entry_data, req->enable_entry, 1463 req->hw_prio, req->req_kw_type); 1464 1465 mutex_unlock(&mcam->lock); 1466 1467 if (rc) { 1468 free_req.hdr.pcifunc = req->hdr.pcifunc; 1469 free_req.entry = entry_rsp.entry; 1470 err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 1471 if (err) 1472 dev_err(rvu->dev, 1473 "%s: Error to free mcam idx %u\n", 1474 __func__, entry_rsp.entry); 1475 return rc; 1476 } 1477 1478 rsp->entry = entry_rsp.entry; 1479 return 0; 1480 } 1481 1482 static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) 1483 { 1484 if (is_lbk_vf(rvu, pcifunc)) 1485 return NIXLF_PROMISC_ENTRY; 1486 1487 return NIXLF_UCAST_ENTRY; 1488 } 1489 1490 int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, 1491 struct msg_req *req, 1492 struct npc_cn20k_mcam_read_base_rule_rsp *rsp) 1493 { 1494 struct npc_mcam *mcam = &rvu->hw->mcam; 1495 int index, blkaddr, nixlf, rc = 0; 1496 u16 pcifunc = req->hdr.pcifunc; 1497 u8 intf, enable, hw_prio; 1498 struct rvu_pfvf *pfvf; 1499 int rl_type; 1500 1501 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1502 if (blkaddr < 0) 1503 return NPC_MCAM_INVALID_REQ; 1504 1505 /* Return the channel number in case of PF */ 1506 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 1507 pfvf = rvu_get_pfvf(rvu, pcifunc); 1508 rsp->entry.kw[0] = pfvf->rx_chan_base; 1509 rsp->entry.kw_mask[0] = 0xFFFULL; 1510 goto out; 1511 } 1512 1513 /* Find the pkt steering rule installed by PF to this VF */ 1514 mutex_lock(&mcam->lock); 1515 for (index = 0; index < mcam->bmap_entries; index++) { 1516 if (mcam->entry2target_pffunc[index] == pcifunc) 1517 goto read_entry; 1518 } 1519 1520 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 1521 if (rc < 0) { 1522 mutex_unlock(&mcam->lock); 1523 goto out; 1524 } 1525 1526 rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); 1527 1528 /* Read the default ucast entry if there is no pkt steering rule */ 1529 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); 1530 if (index < 0) { 1531 mutex_unlock(&mcam->lock); 1532 goto out; 1533 } 1534 1535 read_entry: 1536 /* Read the mcam entry */ 1537 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index, 1538 &rsp->entry, &intf, 1539 &enable, &hw_prio); 1540 mutex_unlock(&mcam->lock); 1541 out: 1542 return rc; 1543 } 1544 1545 static u8 npc_map2cn20k_flag(u8 flag) 1546 { 1547 switch (flag) { 1548 case NPC_F_LC_U_IP_FRAG: 1549 return NPC_CN20K_F_LC_L_IP_FRAG; 1550 1551 case NPC_F_LC_U_IP6_FRAG: 1552 return NPC_CN20K_F_LC_L_IP6_FRAG; 1553 1554 case NPC_F_LC_L_6TO4: 1555 return NPC_CN20K_F_LC_L_6TO4; 1556 1557 case NPC_F_LC_L_MPLS_IN_IP: 1558 return NPC_CN20K_F_LC_U_MPLS_IN_IP; 1559 1560 case NPC_F_LC_L_IP6_TUN_IP6: 1561 return NPC_CN20K_F_LC_U_IP6_TUN_IP6; 1562 1563 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1564 return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; 1565 1566 default: 1567 break; 1568 } 1569 1570 WARN(1, "%s: Invalid flag=%u\n", __func__, flag); 1571 return 0xff; 1572 } 1573 1574 static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act) 1575 { 1576 u8 ltype, val; 1577 1578 if (act->lid != NPC_LID_LC) 1579 return; 1580 1581 ltype = act->ltype; 1582 if (ltype != NPC_LT_LC_IP && 1583 ltype != NPC_LT_LC_IP6 && 1584 ltype != NPC_LT_LC_IP_OPT && 1585 ltype != NPC_LT_LC_IP6_EXT) 1586 return; 1587 1588 switch (act->flags) { 1589 case NPC_F_LC_U_IP_FRAG: 1590 case NPC_F_LC_U_IP6_FRAG: 1591 case NPC_F_LC_L_6TO4: 1592 case NPC_F_LC_L_MPLS_IN_IP: 1593 case NPC_F_LC_L_IP6_TUN_IP6: 1594 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1595 val = npc_map2cn20k_flag(act->flags); 1596 if (val != 0xFF) 1597 act->flags = val; 1598 break; 1599 default: 1600 break; 1601 } 1602 } 1603 1604 void 1605 npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, 1606 struct npc_kpu_profile_adapter *pfl) 1607 { 1608 struct npc_kpu_profile_action *action; 1609 int entries; 1610 1611 for (int i = 0; i < pfl->kpus; i++) { 1612 action = pfl->kpu[i].action; 1613 entries = pfl->kpu[i].action_entries; 1614 1615 for (int j = 0; j < entries; j++) 1616 npc_cn20k_translate_action_flags(&action[j]); 1617 } 1618 } 1619 1620 int npc_cn20k_apply_custom_kpu(struct rvu *rvu, 1621 struct npc_kpu_profile_adapter *profile) 1622 { 1623 size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1624 struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1625 struct npc_kpu_profile_action *action; 1626 struct npc_kpu_profile_cam *cam; 1627 struct npc_kpu_fwdata *fw_kpu; 1628 size_t offset = 0; 1629 u16 kpu, entry; 1630 int entries; 1631 1632 hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1633 1634 if (rvu->kpu_fwdata_sz < hdr_sz) { 1635 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1636 return -EINVAL; 1637 } 1638 1639 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1640 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1641 fw->signature); 1642 return -EINVAL; 1643 } 1644 1645 /* Verify if the using known profile structure */ 1646 if (NPC_KPU_VER_MAJ(profile->version) > 1647 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1648 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1649 NPC_KPU_VER_MAJ(profile->version), 1650 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1651 return -EINVAL; 1652 } 1653 1654 /* Verify if profile is aligned with the required kernel changes */ 1655 if (NPC_KPU_VER_MIN(profile->version) < 1656 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1657 dev_warn(rvu->dev, 1658 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1659 NPC_KPU_VER_MAJ(profile->version), 1660 NPC_KPU_VER_MIN(profile->version), 1661 NPC_KPU_VER_PATCH(profile->version), 1662 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1663 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1664 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1665 return -EINVAL; 1666 } 1667 1668 /* Verify if profile fits the HW */ 1669 if (fw->kpus > profile->kpus) { 1670 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1671 profile->kpus); 1672 return -EINVAL; 1673 } 1674 1675 profile->mcam_kex_prfl.mkex_extr = &fw->mkex; 1676 if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { 1677 dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", 1678 profile->mcam_kex_prfl.mkex_extr->mkex_sign); 1679 return -EINVAL; 1680 } 1681 1682 profile->custom = 1; 1683 profile->name = fw->name; 1684 profile->version = le64_to_cpu(fw->version); 1685 profile->lt_def = &fw->lt_def; 1686 1687 for (kpu = 0; kpu < fw->kpus; kpu++) { 1688 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1689 if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) 1690 dev_warn(rvu->dev, 1691 "Too many custom entries on KPU%d: %d > %d\n", 1692 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1693 entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1694 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1695 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1696 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1697 offset += fw_kpu->entries * sizeof(*action); 1698 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1699 dev_warn(rvu->dev, 1700 "Profile size mismatch on KPU%i parsing.\n", 1701 kpu + 1); 1702 return -EINVAL; 1703 } 1704 1705 for (entry = 0; entry < entries; entry++) { 1706 profile->kpu[kpu].cam[entry] = cam[entry]; 1707 profile->kpu[kpu].action[entry] = action[entry]; 1708 npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]); 1709 } 1710 } 1711 1712 return 0; 1713 } 1714 1715 int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) 1716 { 1717 struct npc_subbank *sb; 1718 int bank_off, sb_id; 1719 1720 /* mcam_idx should be less than (2 * bank depth) */ 1721 if (mcam_idx >= npc_priv.bank_depth * 2) { 1722 dev_err(rvu->dev, "%s: bad params\n", 1723 __func__); 1724 return -EINVAL; 1725 } 1726 1727 /* find mcam offset per bank */ 1728 bank_off = mcam_idx & (npc_priv.bank_depth - 1); 1729 1730 /* Find subbank id */ 1731 sb_id = bank_off / npc_priv.subbank_depth; 1732 1733 /* Check if subbank id is more than maximum 1734 * number of subbanks available 1735 */ 1736 if (sb_id >= npc_priv.num_subbanks) { 1737 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1738 __func__, sb_id); 1739 return -EINVAL; 1740 } 1741 1742 sb = &npc_priv.sb[sb_id]; 1743 1744 *key_type = sb->key_type; 1745 1746 return 0; 1747 } 1748 1749 static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, 1750 u16 sub_off, u16 *mcam_idx) 1751 { 1752 int off, bot; 1753 1754 /* for x4 section, maximum allowed subbank index = 1755 * subsection depth - 1 1756 */ 1757 if (sb->key_type == NPC_MCAM_KEY_X4 && 1758 sub_off >= npc_priv.subbank_depth) { 1759 dev_err(rvu->dev, 1760 "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", 1761 __func__, sb->idx, sub_off); 1762 return -EINVAL; 1763 } 1764 1765 /* for x2 section, maximum allowed subbank index = 1766 * 2 * subsection depth - 1 1767 */ 1768 if (sb->key_type == NPC_MCAM_KEY_X2 && 1769 sub_off >= npc_priv.subbank_depth * 2) { 1770 dev_err(rvu->dev, 1771 "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", 1772 __func__, sb->idx, sub_off); 1773 return -EINVAL; 1774 } 1775 1776 /* Find subbank offset from respective subbank (w.r.t bank) */ 1777 off = sub_off & (npc_priv.subbank_depth - 1); 1778 1779 /* if subsection idx is in bank1, add bank depth, 1780 * which is part of sb->b1b 1781 */ 1782 bot = sub_off >= npc_priv.subbank_depth ? sb->b1b : sb->b0b; 1783 1784 *mcam_idx = bot + off; 1785 return 0; 1786 } 1787 1788 static int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, 1789 struct npc_subbank **sb, 1790 int *sb_off) 1791 { 1792 int bank_off, sb_id; 1793 1794 /* mcam_idx should be less than (2 * bank depth) */ 1795 if (mcam_idx >= npc_priv.bank_depth * 2) { 1796 dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", 1797 __func__, mcam_idx); 1798 return -EINVAL; 1799 } 1800 1801 /* find mcam offset per bank */ 1802 bank_off = mcam_idx & (npc_priv.bank_depth - 1); 1803 1804 /* Find subbank id */ 1805 sb_id = bank_off / npc_priv.subbank_depth; 1806 1807 /* Check if subbank id is more than maximum 1808 * number of subbanks available 1809 */ 1810 if (sb_id >= npc_priv.num_subbanks) { 1811 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1812 __func__, sb_id); 1813 return -EINVAL; 1814 } 1815 1816 *sb = &npc_priv.sb[sb_id]; 1817 1818 /* Subbank offset per bank */ 1819 *sb_off = bank_off % npc_priv.subbank_depth; 1820 1821 /* Index in a subbank should add subbank depth 1822 * if it is in bank1 1823 */ 1824 if (mcam_idx >= npc_priv.bank_depth) 1825 *sb_off += npc_priv.subbank_depth; 1826 1827 return 0; 1828 } 1829 1830 static int __npc_subbank_contig_alloc(struct rvu *rvu, 1831 struct npc_subbank *sb, 1832 int key_type, int sidx, 1833 int eidx, int prio, 1834 int count, int t, int b, 1835 unsigned long *bmap, 1836 u16 *save) 1837 { 1838 int k, offset, delta = 0; 1839 int cnt = 0, sbd; 1840 1841 sbd = npc_priv.subbank_depth; 1842 1843 if (sidx >= npc_priv.bank_depth) 1844 delta = sbd; 1845 1846 switch (prio) { 1847 case NPC_MCAM_LOWER_PRIO: 1848 case NPC_MCAM_ANY_PRIO: 1849 /* Find an area of size 'count' from sidx to eidx */ 1850 offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, 1851 count, 0); 1852 1853 if (offset >= sbd) { 1854 dev_err(rvu->dev, 1855 "%s: Could not find contiguous(%d) entries\n", 1856 __func__, count); 1857 return -EFAULT; 1858 } 1859 1860 dev_dbg(rvu->dev, 1861 "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", 1862 __func__, sidx, eidx, t, b, offset, 1863 count, delta); 1864 1865 for (cnt = 0; cnt < count; cnt++) 1866 save[cnt] = offset + cnt + delta; 1867 1868 break; 1869 1870 case NPC_MCAM_HIGHER_PRIO: 1871 /* Find an area of 'count' from eidx to sidx */ 1872 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1873 /* If an intermediate slot is not free, 1874 * reset the counter (cnt) to zero as 1875 * request is for contiguous. 1876 */ 1877 if (test_bit(k, bmap)) { 1878 cnt = 0; 1879 continue; 1880 } 1881 1882 save[cnt++] = k + delta; 1883 } 1884 break; 1885 } 1886 1887 /* Found 'count' number of free slots */ 1888 if (cnt == count) 1889 return 0; 1890 1891 dev_dbg(rvu->dev, 1892 "%s: Could not find contiguous(%d) entries in subbank=%u\n", 1893 __func__, count, sb->idx); 1894 return -EFAULT; 1895 } 1896 1897 static int __npc_subbank_non_contig_alloc(struct rvu *rvu, 1898 struct npc_subbank *sb, 1899 int key_type, int sidx, 1900 int eidx, int prio, 1901 int t, int b, 1902 unsigned long *bmap, 1903 int count, u16 *save, 1904 bool max_alloc, int *alloc_cnt) 1905 { 1906 unsigned long index; 1907 int cnt = 0, delta; 1908 int k, sbd; 1909 1910 sbd = npc_priv.subbank_depth; 1911 delta = sidx >= npc_priv.bank_depth ? sbd : 0; 1912 1913 switch (prio) { 1914 /* Find an area of size 'count' from sidx to eidx */ 1915 case NPC_MCAM_LOWER_PRIO: 1916 case NPC_MCAM_ANY_PRIO: 1917 index = find_next_zero_bit(bmap, sbd, sidx - b); 1918 if (index >= sbd) { 1919 dev_err(rvu->dev, 1920 "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", 1921 __func__, count, 1922 bitmap_weight(bmap, sbd), 1923 sb->idx); 1924 break; 1925 } 1926 1927 for (k = index; cnt < count && k <= (eidx - b); k++) { 1928 /* Skip used slots */ 1929 if (test_bit(k, bmap)) 1930 continue; 1931 1932 save[cnt++] = k + delta; 1933 } 1934 break; 1935 1936 /* Find an area of 'count' from eidx to sidx */ 1937 case NPC_MCAM_HIGHER_PRIO: 1938 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1939 /* Skip used slots */ 1940 if (test_bit(k, bmap)) 1941 continue; 1942 1943 save[cnt++] = k + delta; 1944 } 1945 break; 1946 } 1947 1948 /* Update allocated 'cnt' to alloc_cnt */ 1949 *alloc_cnt = cnt; 1950 1951 /* Successfully allocated requested count slots */ 1952 if (cnt == count) 1953 return 0; 1954 1955 /* Allocation successful for cnt < count */ 1956 if (max_alloc && cnt > 0) 1957 return 0; 1958 1959 dev_dbg(rvu->dev, 1960 "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", 1961 __func__, count, sb->idx, cnt, max_alloc); 1962 1963 return -EFAULT; 1964 } 1965 1966 static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, 1967 int sb_off, unsigned long **bmap, 1968 int *off) 1969 { 1970 int sbd; 1971 1972 sbd = npc_priv.subbank_depth; 1973 1974 *off = sb_off & (sbd - 1); 1975 *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; 1976 } 1977 1978 /* set/clear bitmap */ 1979 static bool __npc_subbank_mark_slot(struct rvu *rvu, 1980 struct npc_subbank *sb, 1981 int sb_off, bool set) 1982 { 1983 unsigned long *bmap; 1984 int off; 1985 1986 /* if sb_off >= subbank.depth, then slots are in 1987 * bank1 1988 */ 1989 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 1990 1991 dev_dbg(rvu->dev, 1992 "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", 1993 __func__, set, sb_off, sb->idx, off); 1994 1995 if (set) { 1996 /* Slot is already used */ 1997 if (test_bit(off, bmap)) 1998 return false; 1999 2000 sb->free_cnt--; 2001 set_bit(off, bmap); 2002 return true; 2003 } 2004 2005 /* Slot is already free */ 2006 if (!test_bit(off, bmap)) 2007 return false; 2008 2009 sb->free_cnt++; 2010 clear_bit(off, bmap); 2011 return true; 2012 } 2013 2014 static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) 2015 { 2016 int rc, blkaddr; 2017 2018 sb->flags = NPC_SUBBANK_FLAG_FREE; 2019 sb->key_type = 0; 2020 2021 bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); 2022 bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); 2023 2024 if (!xa_erase(&npc_priv.xa_sb_used, sb->arr_idx)) { 2025 dev_err(rvu->dev, 2026 "%s: Error to delete from xa_sb_used array\n", 2027 __func__); 2028 return -EFAULT; 2029 } 2030 2031 rc = xa_insert(&npc_priv.xa_sb_free, sb->arr_idx, 2032 xa_mk_value(sb->idx), GFP_KERNEL); 2033 if (rc) { 2034 rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, 2035 xa_mk_value(sb->idx), GFP_KERNEL); 2036 if (rc) 2037 dev_err(rvu->dev, 2038 "%s: Failed to roll back sb(%u) arr_idx=%d\n", 2039 __func__, sb->idx, sb->arr_idx); 2040 2041 dev_err(rvu->dev, 2042 "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", 2043 __func__, sb->idx, sb->arr_idx); 2044 return rc; 2045 } 2046 2047 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2048 rvu_write64(rvu, blkaddr, 2049 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2050 NPC_MCAM_KEY_X2); 2051 2052 return rc; 2053 } 2054 2055 static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, 2056 int key_type) 2057 { 2058 int rc; 2059 2060 sb->flags = NPC_SUBBANK_FLAG_USED; 2061 sb->key_type = key_type; 2062 if (key_type == NPC_MCAM_KEY_X4) 2063 sb->free_cnt = npc_priv.subbank_depth; 2064 else 2065 sb->free_cnt = 2 * npc_priv.subbank_depth; 2066 2067 bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); 2068 bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); 2069 2070 if (!xa_erase(&npc_priv.xa_sb_free, sb->arr_idx)) { 2071 dev_err(rvu->dev, 2072 "%s: Error to delete from xa_sb_free array\n", 2073 __func__); 2074 return -EFAULT; 2075 } 2076 2077 rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, 2078 xa_mk_value(sb->idx), GFP_KERNEL); 2079 if (rc) 2080 dev_err(rvu->dev, 2081 "%s: Error to add to xa_sb_used array\n", __func__); 2082 2083 return rc; 2084 } 2085 2086 static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, 2087 u16 sb_off) 2088 { 2089 bool deleted = false; 2090 unsigned long *bmap; 2091 int rc, off; 2092 2093 deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); 2094 if (!deleted) 2095 goto done; 2096 2097 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2098 2099 /* Check whether we can mark whole subbank as free */ 2100 if (sb->key_type == NPC_MCAM_KEY_X4) { 2101 if (sb->free_cnt < npc_priv.subbank_depth) 2102 goto done; 2103 } else { 2104 if (sb->free_cnt < 2 * npc_priv.subbank_depth) 2105 goto done; 2106 } 2107 2108 /* All slots in subbank are unused. Mark the subbank as free 2109 * and add to free pool 2110 */ 2111 rc = __npc_subbank_mark_free(rvu, sb); 2112 if (rc) 2113 dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); 2114 2115 done: 2116 return deleted; 2117 } 2118 2119 static int 2120 npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) 2121 { 2122 bool deleted; 2123 2124 mutex_lock(&sb->lock); 2125 deleted = __npc_subbank_free(rvu, sb, sb_off); 2126 mutex_unlock(&sb->lock); 2127 2128 return deleted ? 0 : -EFAULT; 2129 } 2130 2131 static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2132 int key_type, int ref, int limit, int prio, 2133 bool contig, int count, u16 *mcam_idx, 2134 int idx_sz, bool max_alloc, int *alloc_cnt) 2135 { 2136 int cnt, t, b, i, blkaddr; 2137 bool new_sub_bank = false; 2138 unsigned long *bmap; 2139 u16 *save = NULL; 2140 int sidx, eidx; 2141 bool diffbank; 2142 int bw, bfree; 2143 int rc = 0; 2144 bool ret; 2145 2146 /* Check if enough space is there to return requested number of 2147 * mcam indexes in case of contiguous allocation 2148 */ 2149 if (!max_alloc && count > idx_sz) { 2150 dev_err(rvu->dev, 2151 "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", 2152 __func__, count, idx_sz, sb->idx); 2153 return -ENOSPC; 2154 } 2155 2156 /* Allocation on multiple subbank is not supported by this function. 2157 * it means that ref and limit should be on same subbank. 2158 * 2159 * ref and limit values should be validated w.r.t prio as below. 2160 * say ref = 100, limit = 200, 2161 * if NPC_MCAM_LOWER_PRIO, allocate index 100 2162 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. 2163 * if NPC_MCAM_ANY_PRIO, allocate index 100 2164 * 2165 * say ref = 200, limit = 100 2166 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. 2167 * if NPC_MCAM_HIGHER_PRIO, allocate index 200 2168 * if NPC_MCAM_ANY_PRIO, allocate index 100 2169 * 2170 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction 2171 * on "ref" and "limit" values. ie, ref > limit and limit > ref 2172 * are valid cases. 2173 */ 2174 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 2175 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 2176 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 2177 __func__, ref, limit); 2178 return -EINVAL; 2179 } 2180 2181 /* x4 indexes are from 0 to bank size as it combines two x2 banks */ 2182 if (key_type == NPC_MCAM_KEY_X4 && 2183 (ref >= npc_priv.bank_depth || limit >= npc_priv.bank_depth)) { 2184 dev_err(rvu->dev, 2185 "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", 2186 __func__, ref, limit); 2187 return -EINVAL; 2188 } 2189 2190 /* This function is called either bank0 or bank1 portion of a subbank. 2191 * so ref and limit should be on same bank. 2192 */ 2193 diffbank = !!((ref & npc_priv.bank_depth) ^ 2194 (limit & npc_priv.bank_depth)); 2195 if (diffbank) { 2196 dev_err(rvu->dev, 2197 "%s: request ref and limit should be from same bank\n", 2198 __func__); 2199 return -EINVAL; 2200 } 2201 2202 sidx = min_t(int, limit, ref); 2203 eidx = max_t(int, limit, ref); 2204 2205 /* Find total number of slots available; both used and free */ 2206 cnt = eidx - sidx + 1; 2207 if (contig && cnt < count) { 2208 dev_err(rvu->dev, 2209 "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", 2210 __func__, ref, limit, count); 2211 return -EINVAL; 2212 } 2213 2214 /* If subbank is free, check if requested number of indexes is less than 2215 * or equal to mcam entries available in the subbank if contig. 2216 */ 2217 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2218 if (contig && count > npc_priv.subbank_depth) { 2219 dev_err(rvu->dev, "%s: Less number of entries\n", 2220 __func__); 2221 return -ENOSPC; 2222 } 2223 2224 new_sub_bank = true; 2225 goto process; 2226 } 2227 2228 /* Flag should be set for all used subbanks */ 2229 WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), 2230 "Used flag is not set(%#x)\n", sb->flags); 2231 2232 /* If subbank key type does not match with requested key_type, 2233 * return error 2234 */ 2235 if (sb->key_type != key_type) { 2236 dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); 2237 return -EINVAL; 2238 } 2239 2240 process: 2241 /* if ref or limit >= npc_priv.bank_depth, index are in bank1. 2242 * else bank0. 2243 */ 2244 if (ref >= npc_priv.bank_depth) { 2245 bmap = sb->b1map; 2246 t = sb->b1t; 2247 b = sb->b1b; 2248 } else { 2249 bmap = sb->b0map; 2250 t = sb->b0t; 2251 b = sb->b0b; 2252 } 2253 2254 /* Calculate free slots */ 2255 bw = bitmap_weight(bmap, npc_priv.subbank_depth); 2256 bfree = npc_priv.subbank_depth - bw; 2257 2258 if (!bfree) { 2259 dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); 2260 return -ENOSPC; 2261 } 2262 2263 /* If request is for contiguous , then max we can allocate is 2264 * equal to subbank_depth 2265 */ 2266 if (contig && bfree < count) { 2267 dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); 2268 return -ENOSPC; 2269 } 2270 2271 /* 'save' array stores available indexes temporarily before 2272 * marking it as allocated 2273 */ 2274 save = kcalloc(count, sizeof(u16), GFP_KERNEL); 2275 if (!save) { 2276 rc = -ENOMEM; 2277 goto err1; 2278 } 2279 2280 if (contig) { 2281 rc = __npc_subbank_contig_alloc(rvu, sb, key_type, 2282 sidx, eidx, prio, 2283 count, t, b, 2284 bmap, save); 2285 /* contiguous allocation success means that 2286 * requested number of free slots got 2287 * allocated 2288 */ 2289 if (!rc) 2290 *alloc_cnt = count; 2291 2292 } else { 2293 rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, 2294 sidx, eidx, prio, 2295 t, b, bmap, 2296 count, save, 2297 max_alloc, alloc_cnt); 2298 } 2299 2300 if (rc) 2301 goto err1; 2302 2303 /* Mark new subbank bank as used */ 2304 if (new_sub_bank) { 2305 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2306 if (blkaddr < 0) { 2307 dev_err(rvu->dev, 2308 "%s: NPC block not implemented\n", __func__); 2309 rc = -EFAULT; 2310 goto err1; 2311 } 2312 2313 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2314 if (rc) { 2315 dev_err(rvu->dev, 2316 "%s: Error to mark subbank as used\n", 2317 __func__); 2318 goto err2; 2319 } 2320 2321 /* Configure section type to key_type */ 2322 rvu_write64(rvu, blkaddr, 2323 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2324 key_type); 2325 } 2326 2327 for (i = 0; i < *alloc_cnt; i++) { 2328 rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], 2329 &mcam_idx[i]); 2330 if (rc) { 2331 dev_err(rvu->dev, 2332 "%s: Error to find mcam idx for %u\n", 2333 __func__, save[i]); 2334 /* TODO: handle err case gracefully */ 2335 goto err3; 2336 } 2337 2338 /* Mark all slots as used */ 2339 ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); 2340 if (!ret) { 2341 dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", 2342 __func__, mcam_idx[i]); 2343 rc = -EFAULT; 2344 goto err3; 2345 } 2346 } 2347 kfree(save); 2348 return 0; 2349 2350 err3: 2351 for (int j = 0; j < i; j++) 2352 __npc_subbank_mark_slot(rvu, sb, save[j], false); 2353 err2: 2354 if (new_sub_bank) 2355 __npc_subbank_mark_free(rvu, sb); 2356 err1: 2357 kfree(save); 2358 *alloc_cnt = 0; 2359 return rc; 2360 } 2361 2362 static int 2363 npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2364 int key_type, int ref, int limit, int prio, 2365 bool contig, int count, u16 *mcam_idx, 2366 int idx_sz, bool max_alloc, int *alloc_cnt) 2367 { 2368 int rc; 2369 2370 mutex_lock(&sb->lock); 2371 rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, 2372 contig, count, mcam_idx, idx_sz, 2373 max_alloc, alloc_cnt); 2374 mutex_unlock(&sb->lock); 2375 2376 return rc; 2377 } 2378 2379 static int 2380 npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) 2381 { 2382 int pcifunc, idx; 2383 void *map; 2384 2385 map = xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); 2386 if (!map) { 2387 dev_err(rvu->dev, 2388 "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", 2389 __func__, mcam_idx); 2390 return -EFAULT; 2391 } 2392 2393 pcifunc = xa_to_value(map); 2394 map = xa_load(&npc_priv.xa_pf_map, pcifunc); 2395 if (!map) { 2396 dev_err(rvu->dev, 2397 "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", 2398 __func__, pcifunc, mcam_idx); 2399 return -ESRCH; 2400 } 2401 2402 idx = xa_to_value(map); 2403 2404 map = xa_erase(&npc_priv.xa_pf2idx_map[idx], mcam_idx); 2405 if (!map) { 2406 dev_err(rvu->dev, 2407 "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", 2408 __func__, mcam_idx); 2409 return -EFAULT; 2410 } 2411 return 0; 2412 } 2413 2414 static int 2415 npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) 2416 { 2417 int rc, idx; 2418 void *map; 2419 2420 dev_dbg(rvu->dev, 2421 "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", 2422 __func__, mcam_idx, pcifunc); 2423 2424 rc = xa_insert(&npc_priv.xa_idx2pf_map, mcam_idx, 2425 xa_mk_value(pcifunc), GFP_KERNEL); 2426 2427 if (rc) { 2428 map = xa_load(&npc_priv.xa_idx2pf_map, mcam_idx); 2429 dev_err(rvu->dev, 2430 "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", 2431 __func__, mcam_idx, xa_to_value(map)); 2432 return -EFAULT; 2433 } 2434 2435 map = xa_load(&npc_priv.xa_pf_map, pcifunc); 2436 if (!map) { 2437 dev_err(rvu->dev, 2438 "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", 2439 __func__, pcifunc, mcam_idx); 2440 return -ESRCH; 2441 } 2442 2443 idx = xa_to_value(map); 2444 2445 rc = xa_insert(&npc_priv.xa_pf2idx_map[idx], mcam_idx, 2446 xa_mk_value(pcifunc), GFP_KERNEL); 2447 2448 if (rc) { 2449 map = xa_load(&npc_priv.xa_pf2idx_map[idx], mcam_idx); 2450 xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); 2451 dev_err(rvu->dev, 2452 "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", 2453 __func__, mcam_idx, xa_to_value(map), idx); 2454 2455 return -EFAULT; 2456 } 2457 2458 return 0; 2459 } 2460 2461 static bool 2462 npc_subbank_suits(struct npc_subbank *sb, int key_type) 2463 { 2464 mutex_lock(&sb->lock); 2465 2466 if (!sb->key_type) { 2467 mutex_unlock(&sb->lock); 2468 return true; 2469 } 2470 2471 if (sb->key_type == key_type) { 2472 mutex_unlock(&sb->lock); 2473 return true; 2474 } 2475 2476 mutex_unlock(&sb->lock); 2477 return false; 2478 } 2479 2480 #define SB_ALIGN_UP(val) (((val) + npc_priv.subbank_depth) & \ 2481 ~((npc_priv.subbank_depth) - 1)) 2482 #define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv.subbank_depth) 2483 2484 static void npc_subbank_iter_down(struct rvu *rvu, 2485 int ref, int limit, 2486 int *cur_ref, int *cur_limit, 2487 bool *start, bool *stop) 2488 { 2489 int align; 2490 2491 *stop = false; 2492 2493 /* ALIGN_DOWN the limit to current subbank boundary bottom index */ 2494 if (*start) { 2495 *start = false; 2496 *cur_ref = ref; 2497 align = SB_ALIGN_DOWN(ref); 2498 if (align < limit) { 2499 *stop = true; 2500 *cur_limit = limit; 2501 return; 2502 } 2503 *cur_limit = align; 2504 return; 2505 } 2506 2507 *cur_ref = *cur_limit - 1; 2508 align = *cur_ref - npc_priv.subbank_depth + 1; 2509 if (align <= limit) { 2510 *stop = true; 2511 *cur_limit = limit; 2512 return; 2513 } 2514 2515 *cur_limit = align; 2516 } 2517 2518 static void npc_subbank_iter_up(struct rvu *rvu, 2519 int ref, int limit, 2520 int *cur_ref, int *cur_limit, 2521 bool *start, bool *stop) 2522 { 2523 int align; 2524 2525 *stop = false; 2526 2527 /* ALIGN_UP the limit to current subbank boundary top index */ 2528 if (*start) { 2529 *start = false; 2530 *cur_ref = ref; 2531 2532 /* Find next lower prio subbank's bottom index */ 2533 align = SB_ALIGN_UP(ref); 2534 2535 /* Crosses limit ? */ 2536 if (align - 1 > limit) { 2537 *stop = true; 2538 *cur_limit = limit; 2539 return; 2540 } 2541 2542 /* Current subbank's top index */ 2543 *cur_limit = align - 1; 2544 return; 2545 } 2546 2547 *cur_ref = *cur_limit + 1; 2548 align = *cur_ref + npc_priv.subbank_depth - 1; 2549 2550 if (align >= limit) { 2551 *stop = true; 2552 *cur_limit = limit; 2553 return; 2554 } 2555 2556 *cur_limit = align; 2557 } 2558 2559 static int 2560 npc_subbank_iter(struct rvu *rvu, int key_type, 2561 int ref, int limit, int prio, 2562 int *cur_ref, int *cur_limit, 2563 bool *start, bool *stop) 2564 { 2565 if (prio != NPC_MCAM_HIGHER_PRIO) 2566 npc_subbank_iter_up(rvu, ref, limit, 2567 cur_ref, cur_limit, 2568 start, stop); 2569 else 2570 npc_subbank_iter_down(rvu, ref, limit, 2571 cur_ref, cur_limit, 2572 start, stop); 2573 2574 /* limit and ref should < bank_depth for x4 */ 2575 if (key_type == NPC_MCAM_KEY_X4) { 2576 if (*cur_ref >= npc_priv.bank_depth) 2577 return -EINVAL; 2578 2579 if (*cur_limit >= npc_priv.bank_depth) 2580 return -EINVAL; 2581 } 2582 /* limit and ref should < 2 * bank_depth, for x2 */ 2583 if (*cur_ref >= 2 * npc_priv.bank_depth) 2584 return -EINVAL; 2585 2586 if (*cur_limit >= 2 * npc_priv.bank_depth) 2587 return -EINVAL; 2588 2589 return 0; 2590 } 2591 2592 static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, 2593 bool maps_del) 2594 { 2595 struct npc_subbank *sb; 2596 u16 vidx, midx; 2597 int sb_off, i; 2598 bool ret; 2599 int rc; 2600 2601 /* Check if we can dealloc indexes properly ? */ 2602 for (i = 0; i < count; i++) { 2603 rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), 2604 &sb, &sb_off); 2605 if (rc) { 2606 dev_err(rvu->dev, 2607 "Failed to free mcam idx=%u\n", mcam_idx[i]); 2608 return rc; 2609 } 2610 } 2611 2612 for (i = 0; i < count; i++) { 2613 if (npc_is_vidx(mcam_idx[i])) { 2614 vidx = mcam_idx[i]; 2615 midx = npc_vidx2idx(vidx); 2616 } else { 2617 midx = mcam_idx[i]; 2618 vidx = npc_idx2vidx(midx); 2619 } 2620 2621 if (midx >= npc_priv.bank_depth * npc_priv.num_banks) { 2622 dev_err(rvu->dev, 2623 "%s: Invalid mcam_idx=%u cannot be deleted\n", 2624 __func__, mcam_idx[i]); 2625 return -EINVAL; 2626 } 2627 2628 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 2629 &sb, &sb_off); 2630 if (rc) { 2631 dev_err(rvu->dev, 2632 "%s: Failed to find subbank info for vidx=%u\n", 2633 __func__, vidx); 2634 return rc; 2635 } 2636 2637 ret = npc_subbank_free(rvu, sb, sb_off); 2638 if (ret) { 2639 dev_err(rvu->dev, 2640 "%s: Failed to find subbank info for vidx=%u\n", 2641 __func__, vidx); 2642 return -EINVAL; 2643 } 2644 2645 if (!maps_del) 2646 continue; 2647 2648 rc = npc_del_from_pf_maps(rvu, midx); 2649 if (rc) 2650 return rc; 2651 2652 /* If there is no vidx mapping; continue */ 2653 if (vidx == midx) 2654 continue; 2655 2656 rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); 2657 if (rc) 2658 return rc; 2659 } 2660 2661 return 0; 2662 } 2663 2664 static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, 2665 int ref, int limit, int prio, 2666 bool contig, int count, 2667 u16 *mcam_idx) 2668 { 2669 struct npc_subbank *sb; 2670 unsigned long *bmap; 2671 int sb_off, off, rc; 2672 int cnt = 0; 2673 bool bitset; 2674 2675 if (prio != NPC_MCAM_HIGHER_PRIO) { 2676 while (ref <= limit) { 2677 /* Calculate subbank and subbank index */ 2678 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2679 &sb, &sb_off); 2680 if (rc) 2681 goto err; 2682 2683 /* If subbank is not suitable for requested key type 2684 * restart search from next subbank 2685 */ 2686 if (!npc_subbank_suits(sb, key_type)) { 2687 ref = SB_ALIGN_UP(ref); 2688 if (contig) { 2689 rc = npc_idx_free(rvu, mcam_idx, 2690 cnt, false); 2691 if (rc) 2692 return rc; 2693 cnt = 0; 2694 } 2695 continue; 2696 } 2697 2698 mutex_lock(&sb->lock); 2699 2700 /* If subbank is free; mark it as used */ 2701 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2702 rc = __npc_subbank_mark_used(rvu, sb, 2703 key_type); 2704 if (rc) { 2705 mutex_unlock(&sb->lock); 2706 dev_err(rvu->dev, 2707 "%s:Error to add to use array\n", 2708 __func__); 2709 goto err; 2710 } 2711 } 2712 2713 /* Find correct bmap */ 2714 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2715 2716 /* if bit is already set, reset 'cnt' */ 2717 bitset = test_bit(off, bmap); 2718 if (bitset) { 2719 mutex_unlock(&sb->lock); 2720 if (contig) { 2721 rc = npc_idx_free(rvu, mcam_idx, 2722 cnt, false); 2723 if (rc) 2724 return rc; 2725 cnt = 0; 2726 } 2727 2728 ref++; 2729 continue; 2730 } 2731 2732 set_bit(off, bmap); 2733 sb->free_cnt--; 2734 mcam_idx[cnt++] = ref; 2735 mutex_unlock(&sb->lock); 2736 2737 if (cnt == count) 2738 return 0; 2739 ref++; 2740 } 2741 2742 /* Could not allocate request count slots */ 2743 goto err; 2744 } 2745 while (ref >= limit) { 2746 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2747 &sb, &sb_off); 2748 if (rc) 2749 goto err; 2750 2751 if (!npc_subbank_suits(sb, key_type)) { 2752 ref = SB_ALIGN_DOWN(ref) - 1; 2753 if (contig) { 2754 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2755 if (rc) 2756 return rc; 2757 2758 cnt = 0; 2759 } 2760 continue; 2761 } 2762 2763 mutex_lock(&sb->lock); 2764 2765 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2766 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2767 if (rc) { 2768 mutex_unlock(&sb->lock); 2769 dev_err(rvu->dev, 2770 "%s:Error to add to use array\n", 2771 __func__); 2772 goto err; 2773 } 2774 } 2775 2776 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2777 bitset = test_bit(off, bmap); 2778 if (bitset) { 2779 mutex_unlock(&sb->lock); 2780 if (contig) { 2781 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2782 if (rc) 2783 return rc; 2784 2785 cnt = 0; 2786 } 2787 ref--; 2788 continue; 2789 } 2790 2791 mcam_idx[cnt++] = ref; 2792 sb->free_cnt--; 2793 set_bit(off, bmap); 2794 mutex_unlock(&sb->lock); 2795 2796 if (cnt == count) 2797 return 0; 2798 ref--; 2799 } 2800 2801 err: 2802 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2803 if (rc) 2804 dev_err(rvu->dev, 2805 "%s: Error happened while freeing cnt=%u indexes\n", 2806 __func__, cnt); 2807 2808 return -ENOSPC; 2809 } 2810 2811 static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, 2812 int key_type) 2813 { 2814 int cnt, spd; 2815 2816 spd = npc_priv.subbank_depth; 2817 mutex_lock(&sb->lock); 2818 2819 if (sb->flags & NPC_SUBBANK_FLAG_FREE) 2820 cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; 2821 else 2822 cnt = sb->free_cnt; 2823 2824 mutex_unlock(&sb->lock); 2825 return cnt; 2826 } 2827 2828 static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, 2829 int ref, int limit, int prio, 2830 bool contig, int count, 2831 u16 *mcam_idx) 2832 { 2833 struct npc_subbank *sb1, *sb2; 2834 bool max_alloc, start, stop; 2835 int r, l, sb_idx1, sb_idx2; 2836 int tot = 0, rc; 2837 int alloc_cnt; 2838 2839 max_alloc = !contig; 2840 2841 start = true; 2842 stop = false; 2843 2844 /* Loop until we cross the ref/limit boundary */ 2845 while (!stop) { 2846 rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, 2847 &r, &l, &start, &stop); 2848 2849 dev_dbg(rvu->dev, 2850 "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", 2851 __func__, ref, limit, r, l, 2852 start, stop, tot, count, rc); 2853 2854 if (rc) 2855 goto err; 2856 2857 /* Find subbank and subbank index for ref */ 2858 rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, 2859 &sb_idx1); 2860 if (rc) 2861 goto err; 2862 2863 dev_dbg(rvu->dev, 2864 "%s: ref subbank=%d off=%d\n", 2865 __func__, sb1->idx, sb_idx1); 2866 2867 /* Skip subbank if it is not available for the keytype */ 2868 if (!npc_subbank_suits(sb1, key_type)) { 2869 dev_dbg(rvu->dev, 2870 "%s: not suitable sb=%d key_type=%d\n", 2871 __func__, sb1->idx, key_type); 2872 continue; 2873 } 2874 2875 /* Find subbank and subbank index for limit */ 2876 rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, 2877 &sb_idx2); 2878 if (rc) 2879 goto err; 2880 2881 dev_dbg(rvu->dev, 2882 "%s: limit subbank=%d off=%d\n", 2883 __func__, sb_idx1, sb_idx2); 2884 2885 /* subbank of ref and limit should be same */ 2886 if (sb1 != sb2) { 2887 dev_err(rvu->dev, 2888 "%s: l(%d) and r(%d) are not in same subbank\n", 2889 __func__, r, l); 2890 goto err; 2891 } 2892 2893 if (contig && 2894 npc_subbank_free_cnt(rvu, sb1, key_type) < count) { 2895 dev_dbg(rvu->dev, "%s: less count =%d\n", 2896 __func__, 2897 npc_subbank_free_cnt(rvu, sb1, key_type)); 2898 continue; 2899 } 2900 2901 /* Try in one bank of a subbank */ 2902 alloc_cnt = 0; 2903 rc = npc_subbank_alloc(rvu, sb1, key_type, 2904 r, l, prio, contig, 2905 count - tot, mcam_idx + tot, 2906 count - tot, max_alloc, 2907 &alloc_cnt); 2908 2909 tot += alloc_cnt; 2910 2911 dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", 2912 __func__, tot, alloc_cnt); 2913 2914 if (!rc && count == tot) 2915 return 0; 2916 } 2917 err: 2918 dev_dbg(rvu->dev, "%s: Error to allocate\n", 2919 __func__); 2920 2921 /* non contiguous allocation fails. We need to do clean up */ 2922 if (max_alloc) { 2923 rc = npc_idx_free(rvu, mcam_idx, tot, false); 2924 if (rc) 2925 dev_err(rvu->dev, 2926 "%s: failed to free %u indexes\n", 2927 __func__, tot); 2928 } 2929 2930 return -EFAULT; 2931 } 2932 2933 /* Minimize allocation from bottom and top subbanks for noref allocations. 2934 * Default allocations are ref based, and will be allocated from top 2935 * subbanks (least priority subbanks). Since default allocation is at very 2936 * early stage of kernel netdev probes, this subbanks will be moved to 2937 * used subbanks list. This will pave a way for noref allocation from these 2938 * used subbanks. Skip allocation for these top and bottom, and try free 2939 * bank next. If none slot is available, come back and search in these 2940 * subbanks. 2941 */ 2942 2943 static int npc_subbank_restricted_idxs[2]; 2944 static bool restrict_valid = true; 2945 2946 static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) 2947 { 2948 int i; 2949 2950 if (!restrict_valid) 2951 return false; 2952 2953 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { 2954 if (index == npc_subbank_restricted_idxs[i]) 2955 return true; 2956 } 2957 2958 return false; 2959 } 2960 2961 static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, 2962 int count, u16 *mcam_idx) 2963 { 2964 struct npc_subbank *sb; 2965 unsigned long index; 2966 int tot = 0, rc; 2967 bool max_alloc; 2968 int alloc_cnt; 2969 int idx, i; 2970 void *val; 2971 2972 max_alloc = !contig; 2973 2974 /* Check used subbanks for free slots */ 2975 xa_for_each(&npc_priv.xa_sb_used, index, val) { 2976 idx = xa_to_value(val); 2977 2978 /* Minimize allocation from restricted subbanks 2979 * in noref allocations. 2980 */ 2981 if (npc_subbank_restrict_usage(rvu, idx)) 2982 continue; 2983 2984 sb = &npc_priv.sb[idx]; 2985 2986 /* Skip if not suitable subbank */ 2987 if (!npc_subbank_suits(sb, key_type)) 2988 continue; 2989 2990 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 2991 continue; 2992 2993 /* try in bank 0. Try passing ref and limit equal to 2994 * subbank boundaries 2995 */ 2996 alloc_cnt = 0; 2997 rc = npc_subbank_alloc(rvu, sb, key_type, 2998 sb->b0b, sb->b0t, 0, 2999 contig, count - tot, 3000 mcam_idx + tot, 3001 count - tot, 3002 max_alloc, &alloc_cnt); 3003 3004 /* Non contiguous allocation may allocate less than 3005 * requested 'count'. 3006 */ 3007 tot += alloc_cnt; 3008 3009 dev_dbg(rvu->dev, 3010 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3011 __func__, alloc_cnt, sb->idx, tot, count); 3012 3013 /* Successfully allocated */ 3014 if (!rc && count == tot) 3015 return 0; 3016 3017 /* x4 entries can be allocated from bank 0 only */ 3018 if (key_type == NPC_MCAM_KEY_X4) 3019 continue; 3020 3021 /* try in bank 1 for x2 */ 3022 alloc_cnt = 0; 3023 rc = npc_subbank_alloc(rvu, sb, key_type, 3024 sb->b1b, sb->b1t, 0, 3025 contig, count - tot, 3026 mcam_idx + tot, 3027 count - tot, max_alloc, 3028 &alloc_cnt); 3029 3030 tot += alloc_cnt; 3031 3032 dev_dbg(rvu->dev, 3033 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3034 __func__, alloc_cnt, sb->idx, tot, count); 3035 3036 if (!rc && count == tot) 3037 return 0; 3038 } 3039 3040 /* Allocate in free subbanks */ 3041 xa_for_each(&npc_priv.xa_sb_free, index, val) { 3042 idx = xa_to_value(val); 3043 sb = &npc_priv.sb[idx]; 3044 3045 /* Minimize allocation from restricted subbanks 3046 * in noref allocations. 3047 */ 3048 if (npc_subbank_restrict_usage(rvu, idx)) 3049 continue; 3050 3051 if (!npc_subbank_suits(sb, key_type)) 3052 continue; 3053 3054 /* try in bank 0 */ 3055 alloc_cnt = 0; 3056 rc = npc_subbank_alloc(rvu, sb, key_type, 3057 sb->b0b, sb->b0t, 0, 3058 contig, count - tot, 3059 mcam_idx + tot, 3060 count - tot, 3061 max_alloc, &alloc_cnt); 3062 3063 tot += alloc_cnt; 3064 3065 dev_dbg(rvu->dev, 3066 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3067 __func__, alloc_cnt, sb->idx, tot, count); 3068 3069 /* Successfully allocated */ 3070 if (!rc && count == tot) 3071 return 0; 3072 3073 /* x4 entries can be allocated from bank 0 only */ 3074 if (key_type == NPC_MCAM_KEY_X4) 3075 continue; 3076 3077 /* try in bank 1 for x2 */ 3078 alloc_cnt = 0; 3079 rc = npc_subbank_alloc(rvu, sb, 3080 key_type, sb->b1b, sb->b1t, 0, 3081 contig, count - tot, 3082 mcam_idx + tot, count - tot, 3083 max_alloc, &alloc_cnt); 3084 3085 tot += alloc_cnt; 3086 3087 dev_dbg(rvu->dev, 3088 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3089 __func__, alloc_cnt, sb->idx, tot, count); 3090 3091 if (!rc && count == tot) 3092 return 0; 3093 } 3094 3095 /* Allocate from restricted subbanks */ 3096 for (i = 0; restrict_valid && 3097 (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { 3098 idx = npc_subbank_restricted_idxs[i]; 3099 sb = &npc_priv.sb[idx]; 3100 3101 /* Skip if not suitable subbank */ 3102 if (!npc_subbank_suits(sb, key_type)) 3103 continue; 3104 3105 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3106 continue; 3107 3108 /* try in bank 0. Try passing ref and limit equal to 3109 * subbank boundaries 3110 */ 3111 alloc_cnt = 0; 3112 rc = npc_subbank_alloc(rvu, sb, key_type, 3113 sb->b0b, sb->b0t, 0, 3114 contig, count - tot, 3115 mcam_idx + tot, 3116 count - tot, 3117 max_alloc, &alloc_cnt); 3118 3119 /* Non contiguous allocation may allocate less than 3120 * requested 'count'. 3121 */ 3122 tot += alloc_cnt; 3123 3124 dev_dbg(rvu->dev, 3125 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3126 __func__, alloc_cnt, sb->idx, tot, count); 3127 3128 /* Successfully allocated */ 3129 if (!rc && count == tot) 3130 return 0; 3131 3132 /* x4 entries can be allocated from bank 0 only */ 3133 if (key_type == NPC_MCAM_KEY_X4) 3134 continue; 3135 3136 /* try in bank 1 for x2 */ 3137 alloc_cnt = 0; 3138 rc = npc_subbank_alloc(rvu, sb, key_type, 3139 sb->b1b, sb->b1t, 0, 3140 contig, count - tot, 3141 mcam_idx + tot, 3142 count - tot, max_alloc, 3143 &alloc_cnt); 3144 3145 tot += alloc_cnt; 3146 3147 dev_dbg(rvu->dev, 3148 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3149 __func__, alloc_cnt, sb->idx, tot, count); 3150 3151 if (!rc && count == tot) 3152 return 0; 3153 } 3154 3155 /* non contiguous allocation fails. We need to do clean up */ 3156 if (max_alloc) 3157 npc_idx_free(rvu, mcam_idx, tot, false); 3158 3159 dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", 3160 __func__); 3161 3162 return -EFAULT; 3163 } 3164 3165 int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) 3166 { 3167 return npc_idx_free(rvu, mcam_idx, count, true); 3168 } 3169 3170 int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, 3171 int prio, u16 *mcam_idx, int ref, int limit, 3172 bool contig, int count, bool virt) 3173 { 3174 bool defrag_candidate = false; 3175 int i, eidx, rc, bd; 3176 bool ref_valid; 3177 u16 vidx; 3178 3179 bd = npc_priv.bank_depth; 3180 3181 /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 3182 * Here user wants to allocate 0th entry 3183 */ 3184 if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && 3185 count == 1) { 3186 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3187 prio, contig, count, mcam_idx); 3188 3189 if (rc) 3190 return rc; 3191 goto add2map; 3192 } 3193 3194 ref_valid = !!(limit || ref); 3195 defrag_candidate = !ref_valid && !contig && virt; 3196 if (!ref_valid) { 3197 if (contig && count > npc_priv.subbank_depth) 3198 goto try_noref_multi_subbank; 3199 3200 rc = npc_subbank_noref_alloc(rvu, key_type, contig, 3201 count, mcam_idx); 3202 if (!rc) 3203 goto add2map; 3204 3205 try_noref_multi_subbank: 3206 eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; 3207 3208 if (prio == NPC_MCAM_HIGHER_PRIO) 3209 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3210 eidx, 0, 3211 NPC_MCAM_HIGHER_PRIO, 3212 contig, count, 3213 mcam_idx); 3214 else 3215 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3216 0, eidx, 3217 NPC_MCAM_LOWER_PRIO, 3218 contig, count, 3219 mcam_idx); 3220 3221 if (!rc) 3222 goto add2map; 3223 3224 return rc; 3225 } 3226 3227 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 3228 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 3229 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3230 __func__, ref, limit); 3231 return -EINVAL; 3232 } 3233 3234 if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || 3235 (key_type == NPC_MCAM_KEY_X2 && 3236 (ref >= 2 * bd || limit >= 2 * bd))) { 3237 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3238 __func__, ref, limit); 3239 return -EINVAL; 3240 } 3241 3242 if (contig && count > npc_priv.subbank_depth) 3243 goto try_ref_multi_subbank; 3244 3245 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3246 prio, contig, count, mcam_idx); 3247 if (!rc) 3248 goto add2map; 3249 3250 try_ref_multi_subbank: 3251 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3252 ref, limit, prio, 3253 contig, count, mcam_idx); 3254 if (!rc) 3255 goto add2map; 3256 3257 return rc; 3258 3259 add2map: 3260 for (i = 0; i < count; i++) { 3261 rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); 3262 if (rc) 3263 goto err; 3264 3265 if (!defrag_candidate) 3266 continue; 3267 3268 rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); 3269 if (rc) { 3270 npc_del_from_pf_maps(rvu, mcam_idx[i]); 3271 goto err; 3272 } 3273 3274 /* Return vidx to caller */ 3275 mcam_idx[i] = vidx; 3276 } 3277 3278 return 0; 3279 err: 3280 for (int j = 0; j < i; j++) { 3281 npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); 3282 3283 if (!defrag_candidate) 3284 continue; 3285 3286 npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); 3287 } 3288 3289 return rc; 3290 3291 } 3292 3293 void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, 3294 int *x4_free, int *sb_free) 3295 { 3296 struct npc_subbank *sb; 3297 int i; 3298 3299 /* Reset all stats to zero */ 3300 *x2_free = 0; 3301 *x4_free = 0; 3302 *sb_free = 0; 3303 3304 for (i = 0; i < npc_priv.num_subbanks; i++) { 3305 sb = &npc_priv.sb[i]; 3306 mutex_lock(&sb->lock); 3307 3308 /* Count number of free subbanks */ 3309 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 3310 (*sb_free)++; 3311 goto next; 3312 } 3313 3314 /* Sumup x4 free count */ 3315 if (sb->key_type == NPC_MCAM_KEY_X4) { 3316 (*x4_free) += sb->free_cnt; 3317 goto next; 3318 } 3319 3320 /* Sumup x2 free counts */ 3321 (*x2_free) += sb->free_cnt; 3322 next: 3323 mutex_unlock(&sb->lock); 3324 } 3325 } 3326 3327 int 3328 rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, 3329 struct msg_req *req, 3330 struct npc_cn20k_get_fcnt_rsp *rsp) 3331 { 3332 npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, 3333 &rsp->free_x4, &rsp->free_subbanks); 3334 return 0; 3335 } 3336 3337 int 3338 rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, 3339 struct msg_req *req, 3340 struct npc_cn20k_get_kex_cfg_rsp *rsp) 3341 { 3342 int extr, lt; 3343 3344 rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); 3345 rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); 3346 3347 /* Get EXTRACTOR LID */ 3348 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3349 rsp->intf_extr_lid[NIX_INTF_RX][extr] = 3350 CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); 3351 rsp->intf_extr_lid[NIX_INTF_TX][extr] = 3352 CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); 3353 } 3354 3355 /* Get EXTRACTOR LTYPE */ 3356 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3357 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3358 rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = 3359 CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); 3360 rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = 3361 CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); 3362 } 3363 } 3364 3365 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3366 return 0; 3367 } 3368 3369 static int *subbank_srch_order; 3370 3371 static void npc_populate_restricted_idxs(int num_subbanks) 3372 { 3373 npc_subbank_restricted_idxs[0] = num_subbanks - 1; 3374 npc_subbank_restricted_idxs[1] = 0; 3375 } 3376 3377 static int npc_create_srch_order(int cnt) 3378 { 3379 int val = 0; 3380 3381 subbank_srch_order = kcalloc(cnt, sizeof(int), 3382 GFP_KERNEL); 3383 if (!subbank_srch_order) 3384 return -ENOMEM; 3385 3386 /* cnt(subbank depth) is always a power of 2. There is a check in 3387 * npc_priv_init() to check the same. 3388 */ 3389 for (int i = 0; i < cnt; i += 2) { 3390 subbank_srch_order[i] = cnt / 2 - val - 1; 3391 subbank_srch_order[i + 1] = cnt / 2 + 1 + val; 3392 val++; 3393 } 3394 3395 subbank_srch_order[cnt - 1] = cnt / 2; 3396 return 0; 3397 } 3398 3399 static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) 3400 { 3401 mutex_init(&sb->lock); 3402 3403 sb->b0b = idx * npc_priv.subbank_depth; 3404 sb->b0t = sb->b0b + npc_priv.subbank_depth - 1; 3405 3406 sb->b1b = npc_priv.bank_depth + idx * npc_priv.subbank_depth; 3407 sb->b1t = sb->b1b + npc_priv.subbank_depth - 1; 3408 3409 sb->flags = NPC_SUBBANK_FLAG_FREE; 3410 sb->idx = idx; 3411 sb->arr_idx = subbank_srch_order[idx]; 3412 3413 dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", 3414 __func__, sb->idx, sb->arr_idx); 3415 3416 /* Keep first and last subbank at end of free array; so that 3417 * it will be used at last 3418 */ 3419 xa_store(&npc_priv.xa_sb_free, sb->arr_idx, 3420 xa_mk_value(sb->idx), GFP_KERNEL); 3421 } 3422 3423 static int npc_pcifunc_map_create(struct rvu *rvu) 3424 { 3425 int pf, vf, numvfs; 3426 int cnt = 0; 3427 u16 pcifunc; 3428 u64 cfg; 3429 3430 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { 3431 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3432 numvfs = (cfg >> 12) & 0xFF; 3433 3434 /* Skip not enabled PFs */ 3435 if (!(cfg & BIT_ULL(20))) 3436 goto chk_vfs; 3437 3438 /* If Admin function, check on VFs */ 3439 if (cfg & BIT_ULL(21)) 3440 goto chk_vfs; 3441 3442 pcifunc = pf << 9; 3443 3444 xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, 3445 xa_mk_value(cnt), GFP_KERNEL); 3446 3447 cnt++; 3448 3449 chk_vfs: 3450 for (vf = 0; vf < numvfs; vf++) { 3451 pcifunc = (pf << 9) | (vf + 1); 3452 3453 xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, 3454 xa_mk_value(cnt), GFP_KERNEL); 3455 cnt++; 3456 } 3457 } 3458 3459 return cnt; 3460 } 3461 3462 struct npc_defrag_node { 3463 u8 idx; 3464 u8 key_type; 3465 bool valid; 3466 bool refs; 3467 u16 free_cnt; 3468 u16 vidx_cnt; 3469 u16 *vidx; 3470 struct list_head list; 3471 }; 3472 3473 static bool npc_defrag_skip_restricted_sb(int sb_id) 3474 { 3475 int i; 3476 3477 if (!restrict_valid) 3478 return false; 3479 3480 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) 3481 if (sb_id == npc_subbank_restricted_idxs[i]) 3482 return true; 3483 return false; 3484 } 3485 3486 /* Find subbank with minimum number of virtual indexes */ 3487 static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) 3488 { 3489 struct npc_defrag_node *node, *tnode = NULL; 3490 int min = INT_MAX; 3491 3492 list_for_each_entry(node, lh, list) { 3493 if (!node->valid) 3494 continue; 3495 3496 /* if subbank has ref allocated mcam indexes, that subbank 3497 * is not a good candidate to move out indexes. 3498 */ 3499 if (node->refs) 3500 continue; 3501 3502 if (min > node->vidx_cnt) { 3503 min = node->vidx_cnt; 3504 tnode = node; 3505 } 3506 } 3507 3508 return tnode; 3509 } 3510 3511 /* Find subbank with maximum number of free spaces */ 3512 static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) 3513 { 3514 struct npc_defrag_node *node, *tnode = NULL; 3515 int max = INT_MIN; 3516 3517 list_for_each_entry(node, lh, list) { 3518 if (!node->valid) 3519 continue; 3520 3521 if (max < node->free_cnt) { 3522 max = node->free_cnt; 3523 tnode = node; 3524 } 3525 } 3526 3527 return tnode; 3528 } 3529 3530 static int npc_defrag_alloc_free_slots(struct rvu *rvu, 3531 struct npc_defrag_node *f, 3532 int cnt, u16 *save) 3533 { 3534 int alloc_cnt1, alloc_cnt2; 3535 struct npc_subbank *sb; 3536 int rc, sb_off, i, err; 3537 bool deleted; 3538 3539 sb = &npc_priv.sb[f->idx]; 3540 3541 alloc_cnt1 = 0; 3542 alloc_cnt2 = 0; 3543 3544 rc = __npc_subbank_alloc(rvu, sb, 3545 NPC_MCAM_KEY_X2, sb->b0b, 3546 sb->b0t, 3547 NPC_MCAM_LOWER_PRIO, 3548 false, cnt, save, cnt, true, 3549 &alloc_cnt1); 3550 3551 if (alloc_cnt1 < cnt) { 3552 rc = __npc_subbank_alloc(rvu, sb, 3553 NPC_MCAM_KEY_X2, sb->b1b, 3554 sb->b1t, 3555 NPC_MCAM_LOWER_PRIO, 3556 false, cnt - alloc_cnt1, 3557 save + alloc_cnt1, 3558 cnt - alloc_cnt1, 3559 true, &alloc_cnt2); 3560 } 3561 3562 if (alloc_cnt1 + alloc_cnt2 != cnt) { 3563 dev_err(rvu->dev, 3564 "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", 3565 __func__, cnt, alloc_cnt1, alloc_cnt2); 3566 rc = -ENOSPC; 3567 goto fail_free_alloc; 3568 } 3569 3570 return 0; 3571 3572 fail_free_alloc: 3573 for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { 3574 err = npc_mcam_idx_2_subbank_idx(rvu, save[i], 3575 &sb, &sb_off); 3576 if (err) { 3577 dev_err(rvu->dev, 3578 "%s: Error to find subbank for mcam idx=%u\n", 3579 __func__, save[i]); 3580 break; 3581 } 3582 3583 deleted = __npc_subbank_free(rvu, sb, sb_off); 3584 if (!deleted) { 3585 dev_err(rvu->dev, 3586 "%s: Error to free mcam idx=%u\n", 3587 __func__, save[i]); 3588 break; 3589 } 3590 } 3591 3592 return rc; 3593 } 3594 3595 static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, 3596 u16 new_midx, u16 vidx) 3597 { 3598 struct npc_defrag_show_node *node; 3599 3600 node = kcalloc(1, sizeof(*node), GFP_KERNEL); 3601 if (!node) 3602 return -ENOMEM; 3603 3604 node->old_midx = old_midx; 3605 node->new_midx = new_midx; 3606 node->vidx = vidx; 3607 INIT_LIST_HEAD(&node->list); 3608 3609 mutex_lock(&npc_priv.lock); 3610 list_add_tail(&node->list, &npc_priv.defrag_lh); 3611 mutex_unlock(&npc_priv.lock); 3612 3613 return 0; 3614 } 3615 3616 static 3617 int npc_defrag_move_vdx_to_free(struct rvu *rvu, 3618 struct npc_defrag_node *f, 3619 struct npc_defrag_node *v, 3620 int cnt, u16 *save) 3621 { 3622 u16 new_midx, old_midx, vidx, target_pf; 3623 struct npc_mcam *mcam = &rvu->hw->mcam; 3624 struct rvu_npc_mcam_rule *rule, *tmp; 3625 int i, vidx_cnt, rc, sb_off; 3626 struct npc_subbank *sb; 3627 bool deleted; 3628 u16 pcifunc; 3629 int blkaddr; 3630 void *map; 3631 u8 bank; 3632 u16 midx; 3633 u64 stats; 3634 3635 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3636 3637 vidx_cnt = v->vidx_cnt; 3638 for (i = 0; i < cnt; i++) { 3639 vidx = v->vidx[vidx_cnt - i - 1]; 3640 old_midx = npc_vidx2idx(vidx); 3641 new_midx = save[cnt - i - 1]; 3642 3643 dev_dbg(rvu->dev, 3644 "%s: Moving %u ---> %u (vidx=%u)\n", 3645 __func__, 3646 old_midx, new_midx, vidx); 3647 3648 rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); 3649 if (rc) 3650 dev_err(rvu->dev, 3651 "%s: Error happened to add to show list vidx=%u\n", 3652 __func__, vidx); 3653 3654 /* Modify vidx to point to new mcam idx */ 3655 rc = npc_vidx_maps_modify(rvu, vidx, new_midx); 3656 if (rc) 3657 return rc; 3658 3659 midx = old_midx % mcam->banksize; 3660 bank = old_midx / mcam->banksize; 3661 stats = rvu_read64(rvu, blkaddr, 3662 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, 3663 bank)); 3664 3665 /* If bug happened during copy/enable mcam, then there is a bug in allocation 3666 * algorithm itself. There is no point in rewinding and returning, as it 3667 * will face further issue. Return error after printing error 3668 */ 3669 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) { 3670 dev_err(rvu->dev, 3671 "%s: Error happened while disabling old_mid=%u\n", 3672 __func__, old_midx); 3673 return -EFAULT; 3674 } 3675 3676 if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) { 3677 dev_err(rvu->dev, 3678 "%s: Error happened while copying old_midx=%u new_midx=%u\n", 3679 __func__, old_midx, new_midx); 3680 return -EFAULT; 3681 } 3682 3683 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) { 3684 dev_err(rvu->dev, 3685 "%s: Error happened while enabling new_mid=%u\n", 3686 __func__, new_midx); 3687 return -EFAULT; 3688 } 3689 3690 midx = new_midx % mcam->banksize; 3691 bank = new_midx / mcam->banksize; 3692 rvu_write64(rvu, blkaddr, 3693 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), 3694 stats); 3695 3696 /* Free the old mcam idx */ 3697 rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, 3698 &sb, &sb_off); 3699 if (rc) { 3700 dev_err(rvu->dev, 3701 "%s: Unable to calculate subbank off for mcamidx=%u\n", 3702 __func__, old_midx); 3703 return rc; 3704 } 3705 3706 deleted = __npc_subbank_free(rvu, sb, sb_off); 3707 if (!deleted) { 3708 dev_err(rvu->dev, 3709 "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", 3710 __func__, old_midx, sb->idx, sb_off); 3711 return -EFAULT; 3712 } 3713 3714 /* save pcifunc */ 3715 map = xa_load(&npc_priv.xa_idx2pf_map, old_midx); 3716 pcifunc = xa_to_value(map); 3717 3718 /* delete from pf maps */ 3719 rc = npc_del_from_pf_maps(rvu, old_midx); 3720 if (rc) { 3721 dev_err(rvu->dev, 3722 "%s: Failed to delete pf maps for mcamidx=%u\n", 3723 __func__, old_midx); 3724 return rc; 3725 } 3726 3727 /* add new mcam_idx to pf map */ 3728 rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); 3729 if (rc) { 3730 dev_err(rvu->dev, 3731 "%s: Failed to add pf maps for mcamidx=%u\n", 3732 __func__, new_midx); 3733 return rc; 3734 } 3735 3736 /* Remove from mcam maps */ 3737 mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; 3738 mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; 3739 npc_mcam_clear_bit(mcam, old_midx); 3740 3741 mcam->entry2pfvf_map[new_midx] = pcifunc; 3742 /* Counter is not preserved */ 3743 mcam->entry2cntr_map[new_midx] = new_midx; 3744 target_pf = mcam->entry2target_pffunc[old_midx]; 3745 mcam->entry2target_pffunc[new_midx] = target_pf; 3746 mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP; 3747 3748 npc_mcam_set_bit(mcam, new_midx); 3749 3750 /* Note: list order is not functionally required for mcam_rules */ 3751 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 3752 if (rule->entry != old_midx) 3753 continue; 3754 3755 rule->entry = new_midx; 3756 break; 3757 } 3758 3759 /* Mark as invalid */ 3760 v->vidx[vidx_cnt - i - 1] = -1; 3761 save[cnt - i - 1] = -1; 3762 3763 f->free_cnt--; 3764 v->vidx_cnt--; 3765 } 3766 3767 return 0; 3768 } 3769 3770 static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) 3771 { 3772 struct npc_defrag_node *v = NULL; 3773 struct npc_defrag_node *f = NULL; 3774 int rc = 0, cnt; 3775 u16 *save; 3776 3777 while (1) { 3778 /* Find subbank with minimum vidx */ 3779 if (!v) { 3780 v = npc_subbank_min_vidx(lh); 3781 if (!v) 3782 break; 3783 } 3784 3785 /* Find subbank with maximum free slots */ 3786 if (!f) { 3787 f = npc_subbank_max_free(lh); 3788 if (!f) 3789 break; 3790 } 3791 3792 if (!v->vidx_cnt) { 3793 list_del_init(&v->list); 3794 v = NULL; 3795 continue; 3796 } 3797 3798 if (!f->free_cnt) { 3799 list_del_init(&f->list); 3800 f = NULL; 3801 continue; 3802 } 3803 3804 /* If both subbanks are same, choose vidx and 3805 * search for free list again 3806 */ 3807 if (f == v) { 3808 list_del_init(&f->list); 3809 f = NULL; 3810 continue; 3811 } 3812 3813 /* Calculate minimum free slots needs to be allocated */ 3814 cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : 3815 f->free_cnt; 3816 3817 dev_dbg(rvu->dev, 3818 "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", 3819 __func__, cnt, f->free_cnt, f->idx, 3820 v->vidx_cnt, v->idx); 3821 3822 /* Allocate an array to store newly allocated 3823 * free slots (mcam indexes) 3824 */ 3825 save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); 3826 if (!save) { 3827 rc = -ENOMEM; 3828 goto err; 3829 } 3830 3831 /* Alloc free slots for existing vidx */ 3832 rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); 3833 if (rc) { 3834 kfree(save); 3835 goto err; 3836 } 3837 3838 /* Move vidx to free slots; update pf_map and vidx maps, 3839 * and free existing vidx mcam slots 3840 */ 3841 rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); 3842 if (rc) { 3843 kfree(save); 3844 goto err; 3845 } 3846 3847 kfree(save); 3848 3849 if (!f->free_cnt) { 3850 list_del_init(&f->list); 3851 f = NULL; 3852 } 3853 3854 if (!v->vidx_cnt) { 3855 list_del_init(&v->list); 3856 v = NULL; 3857 } 3858 } 3859 3860 err: 3861 /* Whole defragmentation process is done within locks. if there 3862 * is an error, it would be hard to roll back as index remove/add 3863 * can fail again if it failed before. This would mean that there 3864 * is bug in the index management algorithm. 3865 * Return from here than rolling back. 3866 */ 3867 return rc; 3868 } 3869 3870 static void npc_defrag_list_clear(void) 3871 { 3872 struct npc_defrag_show_node *node, *next; 3873 3874 mutex_lock(&npc_priv.lock); 3875 list_for_each_entry_safe(node, next, &npc_priv.defrag_lh, list) { 3876 list_del_init(&node->list); 3877 kfree(node); 3878 } 3879 3880 mutex_unlock(&npc_priv.lock); 3881 } 3882 3883 static void npc_lock_all_subbank(void) 3884 { 3885 int i; 3886 3887 for (i = 0; i < npc_priv.num_subbanks; i++) 3888 mutex_lock(&npc_priv.sb[i].lock); 3889 } 3890 3891 static void npc_unlock_all_subbank(void) 3892 { 3893 int i; 3894 3895 for (i = npc_priv.num_subbanks - 1; i >= 0; i--) 3896 mutex_unlock(&npc_priv.sb[i].lock); 3897 } 3898 3899 /* Only non-ref non-contigous mcam indexes 3900 * are picked for defrag process 3901 */ 3902 int npc_cn20k_defrag(struct rvu *rvu) 3903 { 3904 struct npc_mcam *mcam = &rvu->hw->mcam; 3905 struct npc_defrag_node *node, *tnode; 3906 struct list_head x4lh, x2lh, *lh; 3907 int rc = 0, i, sb_off, tot; 3908 struct npc_subbank *sb; 3909 unsigned long index; 3910 void *map; 3911 u16 midx; 3912 3913 /* Free previous show list */ 3914 npc_defrag_list_clear(); 3915 3916 INIT_LIST_HEAD(&x4lh); 3917 INIT_LIST_HEAD(&x2lh); 3918 3919 node = kcalloc(npc_priv.num_subbanks, sizeof(*node), GFP_KERNEL); 3920 if (!node) 3921 return -ENOMEM; 3922 3923 /* Lock mcam */ 3924 mutex_lock(&mcam->lock); 3925 npc_lock_all_subbank(); 3926 3927 /* Fill in node with subbank properties */ 3928 for (i = 0; i < npc_priv.num_subbanks; i++) { 3929 sb = &npc_priv.sb[i]; 3930 3931 node[i].idx = i; 3932 node[i].key_type = sb->key_type; 3933 node[i].free_cnt = sb->free_cnt; 3934 node[i].vidx = kcalloc(npc_priv.subbank_depth * 2, 3935 sizeof(*node[i].vidx), 3936 GFP_KERNEL); 3937 if (!node[i].vidx) { 3938 rc = -ENOMEM; 3939 goto free_vidx; 3940 } 3941 3942 /* If subbank is empty, dont include it in defrag 3943 * process 3944 */ 3945 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 3946 node[i].valid = false; 3947 continue; 3948 } 3949 3950 if (npc_defrag_skip_restricted_sb(i)) { 3951 node[i].valid = false; 3952 continue; 3953 } 3954 3955 node[i].valid = true; 3956 INIT_LIST_HEAD(&node[i].list); 3957 3958 /* Add node to x2 or x4 list */ 3959 lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; 3960 list_add_tail(&node[i].list, lh); 3961 } 3962 3963 /* Filling vidx[] array with all vidx in that subbank */ 3964 xa_for_each_start(&npc_priv.xa_vidx2idx_map, index, map, 3965 npc_priv.bank_depth * 2) { 3966 midx = xa_to_value(map); 3967 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 3968 &sb, &sb_off); 3969 if (rc) { 3970 dev_err(rvu->dev, 3971 "%s: Error to get mcam_idx for vidx=%lu\n", 3972 __func__, index); 3973 goto free_vidx; 3974 } 3975 3976 tnode = &node[sb->idx]; 3977 tnode->vidx[tnode->vidx_cnt] = index; 3978 tnode->vidx_cnt++; 3979 } 3980 3981 /* Mark all subbank which has ref allocation */ 3982 for (i = 0; i < npc_priv.num_subbanks; i++) { 3983 tnode = &node[i]; 3984 3985 if (!tnode->valid) 3986 continue; 3987 3988 tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? 3989 npc_priv.subbank_depth * 2 : npc_priv.subbank_depth; 3990 3991 if (node[i].vidx_cnt != tot - tnode->free_cnt) 3992 tnode->refs = true; 3993 } 3994 3995 rc = npc_defrag_process(rvu, &x2lh); 3996 if (rc) 3997 goto free_vidx; 3998 3999 rc = npc_defrag_process(rvu, &x4lh); 4000 if (rc) 4001 goto free_vidx; 4002 4003 free_vidx: 4004 npc_unlock_all_subbank(); 4005 mutex_unlock(&mcam->lock); 4006 for (i = 0; i < npc_priv.num_subbanks; i++) 4007 kfree(node[i].vidx); 4008 kfree(node); 4009 return rc; 4010 } 4011 4012 int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, 4013 struct msg_rsp *rsp) 4014 { 4015 return npc_cn20k_defrag(rvu); 4016 } 4017 4018 int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, 4019 u16 *mcast, u16 *promisc, u16 *ucast) 4020 { 4021 u16 *ptr[4] = {promisc, mcast, bcast, ucast}; 4022 unsigned long idx; 4023 bool set = false; 4024 void *val; 4025 int i, j; 4026 4027 for (i = 0; i < ARRAY_SIZE(ptr); i++) { 4028 if (!ptr[i]) 4029 continue; 4030 4031 *ptr[i] = USHRT_MAX; 4032 } 4033 4034 if (!npc_priv.init_done) 4035 return 0; 4036 4037 if (is_lbk_vf(rvu, pcifunc)) { 4038 if (!ptr[0]) 4039 return -EINVAL; 4040 4041 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4042 val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); 4043 if (!val) { 4044 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4045 __func__, 4046 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4047 pcifunc); 4048 4049 return -ESRCH; 4050 } 4051 4052 *ptr[0] = xa_to_value(val); 4053 return 0; 4054 } 4055 4056 if (is_vf(pcifunc)) { 4057 if (!ptr[3]) 4058 return -EINVAL; 4059 4060 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4061 val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); 4062 if (!val) { 4063 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4064 __func__, 4065 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4066 pcifunc); 4067 4068 return -ESRCH; 4069 } 4070 4071 *ptr[3] = xa_to_value(val); 4072 return 0; 4073 } 4074 4075 for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, 4076 j++) { 4077 if (!ptr[j]) 4078 continue; 4079 4080 idx = NPC_DFT_RULE_ID_MK(pcifunc, i); 4081 val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); 4082 if (!val) { 4083 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4084 __func__, 4085 npc_dft_rule_name[i], pcifunc); 4086 4087 continue; 4088 } 4089 4090 *ptr[j] = xa_to_value(val); 4091 set = true; 4092 } 4093 4094 return set ? 0 : -ESRCH; 4095 } 4096 4097 int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, 4098 struct npc_get_pfl_info_rsp *rsp) 4099 { 4100 if (!is_cn20k(rvu->pdev)) { 4101 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4102 return -EOPNOTSUPP; 4103 } 4104 4105 rsp->kw_type = npc_priv.kw; 4106 rsp->x4_slots = npc_priv.bank_depth; 4107 return 0; 4108 } 4109 4110 int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, 4111 struct npc_get_num_kws_req *req, 4112 struct npc_get_num_kws_rsp *rsp) 4113 { 4114 u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4115 u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4116 struct rvu_npc_mcam_rule dummy = { 0 }; 4117 struct mcam_entry_mdata mdata = { }; 4118 struct npc_install_flow_req *fl; 4119 int i, cnt = 0, blkaddr; 4120 4121 if (!is_cn20k(rvu->pdev)) { 4122 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4123 return -EOPNOTSUPP; 4124 } 4125 4126 fl = &req->fl; 4127 4128 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4129 if (blkaddr < 0) { 4130 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4131 return NPC_MCAM_INVALID_REQ; 4132 } 4133 4134 mdata.kw = kw; 4135 mdata.kw_mask = kw_mask; 4136 4137 npc_update_flow(rvu, &mdata, fl->features, &fl->packet, 4138 &fl->mask, &dummy, fl->intf, blkaddr); 4139 4140 /* Find the most significant word valid. Traverse from 4141 * MSB to LSB, check if cam0 or cam1 is set 4142 */ 4143 for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { 4144 if (kw[i] || kw_mask[i]) { 4145 cnt = i + 1; 4146 break; 4147 } 4148 } 4149 4150 rsp->kws = cnt; 4151 4152 return 0; 4153 } 4154 4155 int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, 4156 struct npc_get_dft_rl_idxs_rsp *rsp) 4157 { 4158 u16 bcast, mcast, promisc, ucast; 4159 u16 pcifunc; 4160 int rc; 4161 4162 if (!is_cn20k(rvu->pdev)) { 4163 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4164 return -EOPNOTSUPP; 4165 } 4166 4167 pcifunc = req->hdr.pcifunc; 4168 4169 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 4170 &promisc, &ucast); 4171 if (rc) 4172 return rc; 4173 4174 rsp->bcast = bcast; 4175 rsp->mcast = mcast; 4176 rsp->promisc = promisc; 4177 rsp->ucast = ucast; 4178 return 0; 4179 } 4180 4181 bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) 4182 { 4183 return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || 4184 is_lbk_vf(rvu, pcifunc); 4185 } 4186 4187 void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) 4188 { 4189 struct npc_mcam *mcam = &rvu->hw->mcam; 4190 u16 ptr[4] = {[0 ... 3] = USHRT_MAX}; 4191 struct rvu_npc_mcam_rule *rule, *tmp; 4192 unsigned long index; 4193 int blkaddr, rc, i; 4194 void *map; 4195 4196 if (!npc_priv.init_done) 4197 return; 4198 4199 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4200 dev_dbg(rvu->dev, 4201 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4202 __func__, pcifunc); 4203 return; 4204 } 4205 4206 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 4207 &ptr[2], &ptr[3]); 4208 if (rc) 4209 return; 4210 4211 /* LBK */ 4212 if (is_lbk_vf(rvu, pcifunc)) { 4213 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4214 map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); 4215 if (!map) 4216 dev_dbg(rvu->dev, 4217 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4218 __func__, 4219 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4220 pcifunc); 4221 4222 goto free_rules; 4223 } 4224 4225 /* VF */ 4226 if (is_vf(pcifunc)) { 4227 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4228 map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); 4229 if (!map) 4230 dev_dbg(rvu->dev, 4231 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4232 __func__, 4233 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4234 pcifunc); 4235 4236 goto free_rules; 4237 } 4238 4239 /* PF */ 4240 for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { 4241 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4242 map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); 4243 if (!map) 4244 dev_dbg(rvu->dev, 4245 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4246 __func__, npc_dft_rule_name[i], 4247 pcifunc); 4248 } 4249 4250 free_rules: 4251 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4252 if (blkaddr < 0) 4253 return; 4254 for (int i = 0; i < 4; i++) { 4255 if (ptr[i] == USHRT_MAX) 4256 continue; 4257 4258 mutex_lock(&mcam->lock); 4259 npc_mcam_clear_bit(mcam, ptr[i]); 4260 mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP; 4261 npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false); 4262 mcam->entry2target_pffunc[ptr[i]] = 0x0; 4263 mutex_unlock(&mcam->lock); 4264 4265 rc = npc_cn20k_idx_free(rvu, &ptr[i], 1); 4266 if (rc) { 4267 /* Non recoverable error. Let us WARN and return. Keep system alive to 4268 * enable debugging 4269 */ 4270 WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n", 4271 __func__, pcifunc, ptr[i]); 4272 return; 4273 } 4274 } 4275 4276 mutex_lock(&mcam->lock); 4277 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 4278 for (int i = 0; i < 4; i++) { 4279 if (ptr[i] != rule->entry) 4280 continue; 4281 4282 list_del(&rule->list); 4283 kfree(rule); 4284 break; 4285 } 4286 } 4287 mutex_unlock(&mcam->lock); 4288 } 4289 4290 int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) 4291 { 4292 struct npc_mcam_free_entry_req free_req = { 0 }; 4293 u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; 4294 struct npc_mcam_alloc_entry_req req = { 0 }; 4295 struct npc_mcam_alloc_entry_rsp rsp = { 0 }; 4296 int ret, eidx, i, k, pf, cnt; 4297 struct rvu_pfvf *pfvf; 4298 unsigned long index; 4299 struct msg_rsp free_rsp; 4300 u16 b, m, p, u; 4301 4302 if (!npc_priv.init_done) 4303 return 0; 4304 4305 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4306 dev_dbg(rvu->dev, 4307 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4308 __func__, pcifunc); 4309 return 0; 4310 } 4311 4312 /* Check if default rules are already alloced for this pcifunc */ 4313 ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); 4314 if (!ret) { 4315 dev_dbg(rvu->dev, 4316 "%s: default rules are already installed (pcifunc=%#x)\n", 4317 __func__, pcifunc); 4318 dev_dbg(rvu->dev, 4319 "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", 4320 __func__, b, m, p, u); 4321 return 0; 4322 } 4323 4324 /* Set ref index as lowest priority index */ 4325 eidx = 2 * npc_priv.bank_depth - 1; 4326 4327 /* Install only UCAST for VF */ 4328 cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); 4329 4330 /* For VF pcifunc, allocate default mcam indexes by taking 4331 * ref as PF's ucast index. 4332 */ 4333 if (is_vf(pcifunc)) { 4334 pf = rvu_get_pf(rvu->pdev, pcifunc); 4335 pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; 4336 4337 /* Get PF's ucast entry index */ 4338 ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, 4339 NULL, NULL, &pf_ucast); 4340 4341 /* There is no PF rules installed; and VF installation comes 4342 * first. PF may come later. 4343 * TODO: Install PF rules before installing VF rules. 4344 */ 4345 4346 /* Set PF's ucast as ref entry */ 4347 if (!ret) 4348 eidx = pf_ucast; 4349 } 4350 4351 pfvf = rvu_get_pfvf(rvu, pcifunc); 4352 pfvf->hw_prio = NPC_DFT_RULE_PRIO; 4353 4354 req.contig = false; 4355 req.ref_prio = NPC_MCAM_HIGHER_PRIO; 4356 req.ref_entry = eidx; 4357 req.kw_type = NPC_MCAM_KEY_X2; 4358 req.count = cnt; 4359 req.hdr.pcifunc = pcifunc; 4360 4361 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4362 4363 /* successfully allocated index */ 4364 if (!ret) { 4365 /* Copy indexes to local array */ 4366 for (i = 0; i < cnt; i++) 4367 mcam_idx[i] = rsp.entry_list[i]; 4368 4369 goto chk_sanity; 4370 } 4371 4372 /* If there is no slots available and request is for PF, 4373 * return error. 4374 */ 4375 if (!is_vf(pcifunc)) { 4376 dev_err(rvu->dev, 4377 "%s: Default index allocation failed for pcifunc=%#x\n", 4378 __func__, pcifunc); 4379 return ret; 4380 } 4381 4382 /* We could not find an index with higher priority index for VF. 4383 * Find rule with lower priority index and set hardware priority 4384 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) 4385 */ 4386 req.contig = false; 4387 req.kw_type = NPC_MCAM_KEY_X2; 4388 req.count = cnt; 4389 req.hdr.pcifunc = pcifunc; 4390 req.ref_prio = NPC_MCAM_LOWER_PRIO; 4391 req.ref_entry = eidx + 1; 4392 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4393 if (ret) { 4394 dev_err(rvu->dev, 4395 "%s: Default index allocation failed for pcifunc=%#x\n", 4396 __func__, pcifunc); 4397 return ret; 4398 } 4399 4400 /* Copy indexes to local array */ 4401 for (i = 0; i < cnt; i++) 4402 mcam_idx[i] = rsp.entry_list[i]; 4403 4404 pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; 4405 4406 chk_sanity: 4407 /* LBK */ 4408 if (is_lbk_vf(rvu, pcifunc)) { 4409 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4410 ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, 4411 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4412 if (ret) { 4413 dev_err(rvu->dev, 4414 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4415 __func__, 4416 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4417 pcifunc); 4418 goto err; 4419 } 4420 4421 goto done; 4422 } 4423 4424 /* VF */ 4425 if (is_vf(pcifunc)) { 4426 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4427 ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, 4428 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4429 if (ret) { 4430 dev_err(rvu->dev, 4431 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4432 __func__, 4433 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4434 pcifunc); 4435 goto err; 4436 } 4437 4438 goto done; 4439 } 4440 4441 /* PF */ 4442 for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && 4443 k < cnt; i++, k++) { 4444 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4445 ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, 4446 xa_mk_value(mcam_idx[k]), GFP_KERNEL); 4447 if (ret) { 4448 dev_err(rvu->dev, 4449 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4450 __func__, npc_dft_rule_name[i], 4451 pcifunc); 4452 for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { 4453 index = NPC_DFT_RULE_ID_MK(pcifunc, p); 4454 xa_erase(&npc_priv.xa_pf2dfl_rmap, index); 4455 } 4456 goto err; 4457 } 4458 } 4459 4460 done: 4461 return 0; 4462 4463 err: 4464 free_req.hdr.pcifunc = pcifunc; 4465 free_req.all = 1; 4466 ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 4467 if (ret) 4468 dev_err(rvu->dev, 4469 "%s: Error deleting default entries (pcifunc=%#x\n", 4470 __func__, pcifunc); 4471 4472 return -EFAULT; 4473 } 4474 4475 static int npc_priv_init(struct rvu *rvu) 4476 { 4477 struct npc_mcam *mcam = &rvu->hw->mcam; 4478 int blkaddr, num_banks, bank_depth; 4479 int num_subbanks, subbank_depth; 4480 u64 npc_const1, npc_const2 = 0; 4481 struct npc_subbank *sb; 4482 u64 cfg; 4483 int i; 4484 4485 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4486 if (blkaddr < 0) { 4487 dev_err(rvu->dev, "%s: NPC block not implemented\n", 4488 __func__); 4489 return -ENODEV; 4490 } 4491 4492 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 4493 if (npc_const1 & BIT_ULL(63)) 4494 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 4495 4496 num_banks = mcam->banks; 4497 bank_depth = mcam->banksize; 4498 4499 num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); 4500 if (!num_subbanks) { 4501 dev_err(rvu->dev, "Number of subbanks is zero\n"); 4502 return -EFAULT; 4503 } 4504 4505 if (num_subbanks & (num_subbanks - 1)) { 4506 dev_err(rvu->dev, 4507 "subbanks cnt(%u) should be a power of 2\n", 4508 num_subbanks); 4509 return -EINVAL; 4510 } 4511 4512 npc_priv.num_subbanks = num_subbanks; 4513 4514 subbank_depth = bank_depth / num_subbanks; 4515 4516 npc_priv.bank_depth = bank_depth; 4517 npc_priv.subbank_depth = subbank_depth; 4518 4519 /* Get kex configured key size */ 4520 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); 4521 npc_priv.kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); 4522 4523 dev_info(rvu->dev, 4524 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", 4525 num_banks, bank_depth, num_subbanks, subbank_depth, 4526 npc_kw_name[npc_priv.kw]); 4527 4528 npc_priv.sb = kcalloc(num_subbanks, sizeof(struct npc_subbank), 4529 GFP_KERNEL); 4530 if (!npc_priv.sb) 4531 return -ENOMEM; 4532 4533 xa_init_flags(&npc_priv.xa_sb_used, XA_FLAGS_ALLOC); 4534 xa_init_flags(&npc_priv.xa_sb_free, XA_FLAGS_ALLOC); 4535 xa_init_flags(&npc_priv.xa_idx2pf_map, XA_FLAGS_ALLOC); 4536 xa_init_flags(&npc_priv.xa_pf_map, XA_FLAGS_ALLOC); 4537 xa_init_flags(&npc_priv.xa_pf2dfl_rmap, XA_FLAGS_ALLOC); 4538 xa_init_flags(&npc_priv.xa_idx2vidx_map, XA_FLAGS_ALLOC); 4539 xa_init_flags(&npc_priv.xa_vidx2idx_map, XA_FLAGS_ALLOC); 4540 4541 if (npc_create_srch_order(num_subbanks)) 4542 goto fail1; 4543 4544 npc_populate_restricted_idxs(num_subbanks); 4545 4546 /* Initialize subbanks */ 4547 for (i = 0, sb = npc_priv.sb; i < num_subbanks; i++, sb++) 4548 npc_subbank_init(rvu, sb, i); 4549 4550 /* Get number of pcifuncs in the system */ 4551 npc_priv.pf_cnt = npc_pcifunc_map_create(rvu); 4552 npc_priv.xa_pf2idx_map = kcalloc(npc_priv.pf_cnt, 4553 sizeof(struct xarray), 4554 GFP_KERNEL); 4555 if (!npc_priv.xa_pf2idx_map) 4556 goto fail2; 4557 4558 for (i = 0; i < npc_priv.pf_cnt; i++) 4559 xa_init_flags(&npc_priv.xa_pf2idx_map[i], XA_FLAGS_ALLOC); 4560 4561 INIT_LIST_HEAD(&npc_priv.defrag_lh); 4562 mutex_init(&npc_priv.lock); 4563 4564 return 0; 4565 4566 fail2: 4567 kfree(subbank_srch_order); 4568 subbank_srch_order = NULL; 4569 4570 fail1: 4571 xa_destroy(&npc_priv.xa_sb_used); 4572 xa_destroy(&npc_priv.xa_sb_free); 4573 xa_destroy(&npc_priv.xa_idx2pf_map); 4574 xa_destroy(&npc_priv.xa_pf_map); 4575 xa_destroy(&npc_priv.xa_pf2dfl_rmap); 4576 xa_destroy(&npc_priv.xa_idx2vidx_map); 4577 xa_destroy(&npc_priv.xa_vidx2idx_map); 4578 kfree(npc_priv.sb); 4579 npc_priv.sb = NULL; 4580 return -ENOMEM; 4581 } 4582 4583 void npc_cn20k_deinit(struct rvu *rvu) 4584 { 4585 int i; 4586 4587 xa_destroy(&npc_priv.xa_sb_used); 4588 xa_destroy(&npc_priv.xa_sb_free); 4589 xa_destroy(&npc_priv.xa_idx2pf_map); 4590 xa_destroy(&npc_priv.xa_pf_map); 4591 xa_destroy(&npc_priv.xa_pf2dfl_rmap); 4592 xa_destroy(&npc_priv.xa_idx2vidx_map); 4593 xa_destroy(&npc_priv.xa_vidx2idx_map); 4594 4595 for (i = 0; i < npc_priv.pf_cnt; i++) 4596 xa_destroy(&npc_priv.xa_pf2idx_map[i]); 4597 4598 kfree(npc_priv.xa_pf2idx_map); 4599 /* No need to destroy mutex lock as it is 4600 * part of subbank structure 4601 */ 4602 kfree(npc_priv.sb); 4603 kfree(subbank_srch_order); 4604 } 4605 4606 static int npc_setup_mcam_section(struct rvu *rvu, int key_type) 4607 { 4608 int blkaddr, sec; 4609 4610 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4611 if (blkaddr < 0) { 4612 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4613 return -ENODEV; 4614 } 4615 4616 for (sec = 0; sec < npc_priv.num_subbanks; sec++) 4617 rvu_write64(rvu, blkaddr, 4618 NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); 4619 4620 return 0; 4621 } 4622 4623 int npc_cn20k_init(struct rvu *rvu) 4624 { 4625 int err; 4626 4627 err = npc_priv_init(rvu); 4628 if (err) { 4629 dev_err(rvu->dev, "%s: Error to init\n", 4630 __func__); 4631 return err; 4632 } 4633 4634 err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); 4635 if (err) { 4636 dev_err(rvu->dev, "%s: mcam section configuration failure\n", 4637 __func__); 4638 return err; 4639 } 4640 4641 npc_priv.init_done = true; 4642 4643 return 0; 4644 } 4645