1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2026 Marvell. 5 * 6 */ 7 #include <linux/xarray.h> 8 #include <linux/bitfield.h> 9 10 #include "rvu.h" 11 #include "npc.h" 12 #include "npc_profile.h" 13 #include "rvu_npc_hash.h" 14 #include "rvu_npc.h" 15 #include "cn20k/npc.h" 16 #include "cn20k/reg.h" 17 #include "rvu_npc_fs.h" 18 19 static struct npc_priv_t *npc_priv; 20 21 static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { 22 [NPC_MCAM_KEY_DYN] = "DYNAMIC", 23 [NPC_MCAM_KEY_X2] = "X2", 24 [NPC_MCAM_KEY_X4] = "X4", 25 }; 26 27 static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { 28 [NPC_DFT_RULE_PROMISC_ID] = "Promisc", 29 [NPC_DFT_RULE_MCAST_ID] = "Mcast", 30 [NPC_DFT_RULE_BCAST_ID] = "Bcast", 31 [NPC_DFT_RULE_UCAST_ID] = "Ucast", 32 }; 33 34 #define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ 35 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 36 ((key_ofs) & 0x3F)) 37 38 #define NPC_DFT_RULE_ID_MK(pcifunc, id) \ 39 ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) 40 41 #define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ 42 FIELD_GET(GENMASK_ULL(15, 0), rid) 43 44 #define NPC_DFT_RULE_ID_2_ID(rid) \ 45 FIELD_GET(GENMASK_ULL(31, 16), rid) 46 47 #define NPC_DFT_RULE_PRIO 127 48 49 static const char cn20k_def_pfl_name[] = "default"; 50 51 static struct npc_mcam_kex_extr npc_mkex_extr_default = { 52 .mkex_sign = MKEX_CN20K_SIGN, 53 .name = "default", 54 .kpu_version = NPC_KPU_PROFILE_VER, 55 .keyx_cfg = { 56 /* nibble: LA..LE (ltype only) + Error code + Channel */ 57 [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | 58 NPC_CN20K_PARSE_NIBBLE_INTF_RX | 59 NPC_CN20K_PARSE_NIBBLE_ERRCODE, 60 61 /* nibble: LA..LE (ltype only) */ 62 [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | 63 NPC_CN20K_PARSE_NIBBLE_INTF_TX, 64 }, 65 .intf_extr_lid = { 66 /* Default RX MCAM KEX profile */ 67 [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 68 NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, 69 [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 70 NPC_LID_LC, NPC_LID_LD }, 71 }, 72 .intf_extr_lt = { 73 /* Default RX MCAM KEX profile */ 74 [NIX_INTF_RX] = { 75 [0] = { 76 /* Layer A: Ethernet: */ 77 [NPC_LT_LA_ETHER] = 78 /* DMAC: 6 bytes, KW1[63:15] */ 79 KEX_EXTR_CFG(0x05, 0x0, 0x1, 80 NPC_KEXOF_DMAC + 1), 81 [NPC_LT_LA_CPT_HDR] = 82 /* DMAC: 6 bytes, KW1[63:15] */ 83 KEX_EXTR_CFG(0x05, 0x0, 0x1, 84 NPC_KEXOF_DMAC + 1), 85 }, 86 [1] = { 87 /* Layer A: Ethernet: */ 88 [NPC_LT_LA_ETHER] = 89 /* Ethertype: 2 bytes, KW0[63:48] */ 90 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 91 [NPC_LT_LA_CPT_HDR] = 92 /* Ethertype: 2 bytes, KW0[63:48] */ 93 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 94 }, 95 [2] = { 96 /* Layer B: Single VLAN (CTAG) */ 97 [NPC_LT_LB_CTAG] = 98 /* CTAG VLAN: 2 bytes, KW1[15:0] */ 99 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 100 /* Layer B: Stacked VLAN (STAG|QinQ) */ 101 [NPC_LT_LB_STAG_QINQ] = 102 /* Outer VLAN: 2 bytes, KW1[15:0] */ 103 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 104 [NPC_LT_LB_FDSA] = 105 /* SWITCH PORT: 1 byte, KW1[7:0] */ 106 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), 107 }, 108 [3] = { 109 [NPC_LT_LB_CTAG] = 110 /* Ethertype: 2 bytes, KW0[63:48] */ 111 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 112 [NPC_LT_LB_STAG_QINQ] = 113 /* Ethertype: 2 bytes, KW0[63:48] */ 114 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), 115 [NPC_LT_LB_FDSA] = 116 /* Ethertype: 2 bytes, KW0[63:48] */ 117 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 118 }, 119 [4] = { 120 /* Layer C: IPv4 */ 121 [NPC_LT_LC_IP] = 122 /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ 123 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), 124 /* Layer C: IPv6 */ 125 [NPC_LT_LC_IP6] = 126 /* Everything up to SADDR: 8 bytes, KW3[7:0], 127 * KW2[63:8] 128 */ 129 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), 130 }, 131 [5] = { 132 [NPC_LT_LC_IP] = 133 /* TOS: 1 byte, KW2[7:0] */ 134 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), 135 }, 136 [6] = { 137 /* Layer D:UDP */ 138 [NPC_LT_LD_UDP] = 139 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 140 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 141 /* Layer D:TCP */ 142 [NPC_LT_LD_TCP] = 143 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 144 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 145 }, 146 }, 147 /* Default TX MCAM KEX profile */ 148 [NIX_INTF_TX] = { 149 [0] = { 150 /* Layer A: NIX_INST_HDR_S + Ethernet */ 151 /* NIX appends 8 bytes of NIX_INST_HDR_S at the 152 * start of each TX packet supplied to NPC. 153 */ 154 [NPC_LT_LA_IH_NIX_ETHER] = 155 /* PF_FUNC: 2B , KW0 [47:32] */ 156 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 157 /* Layer A: HiGig2: */ 158 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 159 /* PF_FUNC: 2B , KW0 [47:32] */ 160 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 161 }, 162 [1] = { 163 [NPC_LT_LA_IH_NIX_ETHER] = 164 /* SQ_ID 3 bytes, KW1[63:16] */ 165 KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), 166 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 167 /* VID: 2 bytes, KW1[31:16] */ 168 KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), 169 }, 170 [2] = { 171 /* Layer B: Single VLAN (CTAG) */ 172 [NPC_LT_LB_CTAG] = 173 /* CTAG VLAN[2..3] KW0[63:48] */ 174 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 175 /* Layer B: Stacked VLAN (STAG|QinQ) */ 176 [NPC_LT_LB_STAG_QINQ] = 177 /* Outer VLAN: 2 bytes, KW0[63:48] */ 178 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 179 }, 180 [3] = { 181 [NPC_LT_LB_CTAG] = 182 /* CTAG VLAN[2..3] KW1[15:0] */ 183 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), 184 [NPC_LT_LB_STAG_QINQ] = 185 /* Outer VLAN: 2 Bytes, KW1[15:0] */ 186 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), 187 }, 188 [4] = { 189 /* Layer C: IPv4 */ 190 [NPC_LT_LC_IP] = 191 /* SIP+DIP: 8 bytes, KW2[63:0] */ 192 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), 193 /* Layer C: IPv6 */ 194 [NPC_LT_LC_IP6] = 195 /* Everything up to SADDR: 8 bytes, KW2[63:0] */ 196 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), 197 }, 198 [5] = { 199 /* Layer D:UDP */ 200 [NPC_LT_LD_UDP] = 201 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 202 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 203 /* Layer D:TCP */ 204 [NPC_LT_LD_TCP] = 205 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 206 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 207 }, 208 }, 209 }, 210 }; 211 212 struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) 213 { 214 return &npc_mkex_extr_default; 215 } 216 217 static u16 npc_idx2vidx(u16 idx) 218 { 219 unsigned long index; 220 void *map; 221 u16 vidx; 222 int val; 223 224 vidx = idx; 225 index = idx; 226 227 map = xa_load(&npc_priv->xa_idx2vidx_map, index); 228 if (!map) 229 goto done; 230 231 val = xa_to_value(map); 232 if (val == -1) 233 goto done; 234 235 vidx = val; 236 237 done: 238 return vidx; 239 } 240 241 static bool npc_is_vidx(u16 vidx) 242 { 243 return vidx >= npc_priv->bank_depth * 2; 244 } 245 246 static u16 npc_vidx2idx(u16 vidx) 247 { 248 unsigned long sentinel = (unsigned long)-1; 249 unsigned long index; 250 void *map; 251 int val; 252 u16 idx; 253 254 idx = vidx; 255 index = vidx; 256 257 map = xa_load(&npc_priv->xa_vidx2idx_map, index); 258 if (!map) 259 goto done; 260 261 val = xa_to_value(map); 262 if (val == sentinel) 263 goto done; 264 265 idx = val; 266 267 done: 268 return idx; 269 } 270 271 u16 npc_cn20k_vidx2idx(u16 idx) 272 { 273 if (!npc_priv) 274 return idx; 275 276 if (!npc_is_vidx(idx)) 277 return idx; 278 279 return npc_vidx2idx(idx); 280 } 281 282 u16 npc_cn20k_idx2vidx(u16 idx) 283 { 284 if (!npc_priv) 285 return idx; 286 287 if (npc_is_vidx(idx)) 288 return idx; 289 290 return npc_idx2vidx(idx); 291 } 292 293 static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) 294 { 295 u16 mcam_idx; 296 void *map; 297 298 if (!npc_is_vidx(vidx)) { 299 dev_err(rvu->dev, 300 "%s: vidx(%u) does not map to proper mcam idx\n", 301 __func__, vidx); 302 return -ESRCH; 303 } 304 305 mcam_idx = npc_vidx2idx(vidx); 306 307 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 308 if (!map) { 309 dev_err(rvu->dev, 310 "%s: vidx(%u) does not map to proper mcam idx\n", 311 __func__, vidx); 312 return -ESRCH; 313 } 314 315 map = xa_erase(&npc_priv->xa_idx2vidx_map, mcam_idx); 316 if (!map) { 317 dev_err(rvu->dev, 318 "%s: vidx(%u) is not valid\n", 319 __func__, vidx); 320 return -ESRCH; 321 } 322 323 if (old_midx) 324 *old_midx = mcam_idx; 325 326 return 0; 327 } 328 329 static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) 330 { 331 u16 old_midx; 332 void *map; 333 int rc; 334 335 if (!npc_is_vidx(vidx)) { 336 dev_err(rvu->dev, 337 "%s: vidx(%u) does not map to proper mcam idx\n", 338 __func__, vidx); 339 return -ESRCH; 340 } 341 342 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 343 if (!map) { 344 dev_err(rvu->dev, 345 "%s: vidx(%u) could not be deleted from vidx2idx map\n", 346 __func__, vidx); 347 return -ESRCH; 348 } 349 350 old_midx = xa_to_value(map); 351 352 rc = xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 353 xa_mk_value(new_midx), GFP_KERNEL); 354 if (rc) { 355 dev_err(rvu->dev, 356 "%s: vidx(%u) cannot be added to vidx2idx map\n", 357 __func__, vidx); 358 goto fail1; 359 } 360 361 map = xa_erase(&npc_priv->xa_idx2vidx_map, old_midx); 362 if (!map) { 363 dev_err(rvu->dev, 364 "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 365 __func__, old_midx, vidx); 366 rc = -ESRCH; 367 goto fail2; 368 } 369 370 rc = xa_insert(&npc_priv->xa_idx2vidx_map, new_midx, 371 xa_mk_value(vidx), GFP_KERNEL); 372 if (rc) { 373 dev_err(rvu->dev, 374 "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 375 __func__, new_midx, vidx); 376 goto fail3; 377 } 378 379 return 0; 380 381 fail3: 382 /* Restore vidx at old_midx location */ 383 if (xa_insert(&npc_priv->xa_idx2vidx_map, old_midx, 384 xa_mk_value(vidx), GFP_KERNEL)) 385 dev_err(rvu->dev, 386 "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", 387 __func__, old_midx, vidx); 388 fail2: 389 /* Erase new_midx inserted at vidx */ 390 if (!xa_erase(&npc_priv->xa_vidx2idx_map, vidx)) 391 dev_err(rvu->dev, 392 "%s: Failed to roll back vidx2idx vidx=%u\n", 393 __func__, vidx); 394 395 fail1: 396 /* Restore old_midx at vidx location */ 397 if (xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 398 xa_mk_value(old_midx), GFP_KERNEL)) 399 dev_err(rvu->dev, 400 "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", 401 __func__, old_midx, vidx); 402 403 return rc; 404 } 405 406 static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, 407 u16 *vidx) 408 { 409 int rc, max, min; 410 u32 id; 411 412 /* Virtual index start from maximum mcam index + 1 */ 413 max = npc_priv->bank_depth * 2 * 2 - 1; 414 min = npc_priv->bank_depth * 2; 415 416 rc = xa_alloc(&npc_priv->xa_vidx2idx_map, &id, 417 xa_mk_value(mcam_idx), 418 XA_LIMIT(min, max), GFP_KERNEL); 419 if (rc) { 420 dev_err(rvu->dev, 421 "%s: Failed to add to vidx2idx map (%u)\n", 422 __func__, mcam_idx); 423 goto fail1; 424 } 425 426 rc = xa_insert(&npc_priv->xa_idx2vidx_map, mcam_idx, 427 xa_mk_value(id), GFP_KERNEL); 428 if (rc) { 429 dev_err(rvu->dev, 430 "%s: Failed to add to idx2vidx map (%u)\n", 431 __func__, mcam_idx); 432 goto fail2; 433 } 434 435 if (vidx) 436 *vidx = id; 437 438 return 0; 439 440 fail2: 441 xa_erase(&npc_priv->xa_vidx2idx_map, id); 442 fail1: 443 return rc; 444 } 445 446 static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, 447 const struct npc_kpu_profile_cam *kpucam, 448 int kpm, int entry) 449 { 450 struct npc_kpu_cam cam0 = {0}; 451 struct npc_kpu_cam cam1 = {0}; 452 453 cam1.state = kpucam->state & kpucam->state_mask; 454 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 455 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 456 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 457 458 cam0.state = ~kpucam->state & kpucam->state_mask; 459 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 460 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 461 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 462 463 rvu_write64(rvu, blkaddr, 464 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); 465 rvu_write64(rvu, blkaddr, 466 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); 467 } 468 469 static void 470 npc_config_kpmaction(struct rvu *rvu, int blkaddr, 471 const struct npc_kpu_profile_action *kpuaction, 472 int kpm, int entry, bool pkind) 473 { 474 struct npc_kpm_action0 action0 = {0}; 475 struct npc_kpu_action1 action1 = {0}; 476 u64 reg; 477 478 action1.errlev = kpuaction->errlev; 479 action1.errcode = kpuaction->errcode; 480 action1.dp0_offset = kpuaction->dp0_offset; 481 action1.dp1_offset = kpuaction->dp1_offset; 482 action1.dp2_offset = kpuaction->dp2_offset; 483 484 if (pkind) 485 reg = NPC_AF_PKINDX_ACTION1(entry); 486 else 487 reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); 488 489 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 490 491 action0.byp_count = kpuaction->bypass_count & 0x7; 492 action0.capture_ena = kpuaction->cap_ena & 1; 493 action0.parse_done = kpuaction->parse_done & 1; 494 action0.next_state = kpuaction->next_state & 0xf; 495 action0.capture_lid = kpuaction->lid & 0x7; 496 497 /* Parser functionality will work correctly even though 498 * upper flag bits are silently discarded 499 */ 500 action0.capture_ltype = kpuaction->ltype & 0xf; 501 action0.capture_flags = kpuaction->flags & 0xf; 502 action0.ptr_advance = kpuaction->ptr_advance; 503 504 action0.var_len_offset = kpuaction->offset; 505 action0.var_len_mask = kpuaction->mask; 506 action0.var_len_right = kpuaction->right & 1; 507 action0.var_len_shift = kpuaction->shift & 1; 508 509 if (pkind) 510 reg = NPC_AF_PKINDX_ACTION0(entry); 511 else 512 reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); 513 514 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 515 } 516 517 static void 518 npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, 519 int kpm, int start_entry, 520 const struct npc_kpu_profile *profile) 521 { 522 int num_cam_entries, num_action_entries; 523 int entry, num_entries, max_entries; 524 u64 idx; 525 526 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile); 527 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile); 528 529 if (num_cam_entries != num_action_entries) { 530 dev_err(rvu->dev, 531 "kpm%d: CAM and action entries [%d != %d] not equal\n", 532 kpm, num_cam_entries, num_action_entries); 533 534 WARN(1, "Fatal error\n"); 535 return; 536 } 537 538 max_entries = rvu->hw->npc_kpu_entries / 2; 539 entry = start_entry; 540 /* Program CAM match entries for previous kpm extracted data */ 541 num_entries = min_t(int, num_cam_entries, max_entries); 542 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 543 npc_config_kpmcam(rvu, blkaddr, 544 npc_get_kpu_cam_nth_entry(rvu, profile, idx), 545 kpm, entry); 546 547 entry = start_entry; 548 /* Program this kpm's actions */ 549 num_entries = min_t(int, num_action_entries, max_entries); 550 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 551 npc_config_kpmaction(rvu, blkaddr, 552 npc_get_kpu_action_nth_entry(rvu, profile, idx), 553 kpm, entry, false); 554 } 555 556 static void 557 npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) 558 { 559 u64 entry_mask; 560 561 entry_mask = npc_enable_mask(num_entries); 562 /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ 563 if (!rvu->kpu.custom) 564 entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); 567 if (num_entries <= 64) { 568 /* Disable all the entries in W1, W2 and W3 */ 569 rvu_write64(rvu, blkaddr, 570 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), 571 npc_enable_mask(0)); 572 rvu_write64(rvu, blkaddr, 573 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 574 npc_enable_mask(0)); 575 rvu_write64(rvu, blkaddr, 576 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 577 npc_enable_mask(0)); 578 return; 579 } 580 581 num_entries = num_entries - 64; 582 entry_mask = npc_enable_mask(num_entries); 583 rvu_write64(rvu, blkaddr, 584 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); 585 if (num_entries <= 64) { 586 /* Disable all the entries in W2 and W3 */ 587 rvu_write64(rvu, blkaddr, 588 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 589 npc_enable_mask(0)); 590 rvu_write64(rvu, blkaddr, 591 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 592 npc_enable_mask(0)); 593 return; 594 } 595 596 num_entries = num_entries - 64; 597 entry_mask = npc_enable_mask(num_entries); 598 rvu_write64(rvu, blkaddr, 599 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); 600 if (num_entries <= 64) { 601 /* Disable all the entries in W3 */ 602 rvu_write64(rvu, blkaddr, 603 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 604 npc_enable_mask(0)); 605 return; 606 } 607 608 num_entries = num_entries - 64; 609 entry_mask = npc_enable_mask(num_entries); 610 rvu_write64(rvu, blkaddr, 611 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); 612 } 613 614 #define KPU_OFFSET 8 615 static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) 616 { 617 const struct npc_kpu_profile *profile1, *profile2; 618 int pfl1_num_cam_entries, pfl2_num_cam_entries; 619 int idx, total_cam_entries; 620 621 for (idx = 0; idx < num_kpms; idx++) { 622 profile1 = &rvu->kpu.kpu[idx]; 623 pfl1_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile1); 624 npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); 625 profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; 626 pfl2_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile2); 627 628 npc_program_single_kpm_profile(rvu, blkaddr, idx, 629 pfl1_num_cam_entries, 630 profile2); 631 total_cam_entries = pfl1_num_cam_entries + pfl2_num_cam_entries; 632 npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); 633 rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), 634 pfl1_num_cam_entries); 635 /* Enable the KPUs associated with this KPM */ 636 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); 637 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), 638 0x01); 639 } 640 } 641 642 void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) 643 { 644 struct npc_kpu_profile_action *act; 645 struct rvu_hwinfo *hw = rvu->hw; 646 int num_pkinds, idx; 647 648 /* Disable all KPMs and their entries */ 649 for (idx = 0; idx < hw->npc_kpms; idx++) { 650 rvu_write64(rvu, blkaddr, 651 NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); 652 rvu_write64(rvu, blkaddr, 653 NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); 654 rvu_write64(rvu, blkaddr, 655 NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); 656 rvu_write64(rvu, blkaddr, 657 NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); 658 } 659 660 for (idx = 0; idx < hw->npc_kpus; idx++) 661 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 662 663 /* Load and customize KPU profile. */ 664 npc_load_kpu_profile(rvu); 665 666 /* Configure KPU and KPM mapping for second pass */ 667 rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); 668 669 /* First program IKPU profile i.e PKIND configs. 670 * Check HW max count to avoid configuring junk or 671 * writing to unsupported CSR addresses. 672 */ 673 num_pkinds = rvu->kpu.pkinds; 674 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 675 676 /* Cn20k does not support Custom profile from filesystem */ 677 for (idx = 0; idx < num_pkinds; idx++) { 678 act = npc_get_ikpu_nth_entry(rvu, idx); 679 if (!act) 680 continue; 681 682 npc_config_kpmaction(rvu, blkaddr, act, 683 0, idx, true); 684 } 685 686 /* Program KPM CAM and Action profiles */ 687 npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); 688 } 689 690 struct npc_priv_t *npc_priv_get(void) 691 { 692 return npc_priv; 693 } 694 695 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 696 const struct npc_mcam_kex_extr *mkex_extr, 697 u8 intf) 698 { 699 u8 num_extr = rvu->hw->npc_kex_extr; 700 int extr, lt; 701 u64 val; 702 703 if (is_npc_intf_tx(intf)) 704 return; 705 706 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 707 mkex_extr->keyx_cfg[NIX_INTF_RX]); 708 709 /* Program EXTRACTOR */ 710 for (extr = 0; extr < num_extr; extr++) 711 rvu_write64(rvu, blkaddr, 712 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 713 mkex_extr->intf_extr_lid[intf][extr]); 714 715 /* Program EXTRACTOR_LTYPE */ 716 for (extr = 0; extr < num_extr; extr++) { 717 for (lt = 0; lt < NPC_MAX_LT; lt++) { 718 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 719 CN20K_SET_EXTR_LT(intf, extr, lt, val); 720 } 721 } 722 } 723 724 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 725 const struct npc_mcam_kex_extr *mkex_extr, 726 u8 intf) 727 { 728 u8 num_extr = rvu->hw->npc_kex_extr; 729 int extr, lt; 730 u64 val; 731 732 if (is_npc_intf_rx(intf)) 733 return; 734 735 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 736 mkex_extr->keyx_cfg[NIX_INTF_TX]); 737 738 /* Program EXTRACTOR */ 739 for (extr = 0; extr < num_extr; extr++) 740 rvu_write64(rvu, blkaddr, 741 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 742 mkex_extr->intf_extr_lid[intf][extr]); 743 744 /* Program EXTRACTOR_LTYPE */ 745 for (extr = 0; extr < num_extr; extr++) { 746 for (lt = 0; lt < NPC_MAX_LT; lt++) { 747 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 748 CN20K_SET_EXTR_LT(intf, extr, lt, val); 749 } 750 } 751 } 752 753 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 754 const struct npc_mcam_kex_extr *mkex_extr) 755 { 756 struct rvu_hwinfo *hw = rvu->hw; 757 u8 intf; 758 759 for (intf = 0; intf < hw->npc_intfs; intf++) { 760 npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); 761 npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); 762 } 763 764 /* Programme mkex hash profile */ 765 npc_program_mkex_hash(rvu, blkaddr); 766 } 767 768 void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, 769 const char *mkex_profile) 770 { 771 struct npc_mcam_kex_extr *mcam_kex_extr; 772 struct device *dev = &rvu->pdev->dev; 773 void __iomem *mkex_prfl_addr = NULL; 774 u64 prfl_sz; 775 int ret; 776 777 /* If user not selected mkex profile */ 778 if (rvu->kpu_fwdata_sz || 779 !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) 780 goto program_mkex_extr; 781 782 /* Setting up the mapping for mkex profile image */ 783 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 784 if (ret < 0) 785 goto program_mkex_extr; 786 787 mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; 788 789 while (((s64)prfl_sz > 0) && 790 (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { 791 /* Compare with mkex mod_param name string */ 792 if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && 793 !strncmp(mcam_kex_extr->name, mkex_profile, 794 MKEX_NAME_LEN)) { 795 rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; 796 goto program_mkex_extr; 797 } 798 799 mcam_kex_extr++; 800 prfl_sz -= sizeof(struct npc_mcam_kex_extr); 801 } 802 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 803 rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 804 805 program_mkex_extr: 806 dev_info(rvu->dev, "Using %s mkex profile\n", 807 rvu->kpu.mcam_kex_prfl.mkex_extr->name); 808 /* Program selected mkex profile */ 809 npc_program_mkex_profile(rvu, blkaddr, 810 rvu->kpu.mcam_kex_prfl.mkex_extr); 811 if (mkex_prfl_addr) 812 iounmap(mkex_prfl_addr); 813 } 814 815 int 816 npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, 817 int index, bool enable) 818 { 819 struct npc_mcam *mcam = &rvu->hw->mcam; 820 int mcam_idx = index % mcam->banksize; 821 int bank = index / mcam->banksize; 822 u64 cfg, hw_prio; 823 u8 kw_type; 824 825 if (index < 0 || index >= mcam->total_entries) 826 return -EINVAL; 827 828 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 829 return -EINVAL; 830 831 if (kw_type == NPC_MCAM_KEY_X2) { 832 cfg = rvu_read64(rvu, blkaddr, 833 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 834 bank)); 835 hw_prio = cfg & GENMASK_ULL(30, 24); 836 cfg = enable ? 1 : 0; 837 cfg |= hw_prio; 838 rvu_write64(rvu, blkaddr, 839 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 840 cfg); 841 goto update_en_map; 842 } 843 844 /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks 845 * need to be programmed with the same value. 846 */ 847 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 848 cfg = rvu_read64(rvu, blkaddr, 849 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 850 bank)); 851 hw_prio = cfg & GENMASK_ULL(30, 24); 852 cfg = enable ? 1 : 0; 853 cfg |= hw_prio; 854 rvu_write64(rvu, blkaddr, 855 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 856 cfg); 857 } 858 859 update_en_map: 860 assign_bit(index, npc_priv->en_map, enable); 861 862 return 0; 863 } 864 865 static void 866 npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index) 867 { 868 rvu_write64(rvu, blkaddr, 869 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 870 0); 871 rvu_write64(rvu, blkaddr, 872 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 873 0); 874 875 rvu_write64(rvu, blkaddr, 876 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); 877 rvu_write64(rvu, blkaddr, 878 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); 879 880 rvu_write64(rvu, blkaddr, 881 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); 882 rvu_write64(rvu, blkaddr, 883 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); 884 885 rvu_write64(rvu, blkaddr, 886 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); 887 rvu_write64(rvu, blkaddr, 888 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); 889 890 rvu_write64(rvu, blkaddr, 891 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); 892 rvu_write64(rvu, blkaddr, 893 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); 894 895 /* Clear corresponding stats register */ 896 rvu_write64(rvu, blkaddr, 897 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 898 } 899 900 int 901 npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx) 902 { 903 struct npc_mcam *mcam = &rvu->hw->mcam; 904 int bank = npc_get_bank(mcam, mcam_idx); 905 u8 kw_type; 906 int index; 907 908 if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type)) 909 return -EINVAL; 910 911 index = mcam_idx & (mcam->banksize - 1); 912 913 if (kw_type == NPC_MCAM_KEY_X2) { 914 npc_clear_x2_entry(rvu, blkaddr, bank, index); 915 return 0; 916 } 917 918 /* For NPC_MCAM_KEY_X4 keys, both the banks 919 * need to be programmed with the same value. 920 */ 921 for (bank = 0; bank < mcam->banks_per_entry; bank++) 922 npc_clear_x2_entry(rvu, blkaddr, bank, index); 923 924 return 0; 925 } 926 927 static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, 928 u64 *cam0, u64 *cam1) 929 { 930 u64 kw_mask; 931 932 /* The two banks of every MCAM entry are used as a single double-wide 933 * entry that is compared with the search key as follows: 934 * 935 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] 936 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] 937 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] 938 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] 939 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] 940 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] 941 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] 942 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] 943 */ 944 *cam1 = entry->kw[idx]; 945 kw_mask = entry->kw_mask[idx]; 946 *cam1 &= kw_mask; 947 *cam0 = ~*cam1 & kw_mask; 948 } 949 950 /*------------------------------------------------------------------------- 951 *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | 952 * in | | | | || | | 953 * profile| len | | | ||len | type | 954 *------------------------------------------------------------------------- 955 *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | 956 *------------------------------------------------------------------------| 957 *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 958 *------------------------------------------------------------------------| 959 *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 960 *------------------------------------------------------------------------| 961 *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 962 *------------------------------------------------------------------------| 963 *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 964 *------------------------------------------------------------------------| 965 *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 966 *------------------------------------------------------------------------| 967 *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 968 *------------------------------------------------------------------------| 969 * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | 970 *------------------------------------------------------------------------| 971 */ 972 static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, 973 int blkaddr, int index, u8 intf, 974 struct cn20k_mcam_entry *entry, 975 int bank, u8 kw_type, int kw, 976 u8 req_kw_type) 977 { 978 u64 intf_ext = 0, intf_ext_mask = 0; 979 u8 tx_intf_mask = ~intf & 0x3; 980 u8 tx_intf = intf, kex_type; 981 u8 kw_type_mask = ~kw_type; 982 u64 cam0, cam1, kex_cfg; 983 984 if (is_npc_intf_tx(intf)) { 985 /* Last bit must be set and rest don't care 986 * for TX interfaces 987 */ 988 tx_intf_mask = 0x1; 989 tx_intf = intf & tx_intf_mask; 990 tx_intf_mask = ~tx_intf & tx_intf_mask; 991 } 992 993 kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 994 kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; 995 if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && 996 kw_type == NPC_MCAM_KEY_X2) { 997 kw_type = 0; 998 kw_type_mask = 0; 999 } 1000 1001 /* Say, we need to write x2 keyword in an x4 subbank. 1002 * req_kw_type will be x2, and kw_type will be x4. 1003 * So in the case ignore kw bits in mcam. 1004 */ 1005 if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { 1006 kw_type = 0; 1007 kw_type_mask = 0; 1008 } 1009 1010 intf_ext = ((u64)kw_type << 16) | tx_intf; 1011 intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | 1012 tx_intf_mask; 1013 rvu_write64(rvu, blkaddr, 1014 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 1015 intf_ext); 1016 rvu_write64(rvu, blkaddr, 1017 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 1018 intf_ext_mask); 1019 1020 /* Set the match key */ 1021 npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); 1022 rvu_write64(rvu, blkaddr, 1023 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 1024 cam1); 1025 rvu_write64(rvu, blkaddr, 1026 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 1027 cam0); 1028 1029 npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); 1030 rvu_write64(rvu, blkaddr, 1031 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 1032 cam1); 1033 rvu_write64(rvu, blkaddr, 1034 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 1035 cam0); 1036 1037 npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); 1038 rvu_write64(rvu, blkaddr, 1039 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 1040 cam1); 1041 rvu_write64(rvu, blkaddr, 1042 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 1043 cam0); 1044 1045 npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); 1046 rvu_write64(rvu, blkaddr, 1047 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 1048 cam1); 1049 rvu_write64(rvu, blkaddr, 1050 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 1051 cam0); 1052 } 1053 1054 static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, 1055 int blkaddr, int index, u8 intf, 1056 struct cn20k_mcam_entry *entry, 1057 u8 kw_type, u8 req_kw_type) 1058 { 1059 int kw = 0, bank; 1060 1061 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) 1062 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1063 index, intf, 1064 entry, bank, kw_type, 1065 kw, req_kw_type); 1066 } 1067 1068 int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, 1069 u8 intf, struct cn20k_mcam_entry *entry, 1070 bool enable, u8 hw_prio, u8 req_kw_type) 1071 { 1072 struct npc_mcam *mcam = &rvu->hw->mcam; 1073 int mcam_idx = index % mcam->banksize; 1074 int bank = index / mcam->banksize; 1075 u64 bank_cfg = (u64)hw_prio << 24; 1076 int kw = 0; 1077 u8 kw_type; 1078 1079 if (index < 0 || index >= mcam->total_entries) 1080 return -EINVAL; 1081 1082 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1083 return -EINVAL; 1084 1085 /* Disable before mcam entry update */ 1086 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false)) 1087 return -EINVAL; 1088 1089 /* CAM1 takes the comparison value and 1090 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 1091 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 1092 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 1093 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 1094 */ 1095 if (kw_type == NPC_MCAM_KEY_X2) { 1096 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1097 npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx); 1098 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1099 mcam_idx, intf, entry, 1100 bank, kw_type, kw, req_kw_type); 1101 /* Set 'action' */ 1102 rvu_write64(rvu, blkaddr, 1103 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1104 bank, 0), 1105 entry->action); 1106 1107 /* Set 'action2' for inline receive */ 1108 rvu_write64(rvu, blkaddr, 1109 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1110 bank, 2), 1111 entry->action2); 1112 1113 /* Set TAG 'action' */ 1114 rvu_write64(rvu, blkaddr, 1115 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1116 bank, 1), 1117 entry->vtag_action); 1118 1119 /* Set HW priority */ 1120 rvu_write64(rvu, blkaddr, 1121 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1122 bank_cfg); 1123 1124 } else { 1125 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1126 npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx); 1127 npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx); 1128 1129 npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, 1130 mcam_idx, intf, entry, 1131 kw_type, req_kw_type); 1132 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1133 /* Set 'action' */ 1134 rvu_write64(rvu, blkaddr, 1135 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1136 bank, 0), 1137 entry->action); 1138 1139 /* Set TAG 'action' */ 1140 rvu_write64(rvu, blkaddr, 1141 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1142 bank, 1), 1143 entry->vtag_action); 1144 1145 /* Set 'action2' for inline receive */ 1146 rvu_write64(rvu, blkaddr, 1147 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1148 bank, 2), 1149 entry->action2); 1150 1151 /* Set HW priority */ 1152 rvu_write64(rvu, blkaddr, 1153 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1154 bank_cfg); 1155 } 1156 } 1157 1158 /* TODO: */ 1159 /* PF installing VF rule */ 1160 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 1161 return -EINVAL; 1162 1163 return 0; 1164 } 1165 1166 int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) 1167 { 1168 struct npc_mcam *mcam = &rvu->hw->mcam; 1169 u64 cfg, sreg, dreg, soff, doff; 1170 u8 src_kwtype, dest_kwtype; 1171 int bank, i, sb, db; 1172 int dbank, sbank; 1173 1174 if (src >= mcam->total_entries || dest >= mcam->total_entries) 1175 return -EINVAL; 1176 1177 dbank = npc_get_bank(mcam, dest); 1178 sbank = npc_get_bank(mcam, src); 1179 1180 if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype)) 1181 return -EINVAL; 1182 1183 if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype)) 1184 return -EINVAL; 1185 1186 if (src_kwtype != dest_kwtype) 1187 return -EINVAL; 1188 1189 src &= (mcam->banksize - 1); 1190 dest &= (mcam->banksize - 1); 1191 1192 /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ 1193 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1194 sb = sbank + bank; 1195 sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); 1196 db = dbank + bank; 1197 dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); 1198 for (i = 0; i < 10; i++) { 1199 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 1200 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 1201 } 1202 1203 /* Copy action */ 1204 for (i = 0; i < 3; i++) { 1205 soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, 1206 sb, i); 1207 cfg = rvu_read64(rvu, blkaddr, soff); 1208 1209 doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, 1210 i); 1211 rvu_write64(rvu, blkaddr, doff, cfg); 1212 } 1213 1214 /* Copy bank configuration */ 1215 cfg = rvu_read64(rvu, blkaddr, 1216 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); 1217 rvu_write64(rvu, blkaddr, 1218 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); 1219 if (src_kwtype == NPC_MCAM_KEY_X2) 1220 break; 1221 } 1222 1223 return 0; 1224 } 1225 1226 static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, 1227 u64 cam0, u64 cam1) 1228 { 1229 entry->kw[idx] = cam1; 1230 entry->kw_mask[idx] = cam1 ^ cam0; 1231 } 1232 1233 int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, 1234 struct cn20k_mcam_entry *entry, 1235 u8 *intf, u8 *ena, u8 *hw_prio) 1236 { 1237 struct npc_mcam *mcam = &rvu->hw->mcam; 1238 u64 cam0, cam1, bank_cfg, cfg; 1239 int kw = 0, bank; 1240 u8 kw_type; 1241 1242 if (index >= mcam->total_entries) 1243 return -EINVAL; 1244 1245 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1246 return -EINVAL; 1247 1248 bank = npc_get_bank(mcam, index); 1249 index &= (mcam->banksize - 1); 1250 1251 cfg = rvu_read64(rvu, blkaddr, 1252 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0)); 1253 entry->action = cfg; 1254 1255 cfg = rvu_read64(rvu, blkaddr, 1256 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1)); 1257 entry->vtag_action = cfg; 1258 1259 cfg = rvu_read64(rvu, blkaddr, 1260 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2)); 1261 entry->action2 = cfg; 1262 1263 cfg = rvu_read64(rvu, blkaddr, 1264 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, 1265 bank, 1)) & 3; 1266 *intf = cfg; 1267 1268 bank_cfg = rvu_read64(rvu, blkaddr, 1269 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); 1270 *ena = bank_cfg & 0x1; 1271 *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; 1272 if (kw_type == NPC_MCAM_KEY_X2) { 1273 cam1 = rvu_read64(rvu, blkaddr, 1274 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1275 bank, 1276 1)); 1277 cam0 = rvu_read64(rvu, blkaddr, 1278 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1279 bank, 1280 0)); 1281 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1282 1283 cam1 = rvu_read64(rvu, blkaddr, 1284 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1285 bank, 1286 1)); 1287 cam0 = rvu_read64(rvu, blkaddr, 1288 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1289 bank, 1290 0)); 1291 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1292 1293 cam1 = rvu_read64(rvu, blkaddr, 1294 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1295 bank, 1296 1)); 1297 cam0 = rvu_read64(rvu, blkaddr, 1298 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1299 bank, 1300 0)); 1301 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1302 1303 cam1 = rvu_read64(rvu, blkaddr, 1304 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1305 bank, 1306 1)); 1307 cam0 = rvu_read64(rvu, blkaddr, 1308 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1309 bank, 1310 0)); 1311 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1312 return 0; 1313 } 1314 1315 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { 1316 cam1 = rvu_read64(rvu, blkaddr, 1317 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1318 bank, 1319 1)); 1320 cam0 = rvu_read64(rvu, blkaddr, 1321 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1322 bank, 1323 0)); 1324 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1325 1326 cam1 = rvu_read64(rvu, blkaddr, 1327 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1328 bank, 1329 1)); 1330 cam0 = rvu_read64(rvu, blkaddr, 1331 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1332 bank, 1333 0)); 1334 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1335 1336 cam1 = rvu_read64(rvu, blkaddr, 1337 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1338 bank, 1339 1)); 1340 cam0 = rvu_read64(rvu, blkaddr, 1341 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1342 bank, 1343 0)); 1344 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1345 1346 cam1 = rvu_read64(rvu, blkaddr, 1347 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1348 bank, 1349 1)); 1350 cam0 = rvu_read64(rvu, blkaddr, 1351 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1352 bank, 1353 0)); 1354 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1355 } 1356 1357 return 0; 1358 } 1359 1360 int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, 1361 struct npc_cn20k_mcam_write_entry_req *req, 1362 struct msg_rsp *rsp) 1363 { 1364 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1365 struct npc_mcam *mcam = &rvu->hw->mcam; 1366 u16 pcifunc = req->hdr.pcifunc; 1367 int blkaddr, rc; 1368 u8 nix_intf; 1369 1370 req->entry = npc_cn20k_vidx2idx(req->entry); 1371 1372 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1373 if (blkaddr < 0) 1374 return NPC_MCAM_INVALID_REQ; 1375 1376 mutex_lock(&mcam->lock); 1377 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1378 if (rc) 1379 goto exit; 1380 1381 if (!is_npc_interface_valid(rvu, req->intf)) { 1382 rc = NPC_MCAM_INVALID_REQ; 1383 goto exit; 1384 } 1385 1386 if (is_npc_intf_tx(req->intf)) 1387 nix_intf = pfvf->nix_tx_intf; 1388 else 1389 nix_intf = pfvf->nix_rx_intf; 1390 1391 /* For AF installed rules, the nix_intf should be set to target NIX */ 1392 if (is_pffunc_af(req->hdr.pcifunc)) 1393 nix_intf = req->intf; 1394 1395 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, 1396 &req->entry_data, req->enable_entry, 1397 req->hw_prio, req->req_kw_type); 1398 1399 exit: 1400 mutex_unlock(&mcam->lock); 1401 return rc; 1402 } 1403 1404 int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, 1405 struct npc_mcam_read_entry_req *req, 1406 struct npc_cn20k_mcam_read_entry_rsp *rsp) 1407 { 1408 struct npc_mcam *mcam = &rvu->hw->mcam; 1409 u16 pcifunc = req->hdr.pcifunc; 1410 int blkaddr, rc; 1411 1412 req->entry = npc_cn20k_vidx2idx(req->entry); 1413 1414 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1415 if (blkaddr < 0) 1416 return NPC_MCAM_INVALID_REQ; 1417 1418 mutex_lock(&mcam->lock); 1419 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1420 if (rc) 1421 goto fail; 1422 1423 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, 1424 &rsp->entry_data, &rsp->intf, 1425 &rsp->enable, &rsp->hw_prio); 1426 fail: 1427 mutex_unlock(&mcam->lock); 1428 return rc; 1429 } 1430 1431 int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, 1432 struct npc_cn20k_mcam_alloc_and_write_entry_req *req, 1433 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 1434 { 1435 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1436 struct npc_mcam_free_entry_req free_req = { 0 }; 1437 struct npc_mcam_alloc_entry_req entry_req; 1438 struct npc_mcam_alloc_entry_rsp entry_rsp; 1439 struct npc_mcam *mcam = &rvu->hw->mcam; 1440 u16 entry = NPC_MCAM_ENTRY_INVALID; 1441 struct msg_rsp free_rsp; 1442 int blkaddr, rc, err; 1443 u8 nix_intf; 1444 1445 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1446 if (blkaddr < 0) 1447 return NPC_MCAM_INVALID_REQ; 1448 1449 if (!is_npc_interface_valid(rvu, req->intf)) 1450 return NPC_MCAM_INVALID_REQ; 1451 1452 /* Try to allocate a MCAM entry */ 1453 entry_req.hdr.pcifunc = req->hdr.pcifunc; 1454 entry_req.contig = true; 1455 entry_req.ref_prio = req->ref_prio; 1456 entry_req.ref_entry = req->ref_entry; 1457 entry_req.count = 1; 1458 entry_req.virt = req->virt; 1459 1460 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 1461 &entry_req, &entry_rsp); 1462 if (rc) 1463 return rc; 1464 1465 if (!entry_rsp.count) 1466 return NPC_MCAM_ALLOC_FAILED; 1467 1468 /* entry_req.count is 1, so single entry is allocated */ 1469 entry = npc_cn20k_vidx2idx(entry_rsp.entry); 1470 1471 mutex_lock(&mcam->lock); 1472 1473 if (is_npc_intf_tx(req->intf)) 1474 nix_intf = pfvf->nix_tx_intf; 1475 else 1476 nix_intf = pfvf->nix_rx_intf; 1477 1478 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, 1479 &req->entry_data, req->enable_entry, 1480 req->hw_prio, req->req_kw_type); 1481 1482 mutex_unlock(&mcam->lock); 1483 1484 if (rc) { 1485 free_req.hdr.pcifunc = req->hdr.pcifunc; 1486 free_req.entry = entry_rsp.entry; 1487 err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 1488 if (err) 1489 dev_err(rvu->dev, 1490 "%s: Error to free mcam idx %u\n", 1491 __func__, entry_rsp.entry); 1492 return rc; 1493 } 1494 1495 rsp->entry = entry_rsp.entry; 1496 return 0; 1497 } 1498 1499 static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) 1500 { 1501 if (is_lbk_vf(rvu, pcifunc)) 1502 return NIXLF_PROMISC_ENTRY; 1503 1504 return NIXLF_UCAST_ENTRY; 1505 } 1506 1507 int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, 1508 struct msg_req *req, 1509 struct npc_cn20k_mcam_read_base_rule_rsp *rsp) 1510 { 1511 struct npc_mcam *mcam = &rvu->hw->mcam; 1512 int index, blkaddr, nixlf, rc = 0; 1513 u16 pcifunc = req->hdr.pcifunc; 1514 u8 intf, enable, hw_prio; 1515 struct rvu_pfvf *pfvf; 1516 int rl_type; 1517 1518 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1519 if (blkaddr < 0) 1520 return NPC_MCAM_INVALID_REQ; 1521 1522 /* Return the channel number in case of PF */ 1523 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 1524 pfvf = rvu_get_pfvf(rvu, pcifunc); 1525 rsp->entry.kw[0] = pfvf->rx_chan_base; 1526 rsp->entry.kw_mask[0] = 0xFFFULL; 1527 goto out; 1528 } 1529 1530 /* Find the pkt steering rule installed by PF to this VF */ 1531 mutex_lock(&mcam->lock); 1532 for (index = 0; index < mcam->bmap_entries; index++) { 1533 if (mcam->entry2target_pffunc[index] == pcifunc) 1534 goto read_entry; 1535 } 1536 1537 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 1538 if (rc < 0) { 1539 mutex_unlock(&mcam->lock); 1540 goto out; 1541 } 1542 1543 rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); 1544 1545 /* Read the default ucast entry if there is no pkt steering rule */ 1546 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); 1547 if (index < 0) { 1548 mutex_unlock(&mcam->lock); 1549 goto out; 1550 } 1551 1552 read_entry: 1553 /* Read the mcam entry */ 1554 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index, 1555 &rsp->entry, &intf, 1556 &enable, &hw_prio); 1557 mutex_unlock(&mcam->lock); 1558 out: 1559 return rc; 1560 } 1561 1562 static u8 npc_map2cn20k_flag(u8 flag) 1563 { 1564 switch (flag) { 1565 case NPC_F_LC_U_IP_FRAG: 1566 return NPC_CN20K_F_LC_L_IP_FRAG; 1567 1568 case NPC_F_LC_U_IP6_FRAG: 1569 return NPC_CN20K_F_LC_L_IP6_FRAG; 1570 1571 case NPC_F_LC_L_6TO4: 1572 return NPC_CN20K_F_LC_L_6TO4; 1573 1574 case NPC_F_LC_L_MPLS_IN_IP: 1575 return NPC_CN20K_F_LC_U_MPLS_IN_IP; 1576 1577 case NPC_F_LC_L_IP6_TUN_IP6: 1578 return NPC_CN20K_F_LC_U_IP6_TUN_IP6; 1579 1580 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1581 return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; 1582 1583 default: 1584 break; 1585 } 1586 1587 WARN(1, "%s: Invalid flag=%u\n", __func__, flag); 1588 return 0xff; 1589 } 1590 1591 static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act) 1592 { 1593 u8 ltype, val; 1594 1595 if (act->lid != NPC_LID_LC) 1596 return; 1597 1598 ltype = act->ltype; 1599 if (ltype != NPC_LT_LC_IP && 1600 ltype != NPC_LT_LC_IP6 && 1601 ltype != NPC_LT_LC_IP_OPT && 1602 ltype != NPC_LT_LC_IP6_EXT) 1603 return; 1604 1605 switch (act->flags) { 1606 case NPC_F_LC_U_IP_FRAG: 1607 case NPC_F_LC_U_IP6_FRAG: 1608 case NPC_F_LC_L_6TO4: 1609 case NPC_F_LC_L_MPLS_IN_IP: 1610 case NPC_F_LC_L_IP6_TUN_IP6: 1611 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1612 val = npc_map2cn20k_flag(act->flags); 1613 if (val != 0xFF) 1614 act->flags = val; 1615 break; 1616 default: 1617 break; 1618 } 1619 } 1620 1621 void 1622 npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, 1623 struct npc_kpu_profile_adapter *pfl) 1624 { 1625 struct npc_kpu_profile_action *action; 1626 int entries; 1627 1628 for (int i = 0; i < pfl->kpus; i++) { 1629 action = pfl->kpu[i].action; 1630 entries = pfl->kpu[i].action_entries; 1631 1632 for (int j = 0; j < entries; j++) 1633 npc_cn20k_translate_action_flags(&action[j]); 1634 } 1635 } 1636 1637 int npc_cn20k_apply_custom_kpu(struct rvu *rvu, 1638 struct npc_kpu_profile_adapter *profile) 1639 { 1640 const struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1641 size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1642 struct npc_kpu_profile_action *action; 1643 struct npc_kpu_profile_cam *cam; 1644 struct npc_kpu_fwdata *fw_kpu; 1645 size_t offset = 0; 1646 u16 kpu, entry; 1647 int entries; 1648 1649 hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1650 1651 if (rvu->kpu_fwdata_sz < hdr_sz) { 1652 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1653 return -EINVAL; 1654 } 1655 1656 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1657 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1658 fw->signature); 1659 return -EINVAL; 1660 } 1661 1662 /* Verify if the using known profile structure */ 1663 if (NPC_KPU_VER_MAJ(profile->version) > 1664 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1665 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1666 NPC_KPU_VER_MAJ(profile->version), 1667 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1668 return -EINVAL; 1669 } 1670 1671 /* Verify if profile is aligned with the required kernel changes */ 1672 if (NPC_KPU_VER_MIN(profile->version) < 1673 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1674 dev_warn(rvu->dev, 1675 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1676 NPC_KPU_VER_MAJ(profile->version), 1677 NPC_KPU_VER_MIN(profile->version), 1678 NPC_KPU_VER_PATCH(profile->version), 1679 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1680 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1681 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1682 return -EINVAL; 1683 } 1684 1685 /* Verify if profile fits the HW */ 1686 if (fw->kpus > rvu->hw->npc_kpus) { 1687 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus, 1688 rvu->hw->npc_kpus); 1689 return -EINVAL; 1690 } 1691 1692 /* Check if there is enough memory */ 1693 if (fw->kpus > profile->kpus) { 1694 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus, 1695 profile->kpus); 1696 return -EINVAL; 1697 } 1698 1699 profile->mcam_kex_prfl.mkex_extr = &fw->mkex; 1700 if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { 1701 dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", 1702 profile->mcam_kex_prfl.mkex_extr->mkex_sign); 1703 return -EINVAL; 1704 } 1705 1706 profile->custom = 1; 1707 profile->name = fw->name; 1708 profile->version = le64_to_cpu(fw->version); 1709 profile->lt_def = &fw->lt_def; 1710 1711 for (kpu = 0; kpu < fw->kpus; kpu++) { 1712 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1713 if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) 1714 dev_warn(rvu->dev, 1715 "Too many custom entries on KPU%d: %d > %d\n", 1716 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1717 entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1718 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1719 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1720 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1721 offset += fw_kpu->entries * sizeof(*action); 1722 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1723 dev_warn(rvu->dev, 1724 "Profile size mismatch on KPU%i parsing.\n", 1725 kpu + 1); 1726 return -EINVAL; 1727 } 1728 1729 for (entry = 0; entry < entries; entry++) { 1730 profile->kpu[kpu].cam[entry] = cam[entry]; 1731 profile->kpu[kpu].action[entry] = action[entry]; 1732 npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]); 1733 } 1734 } 1735 1736 return 0; 1737 } 1738 1739 int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) 1740 { 1741 struct npc_subbank *sb; 1742 int bank_off, sb_id; 1743 1744 /* mcam_idx should be less than (2 * bank depth) */ 1745 if (mcam_idx >= npc_priv->bank_depth * 2) { 1746 dev_err(rvu->dev, "%s: bad params\n", 1747 __func__); 1748 return -EINVAL; 1749 } 1750 1751 /* find mcam offset per bank */ 1752 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1753 1754 /* Find subbank id */ 1755 sb_id = bank_off / npc_priv->subbank_depth; 1756 1757 /* Check if subbank id is more than maximum 1758 * number of subbanks available 1759 */ 1760 if (sb_id >= npc_priv->num_subbanks) { 1761 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1762 __func__, sb_id); 1763 return -EINVAL; 1764 } 1765 1766 sb = &npc_priv->sb[sb_id]; 1767 1768 *key_type = sb->key_type; 1769 1770 return 0; 1771 } 1772 1773 static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, 1774 u16 sub_off, u16 *mcam_idx) 1775 { 1776 int off, bot; 1777 1778 /* for x4 section, maximum allowed subbank index = 1779 * subsection depth - 1 1780 */ 1781 if (sb->key_type == NPC_MCAM_KEY_X4 && 1782 sub_off >= npc_priv->subbank_depth) { 1783 dev_err(rvu->dev, 1784 "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", 1785 __func__, sb->idx, sub_off); 1786 return -EINVAL; 1787 } 1788 1789 /* for x2 section, maximum allowed subbank index = 1790 * 2 * subsection depth - 1 1791 */ 1792 if (sb->key_type == NPC_MCAM_KEY_X2 && 1793 sub_off >= npc_priv->subbank_depth * 2) { 1794 dev_err(rvu->dev, 1795 "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", 1796 __func__, sb->idx, sub_off); 1797 return -EINVAL; 1798 } 1799 1800 /* Find subbank offset from respective subbank (w.r.t bank) */ 1801 off = sub_off & (npc_priv->subbank_depth - 1); 1802 1803 /* if subsection idx is in bank1, add bank depth, 1804 * which is part of sb->b1b 1805 */ 1806 bot = sub_off >= npc_priv->subbank_depth ? sb->b1b : sb->b0b; 1807 1808 *mcam_idx = bot + off; 1809 return 0; 1810 } 1811 1812 int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, 1813 struct npc_subbank **sb, 1814 int *sb_off) 1815 { 1816 int bank_off, sb_id; 1817 1818 /* mcam_idx should be less than (2 * bank depth) */ 1819 if (mcam_idx >= npc_priv->bank_depth * 2) { 1820 dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", 1821 __func__, mcam_idx); 1822 return -EINVAL; 1823 } 1824 1825 /* find mcam offset per bank */ 1826 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1827 1828 /* Find subbank id */ 1829 sb_id = bank_off / npc_priv->subbank_depth; 1830 1831 /* Check if subbank id is more than maximum 1832 * number of subbanks available 1833 */ 1834 if (sb_id >= npc_priv->num_subbanks) { 1835 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1836 __func__, sb_id); 1837 return -EINVAL; 1838 } 1839 1840 *sb = &npc_priv->sb[sb_id]; 1841 1842 /* Subbank offset per bank */ 1843 *sb_off = bank_off % npc_priv->subbank_depth; 1844 1845 /* Index in a subbank should add subbank depth 1846 * if it is in bank1 1847 */ 1848 if (mcam_idx >= npc_priv->bank_depth) 1849 *sb_off += npc_priv->subbank_depth; 1850 1851 return 0; 1852 } 1853 1854 static int __npc_subbank_contig_alloc(struct rvu *rvu, 1855 struct npc_subbank *sb, 1856 int key_type, int sidx, 1857 int eidx, int prio, 1858 int count, int t, int b, 1859 unsigned long *bmap, 1860 u16 *save) 1861 { 1862 int k, offset, delta = 0; 1863 int cnt = 0, sbd; 1864 1865 sbd = npc_priv->subbank_depth; 1866 1867 if (sidx >= npc_priv->bank_depth) 1868 delta = sbd; 1869 1870 switch (prio) { 1871 case NPC_MCAM_LOWER_PRIO: 1872 case NPC_MCAM_ANY_PRIO: 1873 /* Find an area of size 'count' from sidx to eidx */ 1874 offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, 1875 count, 0); 1876 1877 if (offset >= sbd) { 1878 dev_err(rvu->dev, 1879 "%s: Could not find contiguous(%d) entries\n", 1880 __func__, count); 1881 return -EFAULT; 1882 } 1883 1884 dev_dbg(rvu->dev, 1885 "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", 1886 __func__, sidx, eidx, t, b, offset, 1887 count, delta); 1888 1889 for (cnt = 0; cnt < count; cnt++) 1890 save[cnt] = offset + cnt + delta; 1891 1892 break; 1893 1894 case NPC_MCAM_HIGHER_PRIO: 1895 /* Find an area of 'count' from eidx to sidx */ 1896 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1897 /* If an intermediate slot is not free, 1898 * reset the counter (cnt) to zero as 1899 * request is for contiguous. 1900 */ 1901 if (test_bit(k, bmap)) { 1902 cnt = 0; 1903 continue; 1904 } 1905 1906 save[cnt++] = k + delta; 1907 } 1908 break; 1909 } 1910 1911 /* Found 'count' number of free slots */ 1912 if (cnt == count) 1913 return 0; 1914 1915 dev_dbg(rvu->dev, 1916 "%s: Could not find contiguous(%d) entries in subbank=%u\n", 1917 __func__, count, sb->idx); 1918 return -EFAULT; 1919 } 1920 1921 static int __npc_subbank_non_contig_alloc(struct rvu *rvu, 1922 struct npc_subbank *sb, 1923 int key_type, int sidx, 1924 int eidx, int prio, 1925 int t, int b, 1926 unsigned long *bmap, 1927 int count, u16 *save, 1928 bool max_alloc, int *alloc_cnt) 1929 { 1930 unsigned long index; 1931 int cnt = 0, delta; 1932 int k, sbd; 1933 1934 sbd = npc_priv->subbank_depth; 1935 delta = sidx >= npc_priv->bank_depth ? sbd : 0; 1936 1937 switch (prio) { 1938 /* Find an area of size 'count' from sidx to eidx */ 1939 case NPC_MCAM_LOWER_PRIO: 1940 case NPC_MCAM_ANY_PRIO: 1941 index = find_next_zero_bit(bmap, sbd, sidx - b); 1942 if (index >= sbd) { 1943 dev_err(rvu->dev, 1944 "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", 1945 __func__, count, 1946 bitmap_weight(bmap, sbd), 1947 sb->idx); 1948 break; 1949 } 1950 1951 for (k = index; cnt < count && k <= (eidx - b); k++) { 1952 /* Skip used slots */ 1953 if (test_bit(k, bmap)) 1954 continue; 1955 1956 save[cnt++] = k + delta; 1957 } 1958 break; 1959 1960 /* Find an area of 'count' from eidx to sidx */ 1961 case NPC_MCAM_HIGHER_PRIO: 1962 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1963 /* Skip used slots */ 1964 if (test_bit(k, bmap)) 1965 continue; 1966 1967 save[cnt++] = k + delta; 1968 } 1969 break; 1970 } 1971 1972 /* Update allocated 'cnt' to alloc_cnt */ 1973 *alloc_cnt = cnt; 1974 1975 /* Successfully allocated requested count slots */ 1976 if (cnt == count) 1977 return 0; 1978 1979 /* Allocation successful for cnt < count */ 1980 if (max_alloc && cnt > 0) 1981 return 0; 1982 1983 dev_dbg(rvu->dev, 1984 "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", 1985 __func__, count, sb->idx, cnt, max_alloc); 1986 1987 return -EFAULT; 1988 } 1989 1990 static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, 1991 int sb_off, unsigned long **bmap, 1992 int *off) 1993 { 1994 int sbd; 1995 1996 sbd = npc_priv->subbank_depth; 1997 1998 *off = sb_off & (sbd - 1); 1999 *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; 2000 } 2001 2002 /* set/clear bitmap */ 2003 static bool __npc_subbank_mark_slot(struct rvu *rvu, 2004 struct npc_subbank *sb, 2005 int sb_off, bool set) 2006 { 2007 unsigned long *bmap; 2008 int off; 2009 2010 /* if sb_off >= subbank.depth, then slots are in 2011 * bank1 2012 */ 2013 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2014 2015 dev_dbg(rvu->dev, 2016 "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", 2017 __func__, set, sb_off, sb->idx, off); 2018 2019 if (set) { 2020 /* Slot is already used */ 2021 if (test_bit(off, bmap)) 2022 return false; 2023 2024 sb->free_cnt--; 2025 set_bit(off, bmap); 2026 return true; 2027 } 2028 2029 /* Slot is already free */ 2030 if (!test_bit(off, bmap)) 2031 return false; 2032 2033 sb->free_cnt++; 2034 clear_bit(off, bmap); 2035 return true; 2036 } 2037 2038 static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) 2039 { 2040 int rc, blkaddr; 2041 2042 sb->flags = NPC_SUBBANK_FLAG_FREE; 2043 sb->key_type = 0; 2044 2045 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2046 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2047 2048 if (!xa_erase(&npc_priv->xa_sb_used, sb->arr_idx)) { 2049 dev_err(rvu->dev, 2050 "%s: Error to delete from xa_sb_used array\n", 2051 __func__); 2052 return -EFAULT; 2053 } 2054 2055 rc = xa_insert(&npc_priv->xa_sb_free, sb->arr_idx, 2056 xa_mk_value(sb->idx), GFP_KERNEL); 2057 if (rc) { 2058 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2059 xa_mk_value(sb->idx), GFP_KERNEL); 2060 if (rc) 2061 dev_err(rvu->dev, 2062 "%s: Failed to roll back sb(%u) arr_idx=%d\n", 2063 __func__, sb->idx, sb->arr_idx); 2064 2065 dev_err(rvu->dev, 2066 "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", 2067 __func__, sb->idx, sb->arr_idx); 2068 return rc; 2069 } 2070 2071 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2072 rvu_write64(rvu, blkaddr, 2073 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2074 NPC_MCAM_KEY_X2); 2075 2076 return rc; 2077 } 2078 2079 static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, 2080 int key_type) 2081 { 2082 int rc; 2083 2084 sb->flags = NPC_SUBBANK_FLAG_USED; 2085 sb->key_type = key_type; 2086 if (key_type == NPC_MCAM_KEY_X4) 2087 sb->free_cnt = npc_priv->subbank_depth; 2088 else 2089 sb->free_cnt = 2 * npc_priv->subbank_depth; 2090 2091 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2092 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2093 2094 if (!xa_erase(&npc_priv->xa_sb_free, sb->arr_idx)) { 2095 dev_err(rvu->dev, 2096 "%s: Error to delete from xa_sb_free array\n", 2097 __func__); 2098 return -EFAULT; 2099 } 2100 2101 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2102 xa_mk_value(sb->idx), GFP_KERNEL); 2103 if (rc) 2104 dev_err(rvu->dev, 2105 "%s: Error to add to xa_sb_used array\n", __func__); 2106 2107 return rc; 2108 } 2109 2110 static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, 2111 u16 sb_off) 2112 { 2113 bool deleted = false; 2114 unsigned long *bmap; 2115 int rc, off; 2116 2117 deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); 2118 if (!deleted) 2119 goto done; 2120 2121 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2122 2123 /* Check whether we can mark whole subbank as free */ 2124 if (sb->key_type == NPC_MCAM_KEY_X4) { 2125 if (sb->free_cnt < npc_priv->subbank_depth) 2126 goto done; 2127 } else { 2128 if (sb->free_cnt < 2 * npc_priv->subbank_depth) 2129 goto done; 2130 } 2131 2132 /* All slots in subbank are unused. Mark the subbank as free 2133 * and add to free pool 2134 */ 2135 rc = __npc_subbank_mark_free(rvu, sb); 2136 if (rc) 2137 dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); 2138 2139 done: 2140 return deleted; 2141 } 2142 2143 static int 2144 npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) 2145 { 2146 bool deleted; 2147 2148 mutex_lock(&sb->lock); 2149 deleted = __npc_subbank_free(rvu, sb, sb_off); 2150 mutex_unlock(&sb->lock); 2151 2152 return deleted ? 0 : -EFAULT; 2153 } 2154 2155 static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2156 int key_type, int ref, int limit, int prio, 2157 bool contig, int count, u16 *mcam_idx, 2158 int idx_sz, bool max_alloc, int *alloc_cnt) 2159 { 2160 int cnt, t, b, i, blkaddr; 2161 bool new_sub_bank = false; 2162 unsigned long *bmap; 2163 u16 *save = NULL; 2164 int sidx, eidx; 2165 bool diffbank; 2166 int bw, bfree; 2167 int rc = 0; 2168 bool ret; 2169 2170 /* Check if enough space is there to return requested number of 2171 * mcam indexes in case of contiguous allocation 2172 */ 2173 if (!max_alloc && count > idx_sz) { 2174 dev_err(rvu->dev, 2175 "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", 2176 __func__, count, idx_sz, sb->idx); 2177 return -ENOSPC; 2178 } 2179 2180 /* Allocation on multiple subbank is not supported by this function. 2181 * it means that ref and limit should be on same subbank. 2182 * 2183 * ref and limit values should be validated w.r.t prio as below. 2184 * say ref = 100, limit = 200, 2185 * if NPC_MCAM_LOWER_PRIO, allocate index 100 2186 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. 2187 * if NPC_MCAM_ANY_PRIO, allocate index 100 2188 * 2189 * say ref = 200, limit = 100 2190 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. 2191 * if NPC_MCAM_HIGHER_PRIO, allocate index 200 2192 * if NPC_MCAM_ANY_PRIO, allocate index 100 2193 * 2194 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction 2195 * on "ref" and "limit" values. ie, ref > limit and limit > ref 2196 * are valid cases. 2197 */ 2198 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 2199 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 2200 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 2201 __func__, ref, limit); 2202 return -EINVAL; 2203 } 2204 2205 /* x4 indexes are from 0 to bank size as it combines two x2 banks */ 2206 if (key_type == NPC_MCAM_KEY_X4 && 2207 (ref >= npc_priv->bank_depth || limit >= npc_priv->bank_depth)) { 2208 dev_err(rvu->dev, 2209 "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", 2210 __func__, ref, limit); 2211 return -EINVAL; 2212 } 2213 2214 /* This function is called either bank0 or bank1 portion of a subbank. 2215 * so ref and limit should be on same bank. 2216 */ 2217 diffbank = !!((ref & npc_priv->bank_depth) ^ 2218 (limit & npc_priv->bank_depth)); 2219 if (diffbank) { 2220 dev_err(rvu->dev, 2221 "%s: request ref and limit should be from same bank\n", 2222 __func__); 2223 return -EINVAL; 2224 } 2225 2226 sidx = min_t(int, limit, ref); 2227 eidx = max_t(int, limit, ref); 2228 2229 /* Find total number of slots available; both used and free */ 2230 cnt = eidx - sidx + 1; 2231 if (contig && cnt < count) { 2232 dev_err(rvu->dev, 2233 "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", 2234 __func__, ref, limit, count); 2235 return -EINVAL; 2236 } 2237 2238 /* If subbank is free, check if requested number of indexes is less than 2239 * or equal to mcam entries available in the subbank if contig. 2240 */ 2241 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2242 if (contig && count > npc_priv->subbank_depth) { 2243 dev_err(rvu->dev, "%s: Less number of entries\n", 2244 __func__); 2245 return -ENOSPC; 2246 } 2247 2248 new_sub_bank = true; 2249 goto process; 2250 } 2251 2252 /* Flag should be set for all used subbanks */ 2253 WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), 2254 "Used flag is not set(%#x)\n", sb->flags); 2255 2256 /* If subbank key type does not match with requested key_type, 2257 * return error 2258 */ 2259 if (sb->key_type != key_type) { 2260 dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); 2261 return -EINVAL; 2262 } 2263 2264 process: 2265 /* if ref or limit >= npc_priv->bank_depth, index are in bank1. 2266 * else bank0. 2267 */ 2268 if (ref >= npc_priv->bank_depth) { 2269 bmap = sb->b1map; 2270 t = sb->b1t; 2271 b = sb->b1b; 2272 } else { 2273 bmap = sb->b0map; 2274 t = sb->b0t; 2275 b = sb->b0b; 2276 } 2277 2278 /* Calculate free slots */ 2279 bw = bitmap_weight(bmap, npc_priv->subbank_depth); 2280 bfree = npc_priv->subbank_depth - bw; 2281 2282 if (!bfree) { 2283 dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); 2284 return -ENOSPC; 2285 } 2286 2287 /* If request is for contiguous , then max we can allocate is 2288 * equal to subbank_depth 2289 */ 2290 if (contig && bfree < count) { 2291 dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); 2292 return -ENOSPC; 2293 } 2294 2295 /* 'save' array stores available indexes temporarily before 2296 * marking it as allocated 2297 */ 2298 save = kcalloc(count, sizeof(u16), GFP_KERNEL); 2299 if (!save) { 2300 rc = -ENOMEM; 2301 goto err1; 2302 } 2303 2304 if (contig) { 2305 rc = __npc_subbank_contig_alloc(rvu, sb, key_type, 2306 sidx, eidx, prio, 2307 count, t, b, 2308 bmap, save); 2309 /* contiguous allocation success means that 2310 * requested number of free slots got 2311 * allocated 2312 */ 2313 if (!rc) 2314 *alloc_cnt = count; 2315 2316 } else { 2317 rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, 2318 sidx, eidx, prio, 2319 t, b, bmap, 2320 count, save, 2321 max_alloc, alloc_cnt); 2322 } 2323 2324 if (rc) 2325 goto err1; 2326 2327 /* Mark new subbank bank as used */ 2328 if (new_sub_bank) { 2329 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2330 if (blkaddr < 0) { 2331 dev_err(rvu->dev, 2332 "%s: NPC block not implemented\n", __func__); 2333 rc = -EFAULT; 2334 goto err1; 2335 } 2336 2337 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2338 if (rc) { 2339 dev_err(rvu->dev, 2340 "%s: Error to mark subbank as used\n", 2341 __func__); 2342 goto err2; 2343 } 2344 2345 /* Configure section type to key_type */ 2346 rvu_write64(rvu, blkaddr, 2347 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2348 key_type); 2349 } 2350 2351 for (i = 0; i < *alloc_cnt; i++) { 2352 rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], 2353 &mcam_idx[i]); 2354 if (rc) { 2355 dev_err(rvu->dev, 2356 "%s: Error to find mcam idx for %u\n", 2357 __func__, save[i]); 2358 /* TODO: handle err case gracefully */ 2359 goto err3; 2360 } 2361 2362 /* Mark all slots as used */ 2363 ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); 2364 if (!ret) { 2365 dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", 2366 __func__, mcam_idx[i]); 2367 rc = -EFAULT; 2368 goto err3; 2369 } 2370 } 2371 kfree(save); 2372 return 0; 2373 2374 err3: 2375 for (int j = 0; j < i; j++) 2376 __npc_subbank_mark_slot(rvu, sb, save[j], false); 2377 err2: 2378 if (new_sub_bank) 2379 __npc_subbank_mark_free(rvu, sb); 2380 err1: 2381 kfree(save); 2382 *alloc_cnt = 0; 2383 return rc; 2384 } 2385 2386 static int 2387 npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2388 int key_type, int ref, int limit, int prio, 2389 bool contig, int count, u16 *mcam_idx, 2390 int idx_sz, bool max_alloc, int *alloc_cnt) 2391 { 2392 int rc; 2393 2394 mutex_lock(&sb->lock); 2395 rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, 2396 contig, count, mcam_idx, idx_sz, 2397 max_alloc, alloc_cnt); 2398 mutex_unlock(&sb->lock); 2399 2400 return rc; 2401 } 2402 2403 static int 2404 npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) 2405 { 2406 int pcifunc, idx; 2407 void *map; 2408 2409 map = xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2410 if (!map) { 2411 dev_err(rvu->dev, 2412 "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", 2413 __func__, mcam_idx); 2414 return -EFAULT; 2415 } 2416 2417 pcifunc = xa_to_value(map); 2418 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2419 if (!map) { 2420 dev_err(rvu->dev, 2421 "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", 2422 __func__, pcifunc, mcam_idx); 2423 return -ESRCH; 2424 } 2425 2426 idx = xa_to_value(map); 2427 2428 map = xa_erase(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2429 if (!map) { 2430 dev_err(rvu->dev, 2431 "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", 2432 __func__, mcam_idx); 2433 return -EFAULT; 2434 } 2435 return 0; 2436 } 2437 2438 static int 2439 npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) 2440 { 2441 int rc, idx; 2442 void *map; 2443 2444 dev_dbg(rvu->dev, 2445 "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", 2446 __func__, mcam_idx, pcifunc); 2447 2448 rc = xa_insert(&npc_priv->xa_idx2pf_map, mcam_idx, 2449 xa_mk_value(pcifunc), GFP_KERNEL); 2450 2451 if (rc) { 2452 map = xa_load(&npc_priv->xa_idx2pf_map, mcam_idx); 2453 dev_err(rvu->dev, 2454 "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", 2455 __func__, mcam_idx, xa_to_value(map)); 2456 return -EFAULT; 2457 } 2458 2459 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2460 if (!map) { 2461 dev_err(rvu->dev, 2462 "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", 2463 __func__, pcifunc, mcam_idx); 2464 return -ESRCH; 2465 } 2466 2467 idx = xa_to_value(map); 2468 2469 rc = xa_insert(&npc_priv->xa_pf2idx_map[idx], mcam_idx, 2470 xa_mk_value(pcifunc), GFP_KERNEL); 2471 2472 if (rc) { 2473 map = xa_load(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2474 xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2475 dev_err(rvu->dev, 2476 "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", 2477 __func__, mcam_idx, xa_to_value(map), idx); 2478 2479 return -EFAULT; 2480 } 2481 2482 return 0; 2483 } 2484 2485 static bool 2486 npc_subbank_suits(struct npc_subbank *sb, int key_type) 2487 { 2488 mutex_lock(&sb->lock); 2489 2490 if (!sb->key_type) { 2491 mutex_unlock(&sb->lock); 2492 return true; 2493 } 2494 2495 if (sb->key_type == key_type) { 2496 mutex_unlock(&sb->lock); 2497 return true; 2498 } 2499 2500 mutex_unlock(&sb->lock); 2501 return false; 2502 } 2503 2504 #define SB_ALIGN_UP(val) (((val) + npc_priv->subbank_depth) & \ 2505 ~((npc_priv->subbank_depth) - 1)) 2506 #define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv->subbank_depth) 2507 2508 static void npc_subbank_iter_down(struct rvu *rvu, 2509 int ref, int limit, 2510 int *cur_ref, int *cur_limit, 2511 bool *start, bool *stop) 2512 { 2513 int align; 2514 2515 *stop = false; 2516 2517 /* ALIGN_DOWN the limit to current subbank boundary bottom index */ 2518 if (*start) { 2519 *start = false; 2520 *cur_ref = ref; 2521 align = SB_ALIGN_DOWN(ref); 2522 if (align < limit) { 2523 *stop = true; 2524 *cur_limit = limit; 2525 return; 2526 } 2527 *cur_limit = align; 2528 return; 2529 } 2530 2531 *cur_ref = *cur_limit - 1; 2532 align = *cur_ref - npc_priv->subbank_depth + 1; 2533 if (align <= limit) { 2534 *stop = true; 2535 *cur_limit = limit; 2536 return; 2537 } 2538 2539 *cur_limit = align; 2540 } 2541 2542 static void npc_subbank_iter_up(struct rvu *rvu, 2543 int ref, int limit, 2544 int *cur_ref, int *cur_limit, 2545 bool *start, bool *stop) 2546 { 2547 int align; 2548 2549 *stop = false; 2550 2551 /* ALIGN_UP the limit to current subbank boundary top index */ 2552 if (*start) { 2553 *start = false; 2554 *cur_ref = ref; 2555 2556 /* Find next lower prio subbank's bottom index */ 2557 align = SB_ALIGN_UP(ref); 2558 2559 /* Crosses limit ? */ 2560 if (align - 1 > limit) { 2561 *stop = true; 2562 *cur_limit = limit; 2563 return; 2564 } 2565 2566 /* Current subbank's top index */ 2567 *cur_limit = align - 1; 2568 return; 2569 } 2570 2571 *cur_ref = *cur_limit + 1; 2572 align = *cur_ref + npc_priv->subbank_depth - 1; 2573 2574 if (align >= limit) { 2575 *stop = true; 2576 *cur_limit = limit; 2577 return; 2578 } 2579 2580 *cur_limit = align; 2581 } 2582 2583 static int 2584 npc_subbank_iter(struct rvu *rvu, int key_type, 2585 int ref, int limit, int prio, 2586 int *cur_ref, int *cur_limit, 2587 bool *start, bool *stop) 2588 { 2589 if (prio != NPC_MCAM_HIGHER_PRIO) 2590 npc_subbank_iter_up(rvu, ref, limit, 2591 cur_ref, cur_limit, 2592 start, stop); 2593 else 2594 npc_subbank_iter_down(rvu, ref, limit, 2595 cur_ref, cur_limit, 2596 start, stop); 2597 2598 /* limit and ref should < bank_depth for x4 */ 2599 if (key_type == NPC_MCAM_KEY_X4) { 2600 if (*cur_ref >= npc_priv->bank_depth) 2601 return -EINVAL; 2602 2603 if (*cur_limit >= npc_priv->bank_depth) 2604 return -EINVAL; 2605 } 2606 /* limit and ref should < 2 * bank_depth, for x2 */ 2607 if (*cur_ref >= 2 * npc_priv->bank_depth) 2608 return -EINVAL; 2609 2610 if (*cur_limit >= 2 * npc_priv->bank_depth) 2611 return -EINVAL; 2612 2613 return 0; 2614 } 2615 2616 static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, 2617 bool maps_del) 2618 { 2619 struct npc_subbank *sb; 2620 u16 vidx, midx; 2621 int sb_off, i; 2622 bool ret; 2623 int rc; 2624 2625 /* Check if we can dealloc indexes properly ? */ 2626 for (i = 0; i < count; i++) { 2627 rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), 2628 &sb, &sb_off); 2629 if (rc) { 2630 dev_err(rvu->dev, 2631 "Failed to free mcam idx=%u\n", mcam_idx[i]); 2632 return rc; 2633 } 2634 } 2635 2636 for (i = 0; i < count; i++) { 2637 if (npc_is_vidx(mcam_idx[i])) { 2638 vidx = mcam_idx[i]; 2639 midx = npc_vidx2idx(vidx); 2640 } else { 2641 midx = mcam_idx[i]; 2642 vidx = npc_idx2vidx(midx); 2643 } 2644 2645 if (midx >= npc_priv->bank_depth * npc_priv->num_banks) { 2646 dev_err(rvu->dev, 2647 "%s: Invalid mcam_idx=%u cannot be deleted\n", 2648 __func__, mcam_idx[i]); 2649 return -EINVAL; 2650 } 2651 2652 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 2653 &sb, &sb_off); 2654 if (rc) { 2655 dev_err(rvu->dev, 2656 "%s: Failed to find subbank info for vidx=%u\n", 2657 __func__, vidx); 2658 return rc; 2659 } 2660 2661 ret = npc_subbank_free(rvu, sb, sb_off); 2662 if (ret) { 2663 dev_err(rvu->dev, 2664 "%s: Failed to find subbank info for vidx=%u\n", 2665 __func__, vidx); 2666 return -EINVAL; 2667 } 2668 2669 if (!maps_del) 2670 continue; 2671 2672 rc = npc_del_from_pf_maps(rvu, midx); 2673 if (rc) 2674 return rc; 2675 2676 /* If there is no vidx mapping; continue */ 2677 if (vidx == midx) 2678 continue; 2679 2680 rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); 2681 if (rc) 2682 return rc; 2683 } 2684 2685 return 0; 2686 } 2687 2688 static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, 2689 int ref, int limit, int prio, 2690 bool contig, int count, 2691 u16 *mcam_idx) 2692 { 2693 struct npc_subbank *sb; 2694 unsigned long *bmap; 2695 int sb_off, off, rc; 2696 int cnt = 0; 2697 bool bitset; 2698 2699 if (prio != NPC_MCAM_HIGHER_PRIO) { 2700 while (ref <= limit) { 2701 /* Calculate subbank and subbank index */ 2702 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2703 &sb, &sb_off); 2704 if (rc) 2705 goto err; 2706 2707 /* If subbank is not suitable for requested key type 2708 * restart search from next subbank 2709 */ 2710 if (!npc_subbank_suits(sb, key_type)) { 2711 ref = SB_ALIGN_UP(ref); 2712 if (contig) { 2713 rc = npc_idx_free(rvu, mcam_idx, 2714 cnt, false); 2715 if (rc) 2716 return rc; 2717 cnt = 0; 2718 } 2719 continue; 2720 } 2721 2722 mutex_lock(&sb->lock); 2723 2724 /* If subbank is free; mark it as used */ 2725 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2726 rc = __npc_subbank_mark_used(rvu, sb, 2727 key_type); 2728 if (rc) { 2729 mutex_unlock(&sb->lock); 2730 dev_err(rvu->dev, 2731 "%s:Error to add to use array\n", 2732 __func__); 2733 goto err; 2734 } 2735 } 2736 2737 /* Find correct bmap */ 2738 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2739 2740 /* if bit is already set, reset 'cnt' */ 2741 bitset = test_bit(off, bmap); 2742 if (bitset) { 2743 mutex_unlock(&sb->lock); 2744 if (contig) { 2745 rc = npc_idx_free(rvu, mcam_idx, 2746 cnt, false); 2747 if (rc) 2748 return rc; 2749 cnt = 0; 2750 } 2751 2752 ref++; 2753 continue; 2754 } 2755 2756 set_bit(off, bmap); 2757 sb->free_cnt--; 2758 mcam_idx[cnt++] = ref; 2759 mutex_unlock(&sb->lock); 2760 2761 if (cnt == count) 2762 return 0; 2763 ref++; 2764 } 2765 2766 /* Could not allocate request count slots */ 2767 goto err; 2768 } 2769 while (ref >= limit) { 2770 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2771 &sb, &sb_off); 2772 if (rc) 2773 goto err; 2774 2775 if (!npc_subbank_suits(sb, key_type)) { 2776 ref = SB_ALIGN_DOWN(ref) - 1; 2777 if (contig) { 2778 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2779 if (rc) 2780 return rc; 2781 2782 cnt = 0; 2783 } 2784 continue; 2785 } 2786 2787 mutex_lock(&sb->lock); 2788 2789 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2790 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2791 if (rc) { 2792 mutex_unlock(&sb->lock); 2793 dev_err(rvu->dev, 2794 "%s:Error to add to use array\n", 2795 __func__); 2796 goto err; 2797 } 2798 } 2799 2800 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2801 bitset = test_bit(off, bmap); 2802 if (bitset) { 2803 mutex_unlock(&sb->lock); 2804 if (contig) { 2805 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2806 if (rc) 2807 return rc; 2808 2809 cnt = 0; 2810 } 2811 ref--; 2812 continue; 2813 } 2814 2815 mcam_idx[cnt++] = ref; 2816 sb->free_cnt--; 2817 set_bit(off, bmap); 2818 mutex_unlock(&sb->lock); 2819 2820 if (cnt == count) 2821 return 0; 2822 ref--; 2823 } 2824 2825 err: 2826 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2827 if (rc) 2828 dev_err(rvu->dev, 2829 "%s: Error happened while freeing cnt=%u indexes\n", 2830 __func__, cnt); 2831 2832 return -ENOSPC; 2833 } 2834 2835 static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, 2836 int key_type) 2837 { 2838 int cnt, spd; 2839 2840 spd = npc_priv->subbank_depth; 2841 mutex_lock(&sb->lock); 2842 2843 if (sb->flags & NPC_SUBBANK_FLAG_FREE) 2844 cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; 2845 else 2846 cnt = sb->free_cnt; 2847 2848 mutex_unlock(&sb->lock); 2849 return cnt; 2850 } 2851 2852 static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, 2853 int ref, int limit, int prio, 2854 bool contig, int count, 2855 u16 *mcam_idx) 2856 { 2857 struct npc_subbank *sb1, *sb2; 2858 bool max_alloc, start, stop; 2859 int r, l, sb_idx1, sb_idx2; 2860 int tot = 0, rc; 2861 int alloc_cnt; 2862 2863 max_alloc = !contig; 2864 2865 start = true; 2866 stop = false; 2867 2868 /* Loop until we cross the ref/limit boundary */ 2869 while (!stop) { 2870 rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, 2871 &r, &l, &start, &stop); 2872 2873 dev_dbg(rvu->dev, 2874 "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", 2875 __func__, ref, limit, r, l, 2876 start, stop, tot, count, rc); 2877 2878 if (rc) 2879 goto err; 2880 2881 /* Find subbank and subbank index for ref */ 2882 rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, 2883 &sb_idx1); 2884 if (rc) 2885 goto err; 2886 2887 dev_dbg(rvu->dev, 2888 "%s: ref subbank=%d off=%d\n", 2889 __func__, sb1->idx, sb_idx1); 2890 2891 /* Skip subbank if it is not available for the keytype */ 2892 if (!npc_subbank_suits(sb1, key_type)) { 2893 dev_dbg(rvu->dev, 2894 "%s: not suitable sb=%d key_type=%d\n", 2895 __func__, sb1->idx, key_type); 2896 continue; 2897 } 2898 2899 /* Find subbank and subbank index for limit */ 2900 rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, 2901 &sb_idx2); 2902 if (rc) 2903 goto err; 2904 2905 dev_dbg(rvu->dev, 2906 "%s: limit subbank=%d off=%d\n", 2907 __func__, sb_idx1, sb_idx2); 2908 2909 /* subbank of ref and limit should be same */ 2910 if (sb1 != sb2) { 2911 dev_err(rvu->dev, 2912 "%s: l(%d) and r(%d) are not in same subbank\n", 2913 __func__, r, l); 2914 goto err; 2915 } 2916 2917 if (contig && 2918 npc_subbank_free_cnt(rvu, sb1, key_type) < count) { 2919 dev_dbg(rvu->dev, "%s: less count =%d\n", 2920 __func__, 2921 npc_subbank_free_cnt(rvu, sb1, key_type)); 2922 continue; 2923 } 2924 2925 /* Try in one bank of a subbank */ 2926 alloc_cnt = 0; 2927 rc = npc_subbank_alloc(rvu, sb1, key_type, 2928 r, l, prio, contig, 2929 count - tot, mcam_idx + tot, 2930 count - tot, max_alloc, 2931 &alloc_cnt); 2932 2933 tot += alloc_cnt; 2934 2935 dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", 2936 __func__, tot, alloc_cnt); 2937 2938 if (!rc && count == tot) 2939 return 0; 2940 } 2941 err: 2942 dev_dbg(rvu->dev, "%s: Error to allocate\n", 2943 __func__); 2944 2945 /* non contiguous allocation fails. We need to do clean up */ 2946 if (max_alloc) { 2947 rc = npc_idx_free(rvu, mcam_idx, tot, false); 2948 if (rc) 2949 dev_err(rvu->dev, 2950 "%s: failed to free %u indexes\n", 2951 __func__, tot); 2952 } 2953 2954 return -EFAULT; 2955 } 2956 2957 /* Minimize allocation from bottom and top subbanks for noref allocations. 2958 * Default allocations are ref based, and will be allocated from top 2959 * subbanks (least priority subbanks). Since default allocation is at very 2960 * early stage of kernel netdev probes, this subbanks will be moved to 2961 * used subbanks list. This will pave a way for noref allocation from these 2962 * used subbanks. Skip allocation for these top and bottom, and try free 2963 * bank next. If none slot is available, come back and search in these 2964 * subbanks. 2965 */ 2966 2967 static int npc_subbank_restricted_idxs[2]; 2968 static bool restrict_valid = true; 2969 2970 static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) 2971 { 2972 int i; 2973 2974 if (!restrict_valid) 2975 return false; 2976 2977 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { 2978 if (index == npc_subbank_restricted_idxs[i]) 2979 return true; 2980 } 2981 2982 return false; 2983 } 2984 2985 static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, 2986 int count, u16 *mcam_idx) 2987 { 2988 struct npc_subbank *sb; 2989 unsigned long index; 2990 int tot = 0, rc; 2991 bool max_alloc; 2992 int alloc_cnt; 2993 int idx, i; 2994 void *val; 2995 2996 max_alloc = !contig; 2997 2998 /* Check used subbanks for free slots */ 2999 xa_for_each(&npc_priv->xa_sb_used, index, val) { 3000 idx = xa_to_value(val); 3001 3002 /* Minimize allocation from restricted subbanks 3003 * in noref allocations. 3004 */ 3005 if (npc_subbank_restrict_usage(rvu, idx)) 3006 continue; 3007 3008 sb = &npc_priv->sb[idx]; 3009 3010 /* Skip if not suitable subbank */ 3011 if (!npc_subbank_suits(sb, key_type)) 3012 continue; 3013 3014 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3015 continue; 3016 3017 /* try in bank 0. Try passing ref and limit equal to 3018 * subbank boundaries 3019 */ 3020 alloc_cnt = 0; 3021 rc = npc_subbank_alloc(rvu, sb, key_type, 3022 sb->b0b, sb->b0t, 0, 3023 contig, count - tot, 3024 mcam_idx + tot, 3025 count - tot, 3026 max_alloc, &alloc_cnt); 3027 3028 /* Non contiguous allocation may allocate less than 3029 * requested 'count'. 3030 */ 3031 tot += alloc_cnt; 3032 3033 dev_dbg(rvu->dev, 3034 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3035 __func__, alloc_cnt, sb->idx, tot, count); 3036 3037 /* Successfully allocated */ 3038 if (!rc && count == tot) 3039 return 0; 3040 3041 /* x4 entries can be allocated from bank 0 only */ 3042 if (key_type == NPC_MCAM_KEY_X4) 3043 continue; 3044 3045 /* try in bank 1 for x2 */ 3046 alloc_cnt = 0; 3047 rc = npc_subbank_alloc(rvu, sb, key_type, 3048 sb->b1b, sb->b1t, 0, 3049 contig, count - tot, 3050 mcam_idx + tot, 3051 count - tot, max_alloc, 3052 &alloc_cnt); 3053 3054 tot += alloc_cnt; 3055 3056 dev_dbg(rvu->dev, 3057 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3058 __func__, alloc_cnt, sb->idx, tot, count); 3059 3060 if (!rc && count == tot) 3061 return 0; 3062 } 3063 3064 /* Allocate in free subbanks */ 3065 xa_for_each(&npc_priv->xa_sb_free, index, val) { 3066 idx = xa_to_value(val); 3067 sb = &npc_priv->sb[idx]; 3068 3069 /* Minimize allocation from restricted subbanks 3070 * in noref allocations. 3071 */ 3072 if (npc_subbank_restrict_usage(rvu, idx)) 3073 continue; 3074 3075 if (!npc_subbank_suits(sb, key_type)) 3076 continue; 3077 3078 /* try in bank 0 */ 3079 alloc_cnt = 0; 3080 rc = npc_subbank_alloc(rvu, sb, key_type, 3081 sb->b0b, sb->b0t, 0, 3082 contig, count - tot, 3083 mcam_idx + tot, 3084 count - tot, 3085 max_alloc, &alloc_cnt); 3086 3087 tot += alloc_cnt; 3088 3089 dev_dbg(rvu->dev, 3090 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3091 __func__, alloc_cnt, sb->idx, tot, count); 3092 3093 /* Successfully allocated */ 3094 if (!rc && count == tot) 3095 return 0; 3096 3097 /* x4 entries can be allocated from bank 0 only */ 3098 if (key_type == NPC_MCAM_KEY_X4) 3099 continue; 3100 3101 /* try in bank 1 for x2 */ 3102 alloc_cnt = 0; 3103 rc = npc_subbank_alloc(rvu, sb, 3104 key_type, sb->b1b, sb->b1t, 0, 3105 contig, count - tot, 3106 mcam_idx + tot, count - tot, 3107 max_alloc, &alloc_cnt); 3108 3109 tot += alloc_cnt; 3110 3111 dev_dbg(rvu->dev, 3112 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3113 __func__, alloc_cnt, sb->idx, tot, count); 3114 3115 if (!rc && count == tot) 3116 return 0; 3117 } 3118 3119 /* Allocate from restricted subbanks */ 3120 for (i = 0; restrict_valid && 3121 (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { 3122 idx = npc_subbank_restricted_idxs[i]; 3123 sb = &npc_priv->sb[idx]; 3124 3125 /* Skip if not suitable subbank */ 3126 if (!npc_subbank_suits(sb, key_type)) 3127 continue; 3128 3129 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3130 continue; 3131 3132 /* try in bank 0. Try passing ref and limit equal to 3133 * subbank boundaries 3134 */ 3135 alloc_cnt = 0; 3136 rc = npc_subbank_alloc(rvu, sb, key_type, 3137 sb->b0b, sb->b0t, 0, 3138 contig, count - tot, 3139 mcam_idx + tot, 3140 count - tot, 3141 max_alloc, &alloc_cnt); 3142 3143 /* Non contiguous allocation may allocate less than 3144 * requested 'count'. 3145 */ 3146 tot += alloc_cnt; 3147 3148 dev_dbg(rvu->dev, 3149 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3150 __func__, alloc_cnt, sb->idx, tot, count); 3151 3152 /* Successfully allocated */ 3153 if (!rc && count == tot) 3154 return 0; 3155 3156 /* x4 entries can be allocated from bank 0 only */ 3157 if (key_type == NPC_MCAM_KEY_X4) 3158 continue; 3159 3160 /* try in bank 1 for x2 */ 3161 alloc_cnt = 0; 3162 rc = npc_subbank_alloc(rvu, sb, key_type, 3163 sb->b1b, sb->b1t, 0, 3164 contig, count - tot, 3165 mcam_idx + tot, 3166 count - tot, max_alloc, 3167 &alloc_cnt); 3168 3169 tot += alloc_cnt; 3170 3171 dev_dbg(rvu->dev, 3172 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3173 __func__, alloc_cnt, sb->idx, tot, count); 3174 3175 if (!rc && count == tot) 3176 return 0; 3177 } 3178 3179 /* non contiguous allocation fails. We need to do clean up */ 3180 if (max_alloc) 3181 npc_idx_free(rvu, mcam_idx, tot, false); 3182 3183 dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", 3184 __func__); 3185 3186 return -EFAULT; 3187 } 3188 3189 int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) 3190 { 3191 return npc_idx_free(rvu, mcam_idx, count, true); 3192 } 3193 3194 int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, 3195 int prio, u16 *mcam_idx, int ref, int limit, 3196 bool contig, int count, bool virt) 3197 { 3198 bool defrag_candidate = false; 3199 int i, eidx, rc, bd; 3200 bool ref_valid; 3201 u16 vidx; 3202 3203 bd = npc_priv->bank_depth; 3204 3205 /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 3206 * Here user wants to allocate 0th entry 3207 */ 3208 if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && 3209 count == 1) { 3210 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3211 prio, contig, count, mcam_idx); 3212 3213 if (rc) 3214 return rc; 3215 goto add2map; 3216 } 3217 3218 ref_valid = !!(limit || ref); 3219 defrag_candidate = !ref_valid && !contig && virt; 3220 if (!ref_valid) { 3221 if (contig && count > npc_priv->subbank_depth) 3222 goto try_noref_multi_subbank; 3223 3224 rc = npc_subbank_noref_alloc(rvu, key_type, contig, 3225 count, mcam_idx); 3226 if (!rc) 3227 goto add2map; 3228 3229 try_noref_multi_subbank: 3230 eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; 3231 3232 if (prio == NPC_MCAM_HIGHER_PRIO) 3233 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3234 eidx, 0, 3235 NPC_MCAM_HIGHER_PRIO, 3236 contig, count, 3237 mcam_idx); 3238 else 3239 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3240 0, eidx, 3241 NPC_MCAM_LOWER_PRIO, 3242 contig, count, 3243 mcam_idx); 3244 3245 if (!rc) 3246 goto add2map; 3247 3248 return rc; 3249 } 3250 3251 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 3252 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 3253 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3254 __func__, ref, limit); 3255 return -EINVAL; 3256 } 3257 3258 if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || 3259 (key_type == NPC_MCAM_KEY_X2 && 3260 (ref >= 2 * bd || limit >= 2 * bd))) { 3261 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3262 __func__, ref, limit); 3263 return -EINVAL; 3264 } 3265 3266 if (contig && count > npc_priv->subbank_depth) 3267 goto try_ref_multi_subbank; 3268 3269 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3270 prio, contig, count, mcam_idx); 3271 if (!rc) 3272 goto add2map; 3273 3274 try_ref_multi_subbank: 3275 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3276 ref, limit, prio, 3277 contig, count, mcam_idx); 3278 if (!rc) 3279 goto add2map; 3280 3281 return rc; 3282 3283 add2map: 3284 for (i = 0; i < count; i++) { 3285 rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); 3286 if (rc) 3287 goto err; 3288 3289 if (!defrag_candidate) 3290 continue; 3291 3292 rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); 3293 if (rc) { 3294 npc_del_from_pf_maps(rvu, mcam_idx[i]); 3295 goto err; 3296 } 3297 3298 /* Return vidx to caller */ 3299 mcam_idx[i] = vidx; 3300 } 3301 3302 return 0; 3303 err: 3304 for (int j = 0; j < i; j++) { 3305 npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); 3306 3307 if (!defrag_candidate) 3308 continue; 3309 3310 npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); 3311 } 3312 3313 return rc; 3314 3315 } 3316 3317 void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, 3318 int *x4_free, int *sb_free) 3319 { 3320 struct npc_subbank *sb; 3321 int i; 3322 3323 /* Reset all stats to zero */ 3324 *x2_free = 0; 3325 *x4_free = 0; 3326 *sb_free = 0; 3327 3328 for (i = 0; i < npc_priv->num_subbanks; i++) { 3329 sb = &npc_priv->sb[i]; 3330 mutex_lock(&sb->lock); 3331 3332 /* Count number of free subbanks */ 3333 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 3334 (*sb_free)++; 3335 goto next; 3336 } 3337 3338 /* Sumup x4 free count */ 3339 if (sb->key_type == NPC_MCAM_KEY_X4) { 3340 (*x4_free) += sb->free_cnt; 3341 goto next; 3342 } 3343 3344 /* Sumup x2 free counts */ 3345 (*x2_free) += sb->free_cnt; 3346 next: 3347 mutex_unlock(&sb->lock); 3348 } 3349 } 3350 3351 int 3352 rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, 3353 struct msg_req *req, 3354 struct npc_cn20k_get_fcnt_rsp *rsp) 3355 { 3356 npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, 3357 &rsp->free_x4, &rsp->free_subbanks); 3358 return 0; 3359 } 3360 3361 int 3362 rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, 3363 struct msg_req *req, 3364 struct npc_cn20k_get_kex_cfg_rsp *rsp) 3365 { 3366 int extr, lt; 3367 3368 rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); 3369 rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); 3370 3371 /* Get EXTRACTOR LID */ 3372 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3373 rsp->intf_extr_lid[NIX_INTF_RX][extr] = 3374 CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); 3375 rsp->intf_extr_lid[NIX_INTF_TX][extr] = 3376 CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); 3377 } 3378 3379 /* Get EXTRACTOR LTYPE */ 3380 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3381 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3382 rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = 3383 CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); 3384 rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = 3385 CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); 3386 } 3387 } 3388 3389 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3390 return 0; 3391 } 3392 3393 static u32 *subbank_srch_order; 3394 3395 static void npc_populate_restricted_idxs(int num_subbanks) 3396 { 3397 npc_subbank_restricted_idxs[0] = num_subbanks - 1; 3398 npc_subbank_restricted_idxs[1] = 0; 3399 } 3400 3401 static int npc_create_srch_order(int cnt) 3402 { 3403 int val = 0; 3404 3405 subbank_srch_order = kcalloc(cnt, sizeof(u32), 3406 GFP_KERNEL); 3407 if (!subbank_srch_order) 3408 return -ENOMEM; 3409 3410 /* cnt(subbank depth) is always a power of 2. There is a check in 3411 * npc_priv_init() to check the same. 3412 */ 3413 for (int i = 0; i < cnt; i += 2) { 3414 subbank_srch_order[i] = cnt / 2 - val - 1; 3415 subbank_srch_order[i + 1] = cnt / 2 + 1 + val; 3416 val++; 3417 } 3418 3419 subbank_srch_order[cnt - 1] = cnt / 2; 3420 return 0; 3421 } 3422 3423 static int npc_subbanks_srch_order_init(struct rvu *rvu) 3424 { 3425 struct npc_subbank *sb; 3426 int sb_idx; 3427 int i, j; 3428 int rc; 3429 3430 for (i = 0; i < npc_priv->num_subbanks; i++) { 3431 sb_idx = subbank_srch_order[i]; 3432 sb = &npc_priv->sb[sb_idx]; 3433 sb->arr_idx = i; 3434 3435 dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", 3436 __func__, sb->idx, sb->arr_idx); 3437 3438 rc = xa_err(xa_store(&npc_priv->xa_sb_free, sb->arr_idx, 3439 xa_mk_value(sb->idx), GFP_KERNEL)); 3440 if (rc) { 3441 dev_err(rvu->dev, 3442 "%s: xa_store(xa_sb_free) failed at slot %d (sb=%d): %d\n", 3443 __func__, i, sb_idx, rc); 3444 for (j = 0; j < i; j++) 3445 xa_erase(&npc_priv->xa_sb_free, j); 3446 return rc; 3447 } 3448 } 3449 3450 return 0; 3451 } 3452 3453 static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) 3454 { 3455 mutex_init(&sb->lock); 3456 3457 sb->b0b = idx * npc_priv->subbank_depth; 3458 sb->b0t = sb->b0b + npc_priv->subbank_depth - 1; 3459 3460 sb->b1b = npc_priv->bank_depth + idx * npc_priv->subbank_depth; 3461 sb->b1t = sb->b1b + npc_priv->subbank_depth - 1; 3462 3463 sb->flags = NPC_SUBBANK_FLAG_FREE; 3464 sb->idx = idx; 3465 } 3466 3467 static int npc_pcifunc_map_create(struct rvu *rvu) 3468 { 3469 int pf, vf, numvfs; 3470 int cnt = 0; 3471 u16 pcifunc; 3472 u64 cfg; 3473 3474 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { 3475 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3476 numvfs = (cfg >> 12) & 0xFF; 3477 3478 /* Skip not enabled PFs */ 3479 if (!(cfg & BIT_ULL(20))) 3480 goto chk_vfs; 3481 3482 /* If Admin function, check on VFs */ 3483 if (cfg & BIT_ULL(21)) 3484 goto chk_vfs; 3485 3486 pcifunc = pf << 9; 3487 3488 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3489 xa_mk_value(cnt), GFP_KERNEL); 3490 3491 cnt++; 3492 3493 chk_vfs: 3494 for (vf = 0; vf < numvfs; vf++) { 3495 pcifunc = (pf << 9) | (vf + 1); 3496 3497 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3498 xa_mk_value(cnt), GFP_KERNEL); 3499 cnt++; 3500 } 3501 } 3502 3503 return cnt; 3504 } 3505 3506 struct npc_defrag_node { 3507 u8 idx; 3508 u8 key_type; 3509 bool valid; 3510 bool refs; 3511 u16 free_cnt; 3512 u16 vidx_cnt; 3513 u16 *vidx; 3514 struct list_head list; 3515 }; 3516 3517 static bool npc_defrag_skip_restricted_sb(int sb_id) 3518 { 3519 int i; 3520 3521 if (!restrict_valid) 3522 return false; 3523 3524 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) 3525 if (sb_id == npc_subbank_restricted_idxs[i]) 3526 return true; 3527 return false; 3528 } 3529 3530 /* Find subbank with minimum number of virtual indexes */ 3531 static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) 3532 { 3533 struct npc_defrag_node *node, *tnode = NULL; 3534 int min = INT_MAX; 3535 3536 list_for_each_entry(node, lh, list) { 3537 if (!node->valid) 3538 continue; 3539 3540 /* if subbank has ref allocated mcam indexes, that subbank 3541 * is not a good candidate to move out indexes. 3542 */ 3543 if (node->refs) 3544 continue; 3545 3546 if (min > node->vidx_cnt) { 3547 min = node->vidx_cnt; 3548 tnode = node; 3549 } 3550 } 3551 3552 return tnode; 3553 } 3554 3555 /* Find subbank with maximum number of free spaces */ 3556 static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) 3557 { 3558 struct npc_defrag_node *node, *tnode = NULL; 3559 int max = INT_MIN; 3560 3561 list_for_each_entry(node, lh, list) { 3562 if (!node->valid) 3563 continue; 3564 3565 if (max < node->free_cnt) { 3566 max = node->free_cnt; 3567 tnode = node; 3568 } 3569 } 3570 3571 return tnode; 3572 } 3573 3574 static int npc_defrag_alloc_free_slots(struct rvu *rvu, 3575 struct npc_defrag_node *f, 3576 int cnt, u16 *save) 3577 { 3578 int alloc_cnt1, alloc_cnt2; 3579 struct npc_subbank *sb; 3580 int rc, sb_off, i, err; 3581 bool deleted; 3582 3583 sb = &npc_priv->sb[f->idx]; 3584 3585 alloc_cnt1 = 0; 3586 alloc_cnt2 = 0; 3587 3588 rc = __npc_subbank_alloc(rvu, sb, 3589 f->key_type, sb->b0b, 3590 sb->b0t, 3591 NPC_MCAM_LOWER_PRIO, 3592 false, cnt, save, cnt, true, 3593 &alloc_cnt1); 3594 3595 /* X4 entries only occupy bank 0 (b0b..b0t); see npc_subbank_idx_2_mcam_idx(). 3596 * X2 uses both halves of the subbank, so spill into bank 1 if needed. 3597 */ 3598 if (alloc_cnt1 < cnt && f->key_type == NPC_MCAM_KEY_X2) { 3599 rc = __npc_subbank_alloc(rvu, sb, 3600 f->key_type, sb->b1b, 3601 sb->b1t, 3602 NPC_MCAM_LOWER_PRIO, 3603 false, cnt - alloc_cnt1, 3604 save + alloc_cnt1, 3605 cnt - alloc_cnt1, 3606 true, &alloc_cnt2); 3607 } 3608 3609 if (alloc_cnt1 + alloc_cnt2 != cnt) { 3610 dev_err(rvu->dev, 3611 "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", 3612 __func__, cnt, alloc_cnt1, alloc_cnt2); 3613 rc = -ENOSPC; 3614 goto fail_free_alloc; 3615 } 3616 3617 return 0; 3618 3619 fail_free_alloc: 3620 for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { 3621 err = npc_mcam_idx_2_subbank_idx(rvu, save[i], 3622 &sb, &sb_off); 3623 if (err) { 3624 dev_err(rvu->dev, 3625 "%s: Error to find subbank for mcam idx=%u\n", 3626 __func__, save[i]); 3627 break; 3628 } 3629 3630 deleted = __npc_subbank_free(rvu, sb, sb_off); 3631 if (!deleted) { 3632 dev_err(rvu->dev, 3633 "%s: Error to free mcam idx=%u\n", 3634 __func__, save[i]); 3635 break; 3636 } 3637 } 3638 3639 return rc; 3640 } 3641 3642 static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, 3643 u16 new_midx, u16 vidx) 3644 { 3645 struct npc_defrag_show_node *node; 3646 3647 node = kzalloc_objs(*node, 1); 3648 if (!node) 3649 return -ENOMEM; 3650 3651 node->old_midx = old_midx; 3652 node->new_midx = new_midx; 3653 node->vidx = vidx; 3654 INIT_LIST_HEAD(&node->list); 3655 3656 mutex_lock(&npc_priv->lock); 3657 list_add_tail(&node->list, &npc_priv->defrag_lh); 3658 mutex_unlock(&npc_priv->lock); 3659 3660 return 0; 3661 } 3662 3663 static 3664 int npc_defrag_move_vdx_to_free(struct rvu *rvu, 3665 struct npc_defrag_node *f, 3666 struct npc_defrag_node *v, 3667 int cnt, u16 *save) 3668 { 3669 u16 new_midx, old_midx, vidx, target_pf; 3670 struct npc_mcam *mcam = &rvu->hw->mcam; 3671 struct rvu_npc_mcam_rule *rule, *tmp; 3672 int i, vidx_cnt, rc, sb_off; 3673 struct npc_subbank *sb; 3674 bool deleted; 3675 u16 pcifunc; 3676 int blkaddr; 3677 void *map; 3678 u8 bank; 3679 u16 midx; 3680 u64 stats; 3681 3682 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3683 3684 vidx_cnt = v->vidx_cnt; 3685 for (i = 0; i < cnt; i++) { 3686 vidx = v->vidx[vidx_cnt - i - 1]; 3687 old_midx = npc_vidx2idx(vidx); 3688 new_midx = save[cnt - i - 1]; 3689 3690 dev_dbg(rvu->dev, 3691 "%s: Moving %u ---> %u (vidx=%u)\n", 3692 __func__, 3693 old_midx, new_midx, vidx); 3694 3695 rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); 3696 if (rc) 3697 dev_err(rvu->dev, 3698 "%s: Error happened to add to show list vidx=%u\n", 3699 __func__, vidx); 3700 3701 /* Modify vidx to point to new mcam idx */ 3702 rc = npc_vidx_maps_modify(rvu, vidx, new_midx); 3703 if (rc) 3704 return rc; 3705 3706 midx = old_midx % mcam->banksize; 3707 bank = old_midx / mcam->banksize; 3708 stats = rvu_read64(rvu, blkaddr, 3709 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, 3710 bank)); 3711 3712 /* If bug happened during copy/enable mcam, then there is a bug in allocation 3713 * algorithm itself. There is no point in rewinding and returning, as it 3714 * will face further issue. Return error after printing error 3715 */ 3716 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) { 3717 dev_err(rvu->dev, 3718 "%s: Error happened while disabling old_mid=%u\n", 3719 __func__, old_midx); 3720 return -EFAULT; 3721 } 3722 3723 if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) { 3724 dev_err(rvu->dev, 3725 "%s: Error happened while copying old_midx=%u new_midx=%u\n", 3726 __func__, old_midx, new_midx); 3727 return -EFAULT; 3728 } 3729 3730 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) { 3731 dev_err(rvu->dev, 3732 "%s: Error happened while enabling new_mid=%u\n", 3733 __func__, new_midx); 3734 return -EFAULT; 3735 } 3736 3737 midx = new_midx % mcam->banksize; 3738 bank = new_midx / mcam->banksize; 3739 rvu_write64(rvu, blkaddr, 3740 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), 3741 stats); 3742 3743 /* Free the old mcam idx */ 3744 rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, 3745 &sb, &sb_off); 3746 if (rc) { 3747 dev_err(rvu->dev, 3748 "%s: Unable to calculate subbank off for mcamidx=%u\n", 3749 __func__, old_midx); 3750 return rc; 3751 } 3752 3753 deleted = __npc_subbank_free(rvu, sb, sb_off); 3754 if (!deleted) { 3755 dev_err(rvu->dev, 3756 "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", 3757 __func__, old_midx, sb->idx, sb_off); 3758 return -EFAULT; 3759 } 3760 3761 /* save pcifunc */ 3762 map = xa_load(&npc_priv->xa_idx2pf_map, old_midx); 3763 pcifunc = xa_to_value(map); 3764 3765 /* delete from pf maps */ 3766 rc = npc_del_from_pf_maps(rvu, old_midx); 3767 if (rc) { 3768 dev_err(rvu->dev, 3769 "%s: Failed to delete pf maps for mcamidx=%u\n", 3770 __func__, old_midx); 3771 return rc; 3772 } 3773 3774 /* add new mcam_idx to pf map */ 3775 rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); 3776 if (rc) { 3777 dev_err(rvu->dev, 3778 "%s: Failed to add pf maps for mcamidx=%u\n", 3779 __func__, new_midx); 3780 return rc; 3781 } 3782 3783 /* Remove from mcam maps */ 3784 mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; 3785 mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; 3786 npc_mcam_clear_bit(mcam, old_midx); 3787 3788 mcam->entry2pfvf_map[new_midx] = pcifunc; 3789 /* Counter is not preserved */ 3790 mcam->entry2cntr_map[new_midx] = new_midx; 3791 target_pf = mcam->entry2target_pffunc[old_midx]; 3792 mcam->entry2target_pffunc[new_midx] = target_pf; 3793 mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP; 3794 3795 npc_mcam_set_bit(mcam, new_midx); 3796 3797 /* Note: list order is not functionally required for mcam_rules */ 3798 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 3799 if (rule->entry != old_midx) 3800 continue; 3801 3802 rule->entry = new_midx; 3803 break; 3804 } 3805 3806 /* Mark as invalid */ 3807 v->vidx[vidx_cnt - i - 1] = -1; 3808 save[cnt - i - 1] = -1; 3809 3810 f->free_cnt--; 3811 v->vidx_cnt--; 3812 } 3813 3814 return 0; 3815 } 3816 3817 static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) 3818 { 3819 struct npc_defrag_node *v = NULL; 3820 struct npc_defrag_node *f = NULL; 3821 int rc = 0, cnt; 3822 u16 *save; 3823 3824 while (1) { 3825 /* Find subbank with minimum vidx */ 3826 if (!v) { 3827 v = npc_subbank_min_vidx(lh); 3828 if (!v) 3829 break; 3830 } 3831 3832 /* Find subbank with maximum free slots */ 3833 if (!f) { 3834 f = npc_subbank_max_free(lh); 3835 if (!f) 3836 break; 3837 } 3838 3839 if (!v->vidx_cnt) { 3840 list_del_init(&v->list); 3841 v = NULL; 3842 continue; 3843 } 3844 3845 if (!f->free_cnt) { 3846 list_del_init(&f->list); 3847 f = NULL; 3848 continue; 3849 } 3850 3851 /* If both subbanks are same, choose vidx and 3852 * search for free list again 3853 */ 3854 if (f == v) { 3855 list_del_init(&f->list); 3856 f = NULL; 3857 continue; 3858 } 3859 3860 /* Calculate minimum free slots needs to be allocated */ 3861 cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : 3862 f->free_cnt; 3863 3864 dev_dbg(rvu->dev, 3865 "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", 3866 __func__, cnt, f->free_cnt, f->idx, 3867 v->vidx_cnt, v->idx); 3868 3869 /* Allocate an array to store newly allocated 3870 * free slots (mcam indexes) 3871 */ 3872 save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); 3873 if (!save) { 3874 rc = -ENOMEM; 3875 goto err; 3876 } 3877 3878 /* Alloc free slots for existing vidx */ 3879 rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); 3880 if (rc) { 3881 kfree(save); 3882 goto err; 3883 } 3884 3885 /* Move vidx to free slots; update pf_map and vidx maps, 3886 * and free existing vidx mcam slots 3887 */ 3888 rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); 3889 if (rc) { 3890 kfree(save); 3891 goto err; 3892 } 3893 3894 kfree(save); 3895 3896 if (!f->free_cnt) { 3897 list_del_init(&f->list); 3898 f = NULL; 3899 } 3900 3901 if (!v->vidx_cnt) { 3902 list_del_init(&v->list); 3903 v = NULL; 3904 } 3905 } 3906 3907 err: 3908 /* Whole defragmentation process is done within locks. if there 3909 * is an error, it would be hard to roll back as index remove/add 3910 * can fail again if it failed before. This would mean that there 3911 * is bug in the index management algorithm. 3912 * Return from here than rolling back. 3913 */ 3914 return rc; 3915 } 3916 3917 static void npc_defrag_list_clear(void) 3918 { 3919 struct npc_defrag_show_node *node, *next; 3920 3921 mutex_lock(&npc_priv->lock); 3922 list_for_each_entry_safe(node, next, &npc_priv->defrag_lh, list) { 3923 list_del_init(&node->list); 3924 kfree(node); 3925 } 3926 3927 mutex_unlock(&npc_priv->lock); 3928 } 3929 3930 static void npc_lock_all_subbank(void) 3931 { 3932 int i; 3933 3934 for (i = 0; i < npc_priv->num_subbanks; i++) 3935 mutex_lock(&npc_priv->sb[i].lock); 3936 } 3937 3938 static void npc_unlock_all_subbank(void) 3939 { 3940 int i; 3941 3942 for (i = npc_priv->num_subbanks - 1; i >= 0; i--) 3943 mutex_unlock(&npc_priv->sb[i].lock); 3944 } 3945 3946 int npc_cn20k_search_order_set(struct rvu *rvu, 3947 u64 narr[MAX_NUM_SUB_BANKS], int cnt) 3948 { 3949 struct npc_mcam *mcam = &rvu->hw->mcam; 3950 int rsrc[2][MAX_NUM_SUB_BANKS] = { }; 3951 u8 save[MAX_NUM_SUB_BANKS] = { }; 3952 struct npc_subbank *sb; 3953 struct xarray *xa; 3954 int prio, rc, err; 3955 int sb_idx; 3956 enum { 3957 FREE = 0, 3958 USED = 1, 3959 }; 3960 3961 if (cnt != npc_priv->num_subbanks) { 3962 dev_err(rvu->dev, "Number of entries(%u) != %u\n", 3963 cnt, npc_priv->num_subbanks); 3964 return -EINVAL; 3965 } 3966 3967 mutex_lock(&mcam->lock); 3968 npc_lock_all_subbank(); 3969 3970 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 3971 sb = &npc_priv->sb[sb_idx]; 3972 save[sb->idx] = sb->arr_idx; 3973 } 3974 3975 for (prio = 0; prio < cnt; prio++) { 3976 sb_idx = narr[prio]; 3977 sb = &npc_priv->sb[sb_idx]; 3978 3979 if (sb->flags & NPC_SUBBANK_FLAG_USED) 3980 xa = &npc_priv->xa_sb_used; 3981 else 3982 xa = &npc_priv->xa_sb_free; 3983 3984 rc = xa_err(xa_store(xa, prio, 3985 xa_mk_value(sb_idx), GFP_KERNEL)); 3986 if (rc) { 3987 dev_err(rvu->dev, 3988 "Setting arr_idx=%d for sb=%d failed\n", 3989 sb->arr_idx, sb_idx); 3990 goto fail; 3991 } 3992 3993 if (sb->flags & NPC_SUBBANK_FLAG_USED) { 3994 rsrc[USED][sb->arr_idx] -= 1; 3995 rsrc[USED][prio] += 1; 3996 } else { 3997 rsrc[FREE][sb->arr_idx] -= 1; 3998 rsrc[FREE][prio] += 1; 3999 } 4000 4001 sb->arr_idx = prio; 4002 } 4003 4004 for (prio = 0; prio < cnt; prio++) { 4005 if (rsrc[FREE][prio] == -1) 4006 xa_erase(&npc_priv->xa_sb_free, prio); 4007 4008 if (rsrc[USED][prio] == -1) 4009 xa_erase(&npc_priv->xa_sb_used, prio); 4010 } 4011 4012 for (int i = 0; i < cnt; i++) 4013 subbank_srch_order[i] = (u32)narr[i]; 4014 4015 restrict_valid = false; 4016 4017 npc_unlock_all_subbank(); 4018 mutex_unlock(&mcam->lock); 4019 4020 return 0; 4021 4022 fail: 4023 for (prio = 0; prio < cnt; prio++) { 4024 if (rsrc[FREE][prio] == 1) 4025 xa_erase(&npc_priv->xa_sb_free, prio); 4026 4027 if (rsrc[USED][prio] == 1) 4028 xa_erase(&npc_priv->xa_sb_used, prio); 4029 } 4030 4031 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 4032 sb = &npc_priv->sb[sb_idx]; 4033 sb->arr_idx = save[sb_idx]; 4034 4035 if (sb->flags & NPC_SUBBANK_FLAG_USED) 4036 xa = &npc_priv->xa_sb_used; 4037 else 4038 xa = &npc_priv->xa_sb_free; 4039 4040 /* Since the entry already exists, xa_store() replaces 4041 * the value without a kmalloc(), making failure highly unlikely. 4042 */ 4043 err = xa_err(xa_store(xa, sb->arr_idx, 4044 xa_mk_value(sb->idx), GFP_KERNEL)); 4045 WARN(!!err, "Failed to rollback sb=%u idx=%u\n", 4046 sb->idx, sb->arr_idx); 4047 } 4048 4049 npc_unlock_all_subbank(); 4050 mutex_unlock(&mcam->lock); 4051 4052 return rc; 4053 } 4054 4055 const u32 *npc_cn20k_search_order_get(bool *restricted_order, u32 *sz) 4056 { 4057 *restricted_order = restrict_valid; 4058 *sz = npc_priv->num_subbanks; 4059 return subbank_srch_order; 4060 } 4061 4062 /* Only non-ref non-contigous mcam indexes 4063 * are picked for defrag process 4064 */ 4065 int npc_cn20k_defrag(struct rvu *rvu) 4066 { 4067 struct npc_mcam *mcam = &rvu->hw->mcam; 4068 struct npc_defrag_node *node, *tnode; 4069 struct list_head x4lh, x2lh, *lh; 4070 int rc = 0, i, sb_off, tot; 4071 struct npc_subbank *sb; 4072 unsigned long index; 4073 void *map; 4074 u16 midx; 4075 4076 /* Free previous show list */ 4077 npc_defrag_list_clear(); 4078 4079 INIT_LIST_HEAD(&x4lh); 4080 INIT_LIST_HEAD(&x2lh); 4081 4082 node = kzalloc_objs(*node, npc_priv->num_subbanks); 4083 if (!node) 4084 return -ENOMEM; 4085 4086 /* Lock mcam */ 4087 mutex_lock(&mcam->lock); 4088 npc_lock_all_subbank(); 4089 4090 /* Fill in node with subbank properties */ 4091 for (i = 0; i < npc_priv->num_subbanks; i++) { 4092 sb = &npc_priv->sb[i]; 4093 4094 node[i].idx = i; 4095 node[i].key_type = sb->key_type; 4096 node[i].free_cnt = sb->free_cnt; 4097 node[i].vidx = kcalloc(npc_priv->subbank_depth * 2, 4098 sizeof(*node[i].vidx), 4099 GFP_KERNEL); 4100 if (!node[i].vidx) { 4101 rc = -ENOMEM; 4102 goto free_vidx; 4103 } 4104 4105 /* If subbank is empty, dont include it in defrag 4106 * process 4107 */ 4108 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 4109 node[i].valid = false; 4110 continue; 4111 } 4112 4113 if (npc_defrag_skip_restricted_sb(i)) { 4114 node[i].valid = false; 4115 continue; 4116 } 4117 4118 node[i].valid = true; 4119 INIT_LIST_HEAD(&node[i].list); 4120 4121 /* Add node to x2 or x4 list */ 4122 lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; 4123 list_add_tail(&node[i].list, lh); 4124 } 4125 4126 /* Filling vidx[] array with all vidx in that subbank */ 4127 xa_for_each_start(&npc_priv->xa_vidx2idx_map, index, map, 4128 npc_priv->bank_depth * 2) { 4129 midx = xa_to_value(map); 4130 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 4131 &sb, &sb_off); 4132 if (rc) { 4133 dev_err(rvu->dev, 4134 "%s: Error to get mcam_idx for vidx=%lu\n", 4135 __func__, index); 4136 goto free_vidx; 4137 } 4138 4139 tnode = &node[sb->idx]; 4140 tnode->vidx[tnode->vidx_cnt] = index; 4141 tnode->vidx_cnt++; 4142 } 4143 4144 /* Mark all subbank which has ref allocation */ 4145 for (i = 0; i < npc_priv->num_subbanks; i++) { 4146 tnode = &node[i]; 4147 4148 if (!tnode->valid) 4149 continue; 4150 4151 tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? 4152 npc_priv->subbank_depth * 2 : npc_priv->subbank_depth; 4153 4154 if (node[i].vidx_cnt != tot - tnode->free_cnt) 4155 tnode->refs = true; 4156 } 4157 4158 rc = npc_defrag_process(rvu, &x2lh); 4159 if (rc) 4160 goto free_vidx; 4161 4162 rc = npc_defrag_process(rvu, &x4lh); 4163 if (rc) 4164 goto free_vidx; 4165 4166 free_vidx: 4167 npc_unlock_all_subbank(); 4168 mutex_unlock(&mcam->lock); 4169 for (i = 0; i < npc_priv->num_subbanks; i++) 4170 kfree(node[i].vidx); 4171 kfree(node); 4172 return rc; 4173 } 4174 4175 int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, 4176 struct msg_rsp *rsp) 4177 { 4178 return npc_cn20k_defrag(rvu); 4179 } 4180 4181 int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, 4182 u16 *mcast, u16 *promisc, u16 *ucast) 4183 { 4184 u16 *ptr[4] = {promisc, mcast, bcast, ucast}; 4185 unsigned long idx; 4186 bool set = false; 4187 void *val; 4188 int i, j; 4189 4190 for (i = 0; i < ARRAY_SIZE(ptr); i++) { 4191 if (!ptr[i]) 4192 continue; 4193 4194 *ptr[i] = USHRT_MAX; 4195 } 4196 4197 if (!npc_priv) 4198 return 0; 4199 4200 if (is_lbk_vf(rvu, pcifunc)) { 4201 if (!ptr[0]) 4202 return -EINVAL; 4203 4204 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4205 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4206 if (!val) { 4207 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4208 __func__, 4209 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4210 pcifunc); 4211 4212 return -ESRCH; 4213 } 4214 4215 *ptr[0] = xa_to_value(val); 4216 return 0; 4217 } 4218 4219 if (is_vf(pcifunc)) { 4220 if (!ptr[3]) 4221 return -EINVAL; 4222 4223 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4224 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4225 if (!val) { 4226 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4227 __func__, 4228 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4229 pcifunc); 4230 4231 return -ESRCH; 4232 } 4233 4234 *ptr[3] = xa_to_value(val); 4235 return 0; 4236 } 4237 4238 for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, 4239 j++) { 4240 if (!ptr[j]) 4241 continue; 4242 4243 idx = NPC_DFT_RULE_ID_MK(pcifunc, i); 4244 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4245 if (!val) { 4246 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4247 __func__, 4248 npc_dft_rule_name[i], pcifunc); 4249 4250 continue; 4251 } 4252 4253 *ptr[j] = xa_to_value(val); 4254 set = true; 4255 } 4256 4257 return set ? 0 : -ESRCH; 4258 } 4259 4260 int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, 4261 struct npc_get_pfl_info_rsp *rsp) 4262 { 4263 if (!is_cn20k(rvu->pdev)) { 4264 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4265 return -EOPNOTSUPP; 4266 } 4267 4268 rsp->kw_type = npc_priv->kw; 4269 rsp->x4_slots = npc_priv->bank_depth; 4270 return 0; 4271 } 4272 4273 int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, 4274 struct npc_get_num_kws_req *req, 4275 struct npc_get_num_kws_rsp *rsp) 4276 { 4277 u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4278 u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4279 struct rvu_npc_mcam_rule dummy = { 0 }; 4280 struct mcam_entry_mdata mdata = { }; 4281 struct npc_install_flow_req *fl; 4282 int i, cnt = 0, blkaddr; 4283 4284 if (!is_cn20k(rvu->pdev)) { 4285 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4286 return -EOPNOTSUPP; 4287 } 4288 4289 fl = &req->fl; 4290 4291 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4292 if (blkaddr < 0) { 4293 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4294 return NPC_MCAM_INVALID_REQ; 4295 } 4296 4297 mdata.kw = kw; 4298 mdata.kw_mask = kw_mask; 4299 4300 npc_update_flow(rvu, &mdata, fl->features, &fl->packet, 4301 &fl->mask, &dummy, fl->intf, blkaddr); 4302 4303 /* Find the most significant word valid. Traverse from 4304 * MSB to LSB, check if cam0 or cam1 is set 4305 */ 4306 for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { 4307 if (kw[i] || kw_mask[i]) { 4308 cnt = i + 1; 4309 break; 4310 } 4311 } 4312 4313 rsp->kws = cnt; 4314 4315 return 0; 4316 } 4317 4318 int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, 4319 struct npc_get_dft_rl_idxs_rsp *rsp) 4320 { 4321 u16 bcast, mcast, promisc, ucast; 4322 u16 pcifunc; 4323 int rc; 4324 4325 if (!is_cn20k(rvu->pdev)) { 4326 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4327 return -EOPNOTSUPP; 4328 } 4329 4330 pcifunc = req->hdr.pcifunc; 4331 4332 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 4333 &promisc, &ucast); 4334 if (rc) 4335 return rc; 4336 4337 rsp->bcast = bcast; 4338 rsp->mcast = mcast; 4339 rsp->promisc = promisc; 4340 rsp->ucast = ucast; 4341 return 0; 4342 } 4343 4344 bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) 4345 { 4346 return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || 4347 is_lbk_vf(rvu, pcifunc); 4348 } 4349 4350 void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) 4351 { 4352 struct npc_mcam *mcam = &rvu->hw->mcam; 4353 u16 ptr[4] = {[0 ... 3] = USHRT_MAX}; 4354 struct rvu_npc_mcam_rule *rule, *tmp; 4355 unsigned long index; 4356 int blkaddr, rc, i; 4357 void *map; 4358 4359 if (!npc_priv) 4360 return; 4361 4362 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4363 dev_dbg(rvu->dev, 4364 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4365 __func__, pcifunc); 4366 return; 4367 } 4368 4369 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 4370 &ptr[2], &ptr[3]); 4371 if (rc) 4372 return; 4373 4374 /* LBK */ 4375 if (is_lbk_vf(rvu, pcifunc)) { 4376 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4377 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4378 if (!map) 4379 dev_dbg(rvu->dev, 4380 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4381 __func__, 4382 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4383 pcifunc); 4384 4385 goto free_rules; 4386 } 4387 4388 /* VF */ 4389 if (is_vf(pcifunc)) { 4390 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4391 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4392 if (!map) 4393 dev_dbg(rvu->dev, 4394 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4395 __func__, 4396 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4397 pcifunc); 4398 4399 goto free_rules; 4400 } 4401 4402 /* PF */ 4403 for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { 4404 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4405 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4406 if (!map) 4407 dev_dbg(rvu->dev, 4408 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4409 __func__, npc_dft_rule_name[i], 4410 pcifunc); 4411 } 4412 4413 free_rules: 4414 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4415 if (blkaddr < 0) 4416 return; 4417 for (int i = 0; i < 4; i++) { 4418 if (ptr[i] == USHRT_MAX) 4419 continue; 4420 4421 mutex_lock(&mcam->lock); 4422 npc_mcam_clear_bit(mcam, ptr[i]); 4423 mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP; 4424 npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false); 4425 mcam->entry2target_pffunc[ptr[i]] = 0x0; 4426 mutex_unlock(&mcam->lock); 4427 4428 rc = npc_cn20k_idx_free(rvu, &ptr[i], 1); 4429 if (rc) { 4430 /* Non recoverable error. Let us WARN and return. Keep system alive to 4431 * enable debugging 4432 */ 4433 WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n", 4434 __func__, pcifunc, ptr[i]); 4435 return; 4436 } 4437 } 4438 4439 mutex_lock(&mcam->lock); 4440 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 4441 for (int i = 0; i < 4; i++) { 4442 if (ptr[i] != rule->entry) 4443 continue; 4444 4445 list_del(&rule->list); 4446 kfree(rule); 4447 break; 4448 } 4449 } 4450 mutex_unlock(&mcam->lock); 4451 } 4452 4453 int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) 4454 { 4455 struct npc_mcam_free_entry_req free_req = { 0 }; 4456 u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; 4457 struct npc_mcam_alloc_entry_req req = { 0 }; 4458 struct npc_mcam_alloc_entry_rsp rsp = { 0 }; 4459 int ret, eidx, i, k, pf, cnt; 4460 struct rvu_pfvf *pfvf; 4461 unsigned long index; 4462 struct msg_rsp free_rsp; 4463 u16 b, m, p, u; 4464 4465 if (!npc_priv) 4466 return 0; 4467 4468 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4469 dev_dbg(rvu->dev, 4470 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4471 __func__, pcifunc); 4472 return 0; 4473 } 4474 4475 /* Check if default rules are already alloced for this pcifunc */ 4476 ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); 4477 if (!ret) { 4478 dev_dbg(rvu->dev, 4479 "%s: default rules are already installed (pcifunc=%#x)\n", 4480 __func__, pcifunc); 4481 dev_dbg(rvu->dev, 4482 "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", 4483 __func__, b, m, p, u); 4484 return 0; 4485 } 4486 4487 /* Set ref index as lowest priority index */ 4488 eidx = 2 * npc_priv->bank_depth - 1; 4489 4490 /* Install only UCAST for VF */ 4491 cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); 4492 4493 /* For VF pcifunc, allocate default mcam indexes by taking 4494 * ref as PF's ucast index. 4495 */ 4496 if (is_vf(pcifunc)) { 4497 pf = rvu_get_pf(rvu->pdev, pcifunc); 4498 pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; 4499 4500 /* Get PF's ucast entry index */ 4501 ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, 4502 NULL, NULL, &pf_ucast); 4503 4504 /* There is no PF rules installed; and VF installation comes 4505 * first. PF may come later. 4506 * TODO: Install PF rules before installing VF rules. 4507 */ 4508 4509 /* Set PF's ucast as ref entry */ 4510 if (!ret) 4511 eidx = pf_ucast; 4512 } 4513 4514 pfvf = rvu_get_pfvf(rvu, pcifunc); 4515 pfvf->hw_prio = NPC_DFT_RULE_PRIO; 4516 4517 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4518 req.kw_type = NPC_MCAM_KEY_X4; 4519 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4520 } else { 4521 req.kw_type = NPC_MCAM_KEY_X2; 4522 req.ref_entry = eidx; 4523 } 4524 4525 req.contig = false; 4526 req.ref_prio = NPC_MCAM_HIGHER_PRIO; 4527 req.count = cnt; 4528 req.hdr.pcifunc = pcifunc; 4529 4530 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4531 4532 /* successfully allocated index */ 4533 if (!ret) { 4534 /* Copy indexes to local array */ 4535 for (i = 0; i < cnt; i++) 4536 mcam_idx[i] = rsp.entry_list[i]; 4537 4538 goto chk_sanity; 4539 } 4540 4541 /* If there is no slots available and request is for PF, 4542 * return error. 4543 */ 4544 if (!is_vf(pcifunc)) { 4545 dev_err(rvu->dev, 4546 "%s: Default index allocation failed for pcifunc=%#x\n", 4547 __func__, pcifunc); 4548 return ret; 4549 } 4550 4551 /* We could not find an index with higher priority index for VF. 4552 * Find rule with lower priority index and set hardware priority 4553 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) 4554 */ 4555 req.contig = false; 4556 req.count = cnt; 4557 req.hdr.pcifunc = pcifunc; 4558 req.ref_prio = NPC_MCAM_LOWER_PRIO; 4559 4560 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4561 req.kw_type = NPC_MCAM_KEY_X4; 4562 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4563 } else { 4564 req.kw_type = NPC_MCAM_KEY_X2; 4565 req.ref_entry = eidx; 4566 } 4567 4568 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4569 if (ret) { 4570 dev_err(rvu->dev, 4571 "%s: Default index allocation failed for pcifunc=%#x\n", 4572 __func__, pcifunc); 4573 return ret; 4574 } 4575 4576 /* Copy indexes to local array */ 4577 for (i = 0; i < cnt; i++) 4578 mcam_idx[i] = rsp.entry_list[i]; 4579 4580 pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; 4581 4582 chk_sanity: 4583 /* LBK */ 4584 if (is_lbk_vf(rvu, pcifunc)) { 4585 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4586 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4587 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4588 if (ret) { 4589 dev_err(rvu->dev, 4590 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4591 __func__, 4592 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4593 pcifunc); 4594 goto err; 4595 } 4596 4597 goto done; 4598 } 4599 4600 /* VF */ 4601 if (is_vf(pcifunc)) { 4602 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4603 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4604 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4605 if (ret) { 4606 dev_err(rvu->dev, 4607 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4608 __func__, 4609 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4610 pcifunc); 4611 goto err; 4612 } 4613 4614 goto done; 4615 } 4616 4617 /* PF */ 4618 for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && 4619 k < cnt; i++, k++) { 4620 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4621 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4622 xa_mk_value(mcam_idx[k]), GFP_KERNEL); 4623 if (ret) { 4624 dev_err(rvu->dev, 4625 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4626 __func__, npc_dft_rule_name[i], 4627 pcifunc); 4628 for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { 4629 index = NPC_DFT_RULE_ID_MK(pcifunc, p); 4630 xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4631 } 4632 goto err; 4633 } 4634 } 4635 4636 done: 4637 return 0; 4638 4639 err: 4640 free_req.hdr.pcifunc = pcifunc; 4641 free_req.all = 1; 4642 ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 4643 if (ret) 4644 dev_err(rvu->dev, 4645 "%s: Error deleting default entries (pcifunc=%#x\n", 4646 __func__, pcifunc); 4647 4648 return -EFAULT; 4649 } 4650 4651 static int npc_priv_init(struct rvu *rvu) 4652 { 4653 struct npc_mcam *mcam = &rvu->hw->mcam; 4654 int blkaddr, num_banks, bank_depth; 4655 int num_subbanks, subbank_depth; 4656 u64 npc_const1, npc_const2 = 0; 4657 struct npc_subbank *sb; 4658 int ret = -ENOMEM; 4659 u64 cfg; 4660 int i; 4661 4662 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4663 if (blkaddr < 0) { 4664 dev_err(rvu->dev, "%s: NPC block not implemented\n", 4665 __func__); 4666 return -ENODEV; 4667 } 4668 4669 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 4670 if (npc_const1 & BIT_ULL(63)) 4671 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 4672 4673 num_banks = mcam->banks; 4674 if (!num_banks || num_banks > MAX_NUM_BANKS) { 4675 dev_err(rvu->dev, 4676 "Number of banks(%u) is invalid\n", num_banks); 4677 return -EINVAL; 4678 } 4679 4680 num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); 4681 if (!num_subbanks || num_subbanks > MAX_NUM_SUB_BANKS) { 4682 dev_err(rvu->dev, 4683 "Number of subbanks is invalid %u\n", 4684 num_subbanks); 4685 return -EFAULT; 4686 } 4687 4688 if (num_subbanks & (num_subbanks - 1)) { 4689 dev_err(rvu->dev, 4690 "subbanks cnt(%u) should be a power of 2\n", 4691 num_subbanks); 4692 return -EINVAL; 4693 } 4694 4695 bank_depth = mcam->banksize; 4696 if (!bank_depth || bank_depth % num_subbanks) { 4697 dev_err(rvu->dev, 4698 "Bank depth(%u) should be a multiple of num_subbanks(%u)\n", 4699 bank_depth, num_subbanks); 4700 return -EINVAL; 4701 } 4702 4703 subbank_depth = bank_depth / num_subbanks; 4704 if (!subbank_depth || subbank_depth > MAX_SUBBANK_DEPTH) { 4705 dev_err(rvu->dev, 4706 "Invalid subbank depth %u\n", 4707 subbank_depth); 4708 return -EINVAL; 4709 } 4710 4711 npc_priv = kzalloc_objs(*npc_priv, 1); 4712 if (!npc_priv) 4713 return -ENOMEM; 4714 4715 npc_priv->num_banks = num_banks; 4716 npc_priv->num_subbanks = num_subbanks; 4717 npc_priv->bank_depth = bank_depth; 4718 npc_priv->subbank_depth = subbank_depth; 4719 4720 /* Get kex configured key size */ 4721 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); 4722 npc_priv->kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); 4723 4724 dev_info(rvu->dev, 4725 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", 4726 num_banks, bank_depth, num_subbanks, subbank_depth, 4727 npc_kw_name[npc_priv->kw]); 4728 4729 npc_priv->sb = kzalloc_objs(struct npc_subbank, num_subbanks); 4730 if (!npc_priv->sb) 4731 goto fail1; 4732 4733 xa_init_flags(&npc_priv->xa_sb_used, XA_FLAGS_ALLOC); 4734 xa_init_flags(&npc_priv->xa_sb_free, XA_FLAGS_ALLOC); 4735 xa_init_flags(&npc_priv->xa_idx2pf_map, XA_FLAGS_ALLOC); 4736 xa_init_flags(&npc_priv->xa_pf_map, XA_FLAGS_ALLOC); 4737 xa_init_flags(&npc_priv->xa_pf2dfl_rmap, XA_FLAGS_ALLOC); 4738 xa_init_flags(&npc_priv->xa_idx2vidx_map, XA_FLAGS_ALLOC); 4739 xa_init_flags(&npc_priv->xa_vidx2idx_map, XA_FLAGS_ALLOC); 4740 4741 if (npc_create_srch_order(num_subbanks)) 4742 goto fail2; 4743 4744 npc_populate_restricted_idxs(num_subbanks); 4745 4746 /* Initialize subbanks */ 4747 for (i = 0, sb = npc_priv->sb; i < num_subbanks; i++, sb++) 4748 npc_subbank_init(rvu, sb, i); 4749 4750 ret = npc_subbanks_srch_order_init(rvu); 4751 if (ret) 4752 goto fail3; 4753 4754 /* Get number of pcifuncs in the system */ 4755 npc_priv->pf_cnt = npc_pcifunc_map_create(rvu); 4756 npc_priv->xa_pf2idx_map = kzalloc_objs(struct xarray, npc_priv->pf_cnt); 4757 if (!npc_priv->xa_pf2idx_map) { 4758 ret = -ENOMEM; 4759 goto fail3; 4760 } 4761 4762 for (i = 0; i < npc_priv->pf_cnt; i++) 4763 xa_init_flags(&npc_priv->xa_pf2idx_map[i], XA_FLAGS_ALLOC); 4764 4765 INIT_LIST_HEAD(&npc_priv->defrag_lh); 4766 mutex_init(&npc_priv->lock); 4767 4768 return 0; 4769 4770 fail3: 4771 kfree(subbank_srch_order); 4772 subbank_srch_order = NULL; 4773 4774 fail2: 4775 xa_destroy(&npc_priv->xa_sb_used); 4776 xa_destroy(&npc_priv->xa_sb_free); 4777 xa_destroy(&npc_priv->xa_idx2pf_map); 4778 xa_destroy(&npc_priv->xa_pf_map); 4779 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4780 xa_destroy(&npc_priv->xa_idx2vidx_map); 4781 xa_destroy(&npc_priv->xa_vidx2idx_map); 4782 kfree(npc_priv->sb); 4783 npc_priv->sb = NULL; 4784 fail1: 4785 kfree(npc_priv); 4786 npc_priv = NULL; 4787 return ret; 4788 } 4789 4790 void npc_cn20k_deinit(struct rvu *rvu) 4791 { 4792 int i; 4793 4794 if (!npc_priv) 4795 return; 4796 4797 xa_destroy(&npc_priv->xa_sb_used); 4798 xa_destroy(&npc_priv->xa_sb_free); 4799 xa_destroy(&npc_priv->xa_idx2pf_map); 4800 xa_destroy(&npc_priv->xa_pf_map); 4801 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4802 xa_destroy(&npc_priv->xa_idx2vidx_map); 4803 xa_destroy(&npc_priv->xa_vidx2idx_map); 4804 4805 for (i = 0; i < npc_priv->pf_cnt; i++) 4806 xa_destroy(&npc_priv->xa_pf2idx_map[i]); 4807 4808 kfree(npc_priv->xa_pf2idx_map); 4809 /* No need to destroy mutex lock as it is 4810 * part of subbank structure 4811 */ 4812 kfree(npc_priv->sb); 4813 kfree(subbank_srch_order); 4814 bitmap_clear(npc_priv->en_map, 0, MAX_NUM_BANKS * MAX_NUM_SUB_BANKS * 4815 MAX_SUBBANK_DEPTH); 4816 npc_defrag_list_clear(); 4817 kfree(npc_priv); 4818 npc_priv = NULL; 4819 } 4820 4821 static int npc_setup_mcam_section(struct rvu *rvu, int key_type) 4822 { 4823 int blkaddr, sec; 4824 4825 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4826 if (blkaddr < 0) { 4827 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4828 return -ENODEV; 4829 } 4830 4831 for (sec = 0; sec < npc_priv->num_subbanks; sec++) 4832 rvu_write64(rvu, blkaddr, 4833 NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); 4834 4835 return 0; 4836 } 4837 4838 int npc_cn20k_init(struct rvu *rvu) 4839 { 4840 int err; 4841 4842 err = npc_priv_init(rvu); 4843 if (err) { 4844 dev_err(rvu->dev, "%s: Error to init\n", 4845 __func__); 4846 return err; 4847 } 4848 4849 err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); 4850 if (err) { 4851 dev_err(rvu->dev, "%s: mcam section configuration failure\n", 4852 __func__); 4853 goto fail; 4854 } 4855 4856 return 0; 4857 4858 fail: 4859 npc_cn20k_deinit(rvu); 4860 return err; 4861 } 4862