1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2026 Marvell. 5 * 6 */ 7 #include <linux/xarray.h> 8 #include <linux/bitfield.h> 9 10 #include "rvu.h" 11 #include "npc.h" 12 #include "npc_profile.h" 13 #include "rvu_npc_hash.h" 14 #include "rvu_npc.h" 15 #include "cn20k/npc.h" 16 #include "cn20k/reg.h" 17 #include "rvu_npc_fs.h" 18 19 static struct npc_priv_t *npc_priv; 20 21 static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { 22 [NPC_MCAM_KEY_DYN] = "DYNAMIC", 23 [NPC_MCAM_KEY_X2] = "X2", 24 [NPC_MCAM_KEY_X4] = "X4", 25 }; 26 27 static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { 28 [NPC_DFT_RULE_PROMISC_ID] = "Promisc", 29 [NPC_DFT_RULE_MCAST_ID] = "Mcast", 30 [NPC_DFT_RULE_BCAST_ID] = "Bcast", 31 [NPC_DFT_RULE_UCAST_ID] = "Ucast", 32 }; 33 34 #define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ 35 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 36 ((key_ofs) & 0x3F)) 37 38 #define NPC_DFT_RULE_ID_MK(pcifunc, id) \ 39 ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) 40 41 #define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ 42 FIELD_GET(GENMASK_ULL(15, 0), rid) 43 44 #define NPC_DFT_RULE_ID_2_ID(rid) \ 45 FIELD_GET(GENMASK_ULL(31, 16), rid) 46 47 #define NPC_DFT_RULE_PRIO 127 48 49 static const char cn20k_def_pfl_name[] = "default"; 50 51 static struct npc_mcam_kex_extr npc_mkex_extr_default = { 52 .mkex_sign = MKEX_CN20K_SIGN, 53 .name = "default", 54 .kpu_version = NPC_KPU_PROFILE_VER, 55 .keyx_cfg = { 56 /* nibble: LA..LE (ltype only) + Error code + Channel */ 57 [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | 58 NPC_CN20K_PARSE_NIBBLE_INTF_RX | 59 NPC_CN20K_PARSE_NIBBLE_ERRCODE, 60 61 /* nibble: LA..LE (ltype only) */ 62 [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | 63 NPC_CN20K_PARSE_NIBBLE_INTF_TX, 64 }, 65 .intf_extr_lid = { 66 /* Default RX MCAM KEX profile */ 67 [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 68 NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, 69 [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, 70 NPC_LID_LC, NPC_LID_LD }, 71 }, 72 .intf_extr_lt = { 73 /* Default RX MCAM KEX profile */ 74 [NIX_INTF_RX] = { 75 [0] = { 76 /* Layer A: Ethernet: */ 77 [NPC_LT_LA_ETHER] = 78 /* DMAC: 6 bytes, KW1[63:15] */ 79 KEX_EXTR_CFG(0x05, 0x0, 0x1, 80 NPC_KEXOF_DMAC + 1), 81 [NPC_LT_LA_CPT_HDR] = 82 /* DMAC: 6 bytes, KW1[63:15] */ 83 KEX_EXTR_CFG(0x05, 0x0, 0x1, 84 NPC_KEXOF_DMAC + 1), 85 }, 86 [1] = { 87 /* Layer A: Ethernet: */ 88 [NPC_LT_LA_ETHER] = 89 /* Ethertype: 2 bytes, KW0[63:48] */ 90 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 91 [NPC_LT_LA_CPT_HDR] = 92 /* Ethertype: 2 bytes, KW0[63:48] */ 93 KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), 94 }, 95 [2] = { 96 /* Layer B: Single VLAN (CTAG) */ 97 [NPC_LT_LB_CTAG] = 98 /* CTAG VLAN: 2 bytes, KW1[15:0] */ 99 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 100 /* Layer B: Stacked VLAN (STAG|QinQ) */ 101 [NPC_LT_LB_STAG_QINQ] = 102 /* Outer VLAN: 2 bytes, KW1[15:0] */ 103 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), 104 [NPC_LT_LB_FDSA] = 105 /* SWITCH PORT: 1 byte, KW1[7:0] */ 106 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), 107 }, 108 [3] = { 109 [NPC_LT_LB_CTAG] = 110 /* Ethertype: 2 bytes, KW0[63:48] */ 111 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 112 [NPC_LT_LB_STAG_QINQ] = 113 /* Ethertype: 2 bytes, KW0[63:48] */ 114 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), 115 [NPC_LT_LB_FDSA] = 116 /* Ethertype: 2 bytes, KW0[63:48] */ 117 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), 118 }, 119 [4] = { 120 /* Layer C: IPv4 */ 121 [NPC_LT_LC_IP] = 122 /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ 123 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), 124 /* Layer C: IPv6 */ 125 [NPC_LT_LC_IP6] = 126 /* Everything up to SADDR: 8 bytes, KW3[7:0], 127 * KW2[63:8] 128 */ 129 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), 130 }, 131 [5] = { 132 [NPC_LT_LC_IP] = 133 /* TOS: 1 byte, KW2[7:0] */ 134 KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), 135 }, 136 [6] = { 137 /* Layer D:UDP */ 138 [NPC_LT_LD_UDP] = 139 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 140 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 141 /* Layer D:TCP */ 142 [NPC_LT_LD_TCP] = 143 /* SPORT+DPORT: 4 bytes, KW3[39:8] */ 144 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), 145 }, 146 }, 147 /* Default TX MCAM KEX profile */ 148 [NIX_INTF_TX] = { 149 [0] = { 150 /* Layer A: NIX_INST_HDR_S + Ethernet */ 151 /* NIX appends 8 bytes of NIX_INST_HDR_S at the 152 * start of each TX packet supplied to NPC. 153 */ 154 [NPC_LT_LA_IH_NIX_ETHER] = 155 /* PF_FUNC: 2B , KW0 [47:32] */ 156 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 157 /* Layer A: HiGig2: */ 158 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 159 /* PF_FUNC: 2B , KW0 [47:32] */ 160 KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), 161 }, 162 [1] = { 163 [NPC_LT_LA_IH_NIX_ETHER] = 164 /* SQ_ID 3 bytes, KW1[63:16] */ 165 KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), 166 [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = 167 /* VID: 2 bytes, KW1[31:16] */ 168 KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), 169 }, 170 [2] = { 171 /* Layer B: Single VLAN (CTAG) */ 172 [NPC_LT_LB_CTAG] = 173 /* CTAG VLAN[2..3] KW0[63:48] */ 174 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 175 /* Layer B: Stacked VLAN (STAG|QinQ) */ 176 [NPC_LT_LB_STAG_QINQ] = 177 /* Outer VLAN: 2 bytes, KW0[63:48] */ 178 KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), 179 }, 180 [3] = { 181 [NPC_LT_LB_CTAG] = 182 /* CTAG VLAN[2..3] KW1[15:0] */ 183 KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), 184 [NPC_LT_LB_STAG_QINQ] = 185 /* Outer VLAN: 2 Bytes, KW1[15:0] */ 186 KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), 187 }, 188 [4] = { 189 /* Layer C: IPv4 */ 190 [NPC_LT_LC_IP] = 191 /* SIP+DIP: 8 bytes, KW2[63:0] */ 192 KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), 193 /* Layer C: IPv6 */ 194 [NPC_LT_LC_IP6] = 195 /* Everything up to SADDR: 8 bytes, KW2[63:0] */ 196 KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), 197 }, 198 [5] = { 199 /* Layer D:UDP */ 200 [NPC_LT_LD_UDP] = 201 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 202 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 203 /* Layer D:TCP */ 204 [NPC_LT_LD_TCP] = 205 /* SPORT+DPORT: 4 bytes, KW3[31:0] */ 206 KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), 207 }, 208 }, 209 }, 210 }; 211 212 struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) 213 { 214 return &npc_mkex_extr_default; 215 } 216 217 static u16 npc_idx2vidx(u16 idx) 218 { 219 unsigned long index; 220 void *map; 221 u16 vidx; 222 int val; 223 224 vidx = idx; 225 index = idx; 226 227 map = xa_load(&npc_priv->xa_idx2vidx_map, index); 228 if (!map) 229 goto done; 230 231 val = xa_to_value(map); 232 if (val == -1) 233 goto done; 234 235 vidx = val; 236 237 done: 238 return vidx; 239 } 240 241 static bool npc_is_vidx(u16 vidx) 242 { 243 return vidx >= npc_priv->bank_depth * 2; 244 } 245 246 static u16 npc_vidx2idx(u16 vidx) 247 { 248 unsigned long sentinel = (unsigned long)-1; 249 unsigned long index; 250 void *map; 251 int val; 252 u16 idx; 253 254 idx = vidx; 255 index = vidx; 256 257 map = xa_load(&npc_priv->xa_vidx2idx_map, index); 258 if (!map) 259 goto done; 260 261 val = xa_to_value(map); 262 if (val == sentinel) 263 goto done; 264 265 idx = val; 266 267 done: 268 return idx; 269 } 270 271 u16 npc_cn20k_vidx2idx(u16 idx) 272 { 273 if (!npc_priv) 274 return idx; 275 276 if (!npc_is_vidx(idx)) 277 return idx; 278 279 return npc_vidx2idx(idx); 280 } 281 282 u16 npc_cn20k_idx2vidx(u16 idx) 283 { 284 if (!npc_priv) 285 return idx; 286 287 if (npc_is_vidx(idx)) 288 return idx; 289 290 return npc_idx2vidx(idx); 291 } 292 293 static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) 294 { 295 u16 mcam_idx; 296 void *map; 297 298 if (!npc_is_vidx(vidx)) { 299 dev_err(rvu->dev, 300 "%s: vidx(%u) does not map to proper mcam idx\n", 301 __func__, vidx); 302 return -ESRCH; 303 } 304 305 mcam_idx = npc_vidx2idx(vidx); 306 307 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 308 if (!map) { 309 dev_err(rvu->dev, 310 "%s: vidx(%u) does not map to proper mcam idx\n", 311 __func__, vidx); 312 return -ESRCH; 313 } 314 315 map = xa_erase(&npc_priv->xa_idx2vidx_map, mcam_idx); 316 if (!map) { 317 dev_err(rvu->dev, 318 "%s: vidx(%u) is not valid\n", 319 __func__, vidx); 320 return -ESRCH; 321 } 322 323 if (old_midx) 324 *old_midx = mcam_idx; 325 326 return 0; 327 } 328 329 static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) 330 { 331 u16 old_midx; 332 void *map; 333 int rc; 334 335 if (!npc_is_vidx(vidx)) { 336 dev_err(rvu->dev, 337 "%s: vidx(%u) does not map to proper mcam idx\n", 338 __func__, vidx); 339 return -ESRCH; 340 } 341 342 map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx); 343 if (!map) { 344 dev_err(rvu->dev, 345 "%s: vidx(%u) could not be deleted from vidx2idx map\n", 346 __func__, vidx); 347 return -ESRCH; 348 } 349 350 old_midx = xa_to_value(map); 351 352 rc = xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 353 xa_mk_value(new_midx), GFP_KERNEL); 354 if (rc) { 355 dev_err(rvu->dev, 356 "%s: vidx(%u) cannot be added to vidx2idx map\n", 357 __func__, vidx); 358 goto fail1; 359 } 360 361 map = xa_erase(&npc_priv->xa_idx2vidx_map, old_midx); 362 if (!map) { 363 dev_err(rvu->dev, 364 "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 365 __func__, old_midx, vidx); 366 rc = -ESRCH; 367 goto fail2; 368 } 369 370 rc = xa_insert(&npc_priv->xa_idx2vidx_map, new_midx, 371 xa_mk_value(vidx), GFP_KERNEL); 372 if (rc) { 373 dev_err(rvu->dev, 374 "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", 375 __func__, new_midx, vidx); 376 goto fail3; 377 } 378 379 return 0; 380 381 fail3: 382 /* Restore vidx at old_midx location */ 383 if (xa_insert(&npc_priv->xa_idx2vidx_map, old_midx, 384 xa_mk_value(vidx), GFP_KERNEL)) 385 dev_err(rvu->dev, 386 "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", 387 __func__, old_midx, vidx); 388 fail2: 389 /* Erase new_midx inserted at vidx */ 390 if (!xa_erase(&npc_priv->xa_vidx2idx_map, vidx)) 391 dev_err(rvu->dev, 392 "%s: Failed to roll back vidx2idx vidx=%u\n", 393 __func__, vidx); 394 395 fail1: 396 /* Restore old_midx at vidx location */ 397 if (xa_insert(&npc_priv->xa_vidx2idx_map, vidx, 398 xa_mk_value(old_midx), GFP_KERNEL)) 399 dev_err(rvu->dev, 400 "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", 401 __func__, old_midx, vidx); 402 403 return rc; 404 } 405 406 static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, 407 u16 *vidx) 408 { 409 int rc, max, min; 410 u32 id; 411 412 /* Virtual index start from maximum mcam index + 1 */ 413 max = npc_priv->bank_depth * 2 * 2 - 1; 414 min = npc_priv->bank_depth * 2; 415 416 rc = xa_alloc(&npc_priv->xa_vidx2idx_map, &id, 417 xa_mk_value(mcam_idx), 418 XA_LIMIT(min, max), GFP_KERNEL); 419 if (rc) { 420 dev_err(rvu->dev, 421 "%s: Failed to add to vidx2idx map (%u)\n", 422 __func__, mcam_idx); 423 goto fail1; 424 } 425 426 rc = xa_insert(&npc_priv->xa_idx2vidx_map, mcam_idx, 427 xa_mk_value(id), GFP_KERNEL); 428 if (rc) { 429 dev_err(rvu->dev, 430 "%s: Failed to add to idx2vidx map (%u)\n", 431 __func__, mcam_idx); 432 goto fail2; 433 } 434 435 if (vidx) 436 *vidx = id; 437 438 return 0; 439 440 fail2: 441 xa_erase(&npc_priv->xa_vidx2idx_map, id); 442 fail1: 443 return rc; 444 } 445 446 static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, 447 const struct npc_kpu_profile_cam *kpucam, 448 int kpm, int entry) 449 { 450 struct npc_kpu_cam cam0 = {0}; 451 struct npc_kpu_cam cam1 = {0}; 452 453 cam1.state = kpucam->state & kpucam->state_mask; 454 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 455 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 456 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 457 458 cam0.state = ~kpucam->state & kpucam->state_mask; 459 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 460 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 461 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 462 463 rvu_write64(rvu, blkaddr, 464 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); 465 rvu_write64(rvu, blkaddr, 466 NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); 467 } 468 469 static void 470 npc_config_kpmaction(struct rvu *rvu, int blkaddr, 471 const struct npc_kpu_profile_action *kpuaction, 472 int kpm, int entry, bool pkind) 473 { 474 struct npc_kpm_action0 action0 = {0}; 475 struct npc_kpu_action1 action1 = {0}; 476 u64 reg; 477 478 action1.errlev = kpuaction->errlev; 479 action1.errcode = kpuaction->errcode; 480 action1.dp0_offset = kpuaction->dp0_offset; 481 action1.dp1_offset = kpuaction->dp1_offset; 482 action1.dp2_offset = kpuaction->dp2_offset; 483 484 if (pkind) 485 reg = NPC_AF_PKINDX_ACTION1(entry); 486 else 487 reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); 488 489 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 490 491 action0.byp_count = kpuaction->bypass_count & 0x7; 492 action0.capture_ena = kpuaction->cap_ena & 1; 493 action0.parse_done = kpuaction->parse_done & 1; 494 action0.next_state = kpuaction->next_state & 0xf; 495 action0.capture_lid = kpuaction->lid & 0x7; 496 497 /* Parser functionality will work correctly even though 498 * upper flag bits are silently discarded 499 */ 500 action0.capture_ltype = kpuaction->ltype & 0xf; 501 action0.capture_flags = kpuaction->flags & 0xf; 502 action0.ptr_advance = kpuaction->ptr_advance; 503 504 action0.var_len_offset = kpuaction->offset; 505 action0.var_len_mask = kpuaction->mask; 506 action0.var_len_right = kpuaction->right & 1; 507 action0.var_len_shift = kpuaction->shift & 1; 508 509 if (pkind) 510 reg = NPC_AF_PKINDX_ACTION0(entry); 511 else 512 reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); 513 514 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 515 } 516 517 static void 518 npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, 519 int kpm, int start_entry, 520 const struct npc_kpu_profile *profile) 521 { 522 int num_cam_entries, num_action_entries; 523 int entry, num_entries, max_entries; 524 u64 idx; 525 526 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile); 527 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile); 528 529 if (num_cam_entries != num_action_entries) { 530 dev_err(rvu->dev, 531 "kpm%d: CAM and action entries [%d != %d] not equal\n", 532 kpm, num_cam_entries, num_action_entries); 533 534 WARN(1, "Fatal error\n"); 535 return; 536 } 537 538 max_entries = rvu->hw->npc_kpu_entries / 2; 539 entry = start_entry; 540 /* Program CAM match entries for previous kpm extracted data */ 541 num_entries = min_t(int, num_cam_entries, max_entries); 542 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 543 npc_config_kpmcam(rvu, blkaddr, 544 npc_get_kpu_cam_nth_entry(rvu, profile, idx), 545 kpm, entry); 546 547 entry = start_entry; 548 /* Program this kpm's actions */ 549 num_entries = min_t(int, num_action_entries, max_entries); 550 for (idx = 0; entry < num_entries + start_entry; entry++, idx++) 551 npc_config_kpmaction(rvu, blkaddr, 552 npc_get_kpu_action_nth_entry(rvu, profile, idx), 553 kpm, entry, false); 554 } 555 556 static void 557 npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) 558 { 559 u64 entry_mask; 560 561 entry_mask = npc_enable_mask(num_entries); 562 /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ 563 if (!rvu->kpu.custom) 564 entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); 567 if (num_entries <= 64) { 568 /* Disable all the entries in W1, W2 and W3 */ 569 rvu_write64(rvu, blkaddr, 570 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), 571 npc_enable_mask(0)); 572 rvu_write64(rvu, blkaddr, 573 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 574 npc_enable_mask(0)); 575 rvu_write64(rvu, blkaddr, 576 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 577 npc_enable_mask(0)); 578 return; 579 } 580 581 num_entries = num_entries - 64; 582 entry_mask = npc_enable_mask(num_entries); 583 rvu_write64(rvu, blkaddr, 584 NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); 585 if (num_entries <= 64) { 586 /* Disable all the entries in W2 and W3 */ 587 rvu_write64(rvu, blkaddr, 588 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), 589 npc_enable_mask(0)); 590 rvu_write64(rvu, blkaddr, 591 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 592 npc_enable_mask(0)); 593 return; 594 } 595 596 num_entries = num_entries - 64; 597 entry_mask = npc_enable_mask(num_entries); 598 rvu_write64(rvu, blkaddr, 599 NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); 600 if (num_entries <= 64) { 601 /* Disable all the entries in W3 */ 602 rvu_write64(rvu, blkaddr, 603 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), 604 npc_enable_mask(0)); 605 return; 606 } 607 608 num_entries = num_entries - 64; 609 entry_mask = npc_enable_mask(num_entries); 610 rvu_write64(rvu, blkaddr, 611 NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); 612 } 613 614 #define KPU_OFFSET 8 615 static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) 616 { 617 const struct npc_kpu_profile *profile1, *profile2; 618 int pfl1_num_cam_entries, pfl2_num_cam_entries; 619 int idx, total_cam_entries; 620 621 for (idx = 0; idx < num_kpms; idx++) { 622 profile1 = &rvu->kpu.kpu[idx]; 623 pfl1_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile1); 624 npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); 625 profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; 626 pfl2_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile2); 627 628 npc_program_single_kpm_profile(rvu, blkaddr, idx, 629 pfl1_num_cam_entries, 630 profile2); 631 total_cam_entries = pfl1_num_cam_entries + pfl2_num_cam_entries; 632 npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); 633 rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), 634 pfl1_num_cam_entries); 635 /* Enable the KPUs associated with this KPM */ 636 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); 637 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), 638 0x01); 639 } 640 } 641 642 void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) 643 { 644 struct npc_kpu_profile_action *act; 645 struct rvu_hwinfo *hw = rvu->hw; 646 int num_pkinds, idx; 647 648 /* Disable all KPMs and their entries */ 649 for (idx = 0; idx < hw->npc_kpms; idx++) { 650 rvu_write64(rvu, blkaddr, 651 NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); 652 rvu_write64(rvu, blkaddr, 653 NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); 654 rvu_write64(rvu, blkaddr, 655 NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); 656 rvu_write64(rvu, blkaddr, 657 NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); 658 } 659 660 for (idx = 0; idx < hw->npc_kpus; idx++) 661 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 662 663 /* Load and customize KPU profile. */ 664 npc_load_kpu_profile(rvu); 665 666 /* Configure KPU and KPM mapping for second pass */ 667 rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); 668 669 /* First program IKPU profile i.e PKIND configs. 670 * Check HW max count to avoid configuring junk or 671 * writing to unsupported CSR addresses. 672 */ 673 num_pkinds = rvu->kpu.pkinds; 674 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 675 676 /* Cn20k does not support Custom profile from filesystem */ 677 for (idx = 0; idx < num_pkinds; idx++) { 678 act = npc_get_ikpu_nth_entry(rvu, idx); 679 if (!act) 680 continue; 681 682 npc_config_kpmaction(rvu, blkaddr, act, 683 0, idx, true); 684 } 685 686 /* Program KPM CAM and Action profiles */ 687 npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); 688 } 689 690 struct npc_priv_t *npc_priv_get(void) 691 { 692 return npc_priv; 693 } 694 695 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 696 const struct npc_mcam_kex_extr *mkex_extr, 697 u8 intf) 698 { 699 u8 num_extr = rvu->hw->npc_kex_extr; 700 int extr, lt; 701 u64 val; 702 703 if (is_npc_intf_tx(intf)) 704 return; 705 706 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 707 mkex_extr->keyx_cfg[NIX_INTF_RX]); 708 709 /* Program EXTRACTOR */ 710 for (extr = 0; extr < num_extr; extr++) 711 rvu_write64(rvu, blkaddr, 712 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 713 mkex_extr->intf_extr_lid[intf][extr]); 714 715 /* Program EXTRACTOR_LTYPE */ 716 for (extr = 0; extr < num_extr; extr++) { 717 for (lt = 0; lt < NPC_MAX_LT; lt++) { 718 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 719 CN20K_SET_EXTR_LT(intf, extr, lt, val); 720 } 721 } 722 } 723 724 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 725 const struct npc_mcam_kex_extr *mkex_extr, 726 u8 intf) 727 { 728 u8 num_extr = rvu->hw->npc_kex_extr; 729 int extr, lt; 730 u64 val; 731 732 if (is_npc_intf_rx(intf)) 733 return; 734 735 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 736 mkex_extr->keyx_cfg[NIX_INTF_TX]); 737 738 /* Program EXTRACTOR */ 739 for (extr = 0; extr < num_extr; extr++) 740 rvu_write64(rvu, blkaddr, 741 NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), 742 mkex_extr->intf_extr_lid[intf][extr]); 743 744 /* Program EXTRACTOR_LTYPE */ 745 for (extr = 0; extr < num_extr; extr++) { 746 for (lt = 0; lt < NPC_MAX_LT; lt++) { 747 val = mkex_extr->intf_extr_lt[intf][extr][lt]; 748 CN20K_SET_EXTR_LT(intf, extr, lt, val); 749 } 750 } 751 } 752 753 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 754 const struct npc_mcam_kex_extr *mkex_extr) 755 { 756 struct rvu_hwinfo *hw = rvu->hw; 757 u8 intf; 758 759 for (intf = 0; intf < hw->npc_intfs; intf++) { 760 npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); 761 npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); 762 } 763 764 /* Programme mkex hash profile */ 765 npc_program_mkex_hash(rvu, blkaddr); 766 } 767 768 void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, 769 const char *mkex_profile) 770 { 771 struct npc_mcam_kex_extr *mcam_kex_extr; 772 struct device *dev = &rvu->pdev->dev; 773 void __iomem *mkex_prfl_addr = NULL; 774 u64 prfl_sz; 775 int ret; 776 777 /* If user not selected mkex profile */ 778 if (rvu->kpu_fwdata_sz || 779 !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) 780 goto program_mkex_extr; 781 782 /* Setting up the mapping for mkex profile image */ 783 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 784 if (ret < 0) 785 goto program_mkex_extr; 786 787 mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; 788 789 while (((s64)prfl_sz > 0) && 790 (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { 791 /* Compare with mkex mod_param name string */ 792 if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && 793 !strncmp(mcam_kex_extr->name, mkex_profile, 794 MKEX_NAME_LEN)) { 795 rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; 796 goto program_mkex_extr; 797 } 798 799 mcam_kex_extr++; 800 prfl_sz -= sizeof(struct npc_mcam_kex_extr); 801 } 802 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 803 rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 804 805 program_mkex_extr: 806 dev_info(rvu->dev, "Using %s mkex profile\n", 807 rvu->kpu.mcam_kex_prfl.mkex_extr->name); 808 /* Program selected mkex profile */ 809 npc_program_mkex_profile(rvu, blkaddr, 810 rvu->kpu.mcam_kex_prfl.mkex_extr); 811 if (mkex_prfl_addr) 812 iounmap(mkex_prfl_addr); 813 } 814 815 int 816 npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, 817 int index, bool enable) 818 { 819 struct npc_mcam *mcam = &rvu->hw->mcam; 820 int mcam_idx = index % mcam->banksize; 821 int bank = index / mcam->banksize; 822 u64 cfg, hw_prio; 823 u8 kw_type; 824 825 if (index < 0 || index >= mcam->total_entries) 826 return -EINVAL; 827 828 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 829 return -EINVAL; 830 831 if (kw_type == NPC_MCAM_KEY_X2) { 832 cfg = rvu_read64(rvu, blkaddr, 833 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 834 bank)); 835 hw_prio = cfg & GENMASK_ULL(30, 24); 836 cfg = enable ? 1 : 0; 837 cfg |= hw_prio; 838 rvu_write64(rvu, blkaddr, 839 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 840 cfg); 841 goto update_en_map; 842 } 843 844 /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks 845 * need to be programmed with the same value. 846 */ 847 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 848 cfg = rvu_read64(rvu, blkaddr, 849 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, 850 bank)); 851 hw_prio = cfg & GENMASK_ULL(30, 24); 852 cfg = enable ? 1 : 0; 853 cfg |= hw_prio; 854 rvu_write64(rvu, blkaddr, 855 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 856 cfg); 857 } 858 859 update_en_map: 860 if (enable) 861 set_bit(index, npc_priv->en_map); 862 else 863 clear_bit(index, npc_priv->en_map); 864 865 return 0; 866 } 867 868 static void 869 npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index) 870 { 871 rvu_write64(rvu, blkaddr, 872 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 873 0); 874 rvu_write64(rvu, blkaddr, 875 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 876 0); 877 878 rvu_write64(rvu, blkaddr, 879 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); 880 rvu_write64(rvu, blkaddr, 881 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); 882 883 rvu_write64(rvu, blkaddr, 884 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); 885 rvu_write64(rvu, blkaddr, 886 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); 887 888 rvu_write64(rvu, blkaddr, 889 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); 890 rvu_write64(rvu, blkaddr, 891 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); 892 893 rvu_write64(rvu, blkaddr, 894 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); 895 rvu_write64(rvu, blkaddr, 896 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); 897 898 /* Clear corresponding stats register */ 899 rvu_write64(rvu, blkaddr, 900 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 901 } 902 903 int 904 npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx) 905 { 906 struct npc_mcam *mcam = &rvu->hw->mcam; 907 int bank = npc_get_bank(mcam, mcam_idx); 908 u8 kw_type; 909 int index; 910 911 if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type)) 912 return -EINVAL; 913 914 index = mcam_idx & (mcam->banksize - 1); 915 916 if (kw_type == NPC_MCAM_KEY_X2) { 917 npc_clear_x2_entry(rvu, blkaddr, bank, index); 918 return 0; 919 } 920 921 /* For NPC_MCAM_KEY_X4 keys, both the banks 922 * need to be programmed with the same value. 923 */ 924 for (bank = 0; bank < mcam->banks_per_entry; bank++) 925 npc_clear_x2_entry(rvu, blkaddr, bank, index); 926 927 return 0; 928 } 929 930 static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, 931 u64 *cam0, u64 *cam1) 932 { 933 u64 kw_mask; 934 935 /* The two banks of every MCAM entry are used as a single double-wide 936 * entry that is compared with the search key as follows: 937 * 938 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] 939 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] 940 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] 941 * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] 942 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] 943 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] 944 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] 945 * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] 946 */ 947 *cam1 = entry->kw[idx]; 948 kw_mask = entry->kw_mask[idx]; 949 *cam1 &= kw_mask; 950 *cam0 = ~*cam1 & kw_mask; 951 } 952 953 /*------------------------------------------------------------------------- 954 *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | 955 * in | | | | || | | 956 * profile| len | | | ||len | type | 957 *------------------------------------------------------------------------- 958 *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | 959 *------------------------------------------------------------------------| 960 *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 961 *------------------------------------------------------------------------| 962 *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 963 *------------------------------------------------------------------------| 964 *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | 965 *------------------------------------------------------------------------| 966 *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | 967 *------------------------------------------------------------------------| 968 *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 969 *------------------------------------------------------------------------| 970 *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | 971 *------------------------------------------------------------------------| 972 * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | 973 *------------------------------------------------------------------------| 974 */ 975 static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, 976 int blkaddr, int index, u8 intf, 977 struct cn20k_mcam_entry *entry, 978 int bank, u8 kw_type, int kw, 979 u8 req_kw_type) 980 { 981 u64 intf_ext = 0, intf_ext_mask = 0; 982 u8 tx_intf_mask = ~intf & 0x3; 983 u8 tx_intf = intf, kex_type; 984 u8 kw_type_mask = ~kw_type; 985 u64 cam0, cam1, kex_cfg; 986 987 if (is_npc_intf_tx(intf)) { 988 /* Last bit must be set and rest don't care 989 * for TX interfaces 990 */ 991 tx_intf_mask = 0x1; 992 tx_intf = intf & tx_intf_mask; 993 tx_intf_mask = ~tx_intf & tx_intf_mask; 994 } 995 996 kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 997 kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; 998 if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && 999 kw_type == NPC_MCAM_KEY_X2) { 1000 kw_type = 0; 1001 kw_type_mask = 0; 1002 } 1003 1004 /* Say, we need to write x2 keyword in an x4 subbank. 1005 * req_kw_type will be x2, and kw_type will be x4. 1006 * So in the case ignore kw bits in mcam. 1007 */ 1008 if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { 1009 kw_type = 0; 1010 kw_type_mask = 0; 1011 } 1012 1013 intf_ext = ((u64)kw_type << 16) | tx_intf; 1014 intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | 1015 tx_intf_mask; 1016 rvu_write64(rvu, blkaddr, 1017 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), 1018 intf_ext); 1019 rvu_write64(rvu, blkaddr, 1020 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), 1021 intf_ext_mask); 1022 1023 /* Set the match key */ 1024 npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); 1025 rvu_write64(rvu, blkaddr, 1026 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 1027 cam1); 1028 rvu_write64(rvu, blkaddr, 1029 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 1030 cam0); 1031 1032 npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); 1033 rvu_write64(rvu, blkaddr, 1034 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 1035 cam1); 1036 rvu_write64(rvu, blkaddr, 1037 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 1038 cam0); 1039 1040 npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); 1041 rvu_write64(rvu, blkaddr, 1042 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 1043 cam1); 1044 rvu_write64(rvu, blkaddr, 1045 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 1046 cam0); 1047 1048 npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); 1049 rvu_write64(rvu, blkaddr, 1050 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 1051 cam1); 1052 rvu_write64(rvu, blkaddr, 1053 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 1054 cam0); 1055 } 1056 1057 static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, 1058 int blkaddr, int index, u8 intf, 1059 struct cn20k_mcam_entry *entry, 1060 u8 kw_type, u8 req_kw_type) 1061 { 1062 int kw = 0, bank; 1063 1064 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) 1065 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1066 index, intf, 1067 entry, bank, kw_type, 1068 kw, req_kw_type); 1069 } 1070 1071 int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, 1072 u8 intf, struct cn20k_mcam_entry *entry, 1073 bool enable, u8 hw_prio, u8 req_kw_type) 1074 { 1075 struct npc_mcam *mcam = &rvu->hw->mcam; 1076 int mcam_idx = index % mcam->banksize; 1077 int bank = index / mcam->banksize; 1078 u64 bank_cfg = (u64)hw_prio << 24; 1079 int kw = 0; 1080 u8 kw_type; 1081 1082 if (index < 0 || index >= mcam->total_entries) 1083 return -EINVAL; 1084 1085 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1086 return -EINVAL; 1087 1088 /* Disable before mcam entry update */ 1089 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false)) 1090 return -EINVAL; 1091 1092 /* CAM1 takes the comparison value and 1093 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 1094 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 1095 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 1096 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 1097 */ 1098 if (kw_type == NPC_MCAM_KEY_X2) { 1099 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1100 npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx); 1101 npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, 1102 mcam_idx, intf, entry, 1103 bank, kw_type, kw, req_kw_type); 1104 /* Set 'action' */ 1105 rvu_write64(rvu, blkaddr, 1106 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1107 bank, 0), 1108 entry->action); 1109 1110 /* Set 'action2' for inline receive */ 1111 rvu_write64(rvu, blkaddr, 1112 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1113 bank, 2), 1114 entry->action2); 1115 1116 /* Set TAG 'action' */ 1117 rvu_write64(rvu, blkaddr, 1118 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1119 bank, 1), 1120 entry->vtag_action); 1121 1122 /* Set HW priority */ 1123 rvu_write64(rvu, blkaddr, 1124 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1125 bank_cfg); 1126 1127 } else { 1128 /* Clear mcam entry to avoid writes being suppressed by NPC */ 1129 npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx); 1130 npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx); 1131 1132 npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, 1133 mcam_idx, intf, entry, 1134 kw_type, req_kw_type); 1135 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1136 /* Set 'action' */ 1137 rvu_write64(rvu, blkaddr, 1138 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1139 bank, 0), 1140 entry->action); 1141 1142 /* Set TAG 'action' */ 1143 rvu_write64(rvu, blkaddr, 1144 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1145 bank, 1), 1146 entry->vtag_action); 1147 1148 /* Set 'action2' for inline receive */ 1149 rvu_write64(rvu, blkaddr, 1150 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, 1151 bank, 2), 1152 entry->action2); 1153 1154 /* Set HW priority */ 1155 rvu_write64(rvu, blkaddr, 1156 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), 1157 bank_cfg); 1158 } 1159 } 1160 1161 /* TODO: */ 1162 /* PF installing VF rule */ 1163 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 1164 return -EINVAL; 1165 1166 return 0; 1167 } 1168 1169 int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) 1170 { 1171 struct npc_mcam *mcam = &rvu->hw->mcam; 1172 u64 cfg, sreg, dreg, soff, doff; 1173 u8 src_kwtype, dest_kwtype; 1174 int bank, i, sb, db; 1175 int dbank, sbank; 1176 1177 if (src >= mcam->total_entries || dest >= mcam->total_entries) 1178 return -EINVAL; 1179 1180 dbank = npc_get_bank(mcam, dest); 1181 sbank = npc_get_bank(mcam, src); 1182 1183 if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype)) 1184 return -EINVAL; 1185 1186 if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype)) 1187 return -EINVAL; 1188 1189 if (src_kwtype != dest_kwtype) 1190 return -EINVAL; 1191 1192 src &= (mcam->banksize - 1); 1193 dest &= (mcam->banksize - 1); 1194 1195 /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ 1196 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 1197 sb = sbank + bank; 1198 sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); 1199 db = dbank + bank; 1200 dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); 1201 for (i = 0; i < 10; i++) { 1202 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 1203 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 1204 } 1205 1206 /* Copy action */ 1207 for (i = 0; i < 3; i++) { 1208 soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, 1209 sb, i); 1210 cfg = rvu_read64(rvu, blkaddr, soff); 1211 1212 doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, 1213 i); 1214 rvu_write64(rvu, blkaddr, doff, cfg); 1215 } 1216 1217 /* Copy bank configuration */ 1218 cfg = rvu_read64(rvu, blkaddr, 1219 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); 1220 rvu_write64(rvu, blkaddr, 1221 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); 1222 if (src_kwtype == NPC_MCAM_KEY_X2) 1223 break; 1224 } 1225 1226 return 0; 1227 } 1228 1229 static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, 1230 u64 cam0, u64 cam1) 1231 { 1232 entry->kw[idx] = cam1; 1233 entry->kw_mask[idx] = cam1 ^ cam0; 1234 } 1235 1236 int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, 1237 struct cn20k_mcam_entry *entry, 1238 u8 *intf, u8 *ena, u8 *hw_prio) 1239 { 1240 struct npc_mcam *mcam = &rvu->hw->mcam; 1241 u64 cam0, cam1, bank_cfg, cfg; 1242 int kw = 0, bank; 1243 u8 kw_type; 1244 1245 if (index >= mcam->total_entries) 1246 return -EINVAL; 1247 1248 if (npc_mcam_idx_2_key_type(rvu, index, &kw_type)) 1249 return -EINVAL; 1250 1251 bank = npc_get_bank(mcam, index); 1252 index &= (mcam->banksize - 1); 1253 1254 cfg = rvu_read64(rvu, blkaddr, 1255 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0)); 1256 entry->action = cfg; 1257 1258 cfg = rvu_read64(rvu, blkaddr, 1259 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1)); 1260 entry->vtag_action = cfg; 1261 1262 cfg = rvu_read64(rvu, blkaddr, 1263 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2)); 1264 entry->action2 = cfg; 1265 1266 cfg = rvu_read64(rvu, blkaddr, 1267 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, 1268 bank, 1)) & 3; 1269 *intf = cfg; 1270 1271 bank_cfg = rvu_read64(rvu, blkaddr, 1272 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); 1273 *ena = bank_cfg & 0x1; 1274 *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; 1275 if (kw_type == NPC_MCAM_KEY_X2) { 1276 cam1 = rvu_read64(rvu, blkaddr, 1277 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1278 bank, 1279 1)); 1280 cam0 = rvu_read64(rvu, blkaddr, 1281 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1282 bank, 1283 0)); 1284 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1285 1286 cam1 = rvu_read64(rvu, blkaddr, 1287 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1288 bank, 1289 1)); 1290 cam0 = rvu_read64(rvu, blkaddr, 1291 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1292 bank, 1293 0)); 1294 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1295 1296 cam1 = rvu_read64(rvu, blkaddr, 1297 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1298 bank, 1299 1)); 1300 cam0 = rvu_read64(rvu, blkaddr, 1301 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1302 bank, 1303 0)); 1304 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1305 1306 cam1 = rvu_read64(rvu, blkaddr, 1307 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1308 bank, 1309 1)); 1310 cam0 = rvu_read64(rvu, blkaddr, 1311 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1312 bank, 1313 0)); 1314 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1315 return 0; 1316 } 1317 1318 for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { 1319 cam1 = rvu_read64(rvu, blkaddr, 1320 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1321 bank, 1322 1)); 1323 cam0 = rvu_read64(rvu, blkaddr, 1324 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, 1325 bank, 1326 0)); 1327 npc_cn20k_fill_entryword(entry, kw, cam0, cam1); 1328 1329 cam1 = rvu_read64(rvu, blkaddr, 1330 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1331 bank, 1332 1)); 1333 cam0 = rvu_read64(rvu, blkaddr, 1334 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, 1335 bank, 1336 0)); 1337 npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); 1338 1339 cam1 = rvu_read64(rvu, blkaddr, 1340 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1341 bank, 1342 1)); 1343 cam0 = rvu_read64(rvu, blkaddr, 1344 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, 1345 bank, 1346 0)); 1347 npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); 1348 1349 cam1 = rvu_read64(rvu, blkaddr, 1350 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1351 bank, 1352 1)); 1353 cam0 = rvu_read64(rvu, blkaddr, 1354 NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, 1355 bank, 1356 0)); 1357 npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); 1358 } 1359 1360 return 0; 1361 } 1362 1363 int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, 1364 struct npc_cn20k_mcam_write_entry_req *req, 1365 struct msg_rsp *rsp) 1366 { 1367 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1368 struct npc_mcam *mcam = &rvu->hw->mcam; 1369 u16 pcifunc = req->hdr.pcifunc; 1370 int blkaddr, rc; 1371 u8 nix_intf; 1372 1373 req->entry = npc_cn20k_vidx2idx(req->entry); 1374 1375 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1376 if (blkaddr < 0) 1377 return NPC_MCAM_INVALID_REQ; 1378 1379 mutex_lock(&mcam->lock); 1380 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1381 if (rc) 1382 goto exit; 1383 1384 if (!is_npc_interface_valid(rvu, req->intf)) { 1385 rc = NPC_MCAM_INVALID_REQ; 1386 goto exit; 1387 } 1388 1389 if (is_npc_intf_tx(req->intf)) 1390 nix_intf = pfvf->nix_tx_intf; 1391 else 1392 nix_intf = pfvf->nix_rx_intf; 1393 1394 /* For AF installed rules, the nix_intf should be set to target NIX */ 1395 if (is_pffunc_af(req->hdr.pcifunc)) 1396 nix_intf = req->intf; 1397 1398 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, 1399 &req->entry_data, req->enable_entry, 1400 req->hw_prio, req->req_kw_type); 1401 1402 exit: 1403 mutex_unlock(&mcam->lock); 1404 return rc; 1405 } 1406 1407 int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, 1408 struct npc_mcam_read_entry_req *req, 1409 struct npc_cn20k_mcam_read_entry_rsp *rsp) 1410 { 1411 struct npc_mcam *mcam = &rvu->hw->mcam; 1412 u16 pcifunc = req->hdr.pcifunc; 1413 int blkaddr, rc; 1414 1415 req->entry = npc_cn20k_vidx2idx(req->entry); 1416 1417 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1418 if (blkaddr < 0) 1419 return NPC_MCAM_INVALID_REQ; 1420 1421 mutex_lock(&mcam->lock); 1422 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1423 if (rc) 1424 goto fail; 1425 1426 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, 1427 &rsp->entry_data, &rsp->intf, 1428 &rsp->enable, &rsp->hw_prio); 1429 fail: 1430 mutex_unlock(&mcam->lock); 1431 return rc; 1432 } 1433 1434 int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, 1435 struct npc_cn20k_mcam_alloc_and_write_entry_req *req, 1436 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 1437 { 1438 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1439 struct npc_mcam_free_entry_req free_req = { 0 }; 1440 struct npc_mcam_alloc_entry_req entry_req; 1441 struct npc_mcam_alloc_entry_rsp entry_rsp; 1442 struct npc_mcam *mcam = &rvu->hw->mcam; 1443 u16 entry = NPC_MCAM_ENTRY_INVALID; 1444 struct msg_rsp free_rsp; 1445 int blkaddr, rc, err; 1446 u8 nix_intf; 1447 1448 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1449 if (blkaddr < 0) 1450 return NPC_MCAM_INVALID_REQ; 1451 1452 if (!is_npc_interface_valid(rvu, req->intf)) 1453 return NPC_MCAM_INVALID_REQ; 1454 1455 /* Try to allocate a MCAM entry */ 1456 entry_req.hdr.pcifunc = req->hdr.pcifunc; 1457 entry_req.contig = true; 1458 entry_req.ref_prio = req->ref_prio; 1459 entry_req.ref_entry = req->ref_entry; 1460 entry_req.count = 1; 1461 entry_req.virt = req->virt; 1462 1463 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 1464 &entry_req, &entry_rsp); 1465 if (rc) 1466 return rc; 1467 1468 if (!entry_rsp.count) 1469 return NPC_MCAM_ALLOC_FAILED; 1470 1471 /* entry_req.count is 1, so single entry is allocated */ 1472 entry = npc_cn20k_vidx2idx(entry_rsp.entry); 1473 1474 mutex_lock(&mcam->lock); 1475 1476 if (is_npc_intf_tx(req->intf)) 1477 nix_intf = pfvf->nix_tx_intf; 1478 else 1479 nix_intf = pfvf->nix_rx_intf; 1480 1481 rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, 1482 &req->entry_data, req->enable_entry, 1483 req->hw_prio, req->req_kw_type); 1484 1485 mutex_unlock(&mcam->lock); 1486 1487 if (rc) { 1488 free_req.hdr.pcifunc = req->hdr.pcifunc; 1489 free_req.entry = entry_rsp.entry; 1490 err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 1491 if (err) 1492 dev_err(rvu->dev, 1493 "%s: Error to free mcam idx %u\n", 1494 __func__, entry_rsp.entry); 1495 return rc; 1496 } 1497 1498 rsp->entry = entry_rsp.entry; 1499 return 0; 1500 } 1501 1502 static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) 1503 { 1504 if (is_lbk_vf(rvu, pcifunc)) 1505 return NIXLF_PROMISC_ENTRY; 1506 1507 return NIXLF_UCAST_ENTRY; 1508 } 1509 1510 int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, 1511 struct msg_req *req, 1512 struct npc_cn20k_mcam_read_base_rule_rsp *rsp) 1513 { 1514 struct npc_mcam *mcam = &rvu->hw->mcam; 1515 int index, blkaddr, nixlf, rc = 0; 1516 u16 pcifunc = req->hdr.pcifunc; 1517 u8 intf, enable, hw_prio; 1518 struct rvu_pfvf *pfvf; 1519 int rl_type; 1520 1521 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1522 if (blkaddr < 0) 1523 return NPC_MCAM_INVALID_REQ; 1524 1525 /* Return the channel number in case of PF */ 1526 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 1527 pfvf = rvu_get_pfvf(rvu, pcifunc); 1528 rsp->entry.kw[0] = pfvf->rx_chan_base; 1529 rsp->entry.kw_mask[0] = 0xFFFULL; 1530 goto out; 1531 } 1532 1533 /* Find the pkt steering rule installed by PF to this VF */ 1534 mutex_lock(&mcam->lock); 1535 for (index = 0; index < mcam->bmap_entries; index++) { 1536 if (mcam->entry2target_pffunc[index] == pcifunc) 1537 goto read_entry; 1538 } 1539 1540 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 1541 if (rc < 0) { 1542 mutex_unlock(&mcam->lock); 1543 goto out; 1544 } 1545 1546 rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); 1547 1548 /* Read the default ucast entry if there is no pkt steering rule */ 1549 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); 1550 if (index < 0) { 1551 mutex_unlock(&mcam->lock); 1552 goto out; 1553 } 1554 1555 read_entry: 1556 /* Read the mcam entry */ 1557 rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index, 1558 &rsp->entry, &intf, 1559 &enable, &hw_prio); 1560 mutex_unlock(&mcam->lock); 1561 out: 1562 return rc; 1563 } 1564 1565 static u8 npc_map2cn20k_flag(u8 flag) 1566 { 1567 switch (flag) { 1568 case NPC_F_LC_U_IP_FRAG: 1569 return NPC_CN20K_F_LC_L_IP_FRAG; 1570 1571 case NPC_F_LC_U_IP6_FRAG: 1572 return NPC_CN20K_F_LC_L_IP6_FRAG; 1573 1574 case NPC_F_LC_L_6TO4: 1575 return NPC_CN20K_F_LC_L_6TO4; 1576 1577 case NPC_F_LC_L_MPLS_IN_IP: 1578 return NPC_CN20K_F_LC_U_MPLS_IN_IP; 1579 1580 case NPC_F_LC_L_IP6_TUN_IP6: 1581 return NPC_CN20K_F_LC_U_IP6_TUN_IP6; 1582 1583 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1584 return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; 1585 1586 default: 1587 break; 1588 } 1589 1590 WARN(1, "%s: Invalid flag=%u\n", __func__, flag); 1591 return 0xff; 1592 } 1593 1594 static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act) 1595 { 1596 u8 ltype, val; 1597 1598 if (act->lid != NPC_LID_LC) 1599 return; 1600 1601 ltype = act->ltype; 1602 if (ltype != NPC_LT_LC_IP && 1603 ltype != NPC_LT_LC_IP6 && 1604 ltype != NPC_LT_LC_IP_OPT && 1605 ltype != NPC_LT_LC_IP6_EXT) 1606 return; 1607 1608 switch (act->flags) { 1609 case NPC_F_LC_U_IP_FRAG: 1610 case NPC_F_LC_U_IP6_FRAG: 1611 case NPC_F_LC_L_6TO4: 1612 case NPC_F_LC_L_MPLS_IN_IP: 1613 case NPC_F_LC_L_IP6_TUN_IP6: 1614 case NPC_F_LC_L_IP6_MPLS_IN_IP: 1615 val = npc_map2cn20k_flag(act->flags); 1616 if (val != 0xFF) 1617 act->flags = val; 1618 break; 1619 default: 1620 break; 1621 } 1622 } 1623 1624 void 1625 npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, 1626 struct npc_kpu_profile_adapter *pfl) 1627 { 1628 struct npc_kpu_profile_action *action; 1629 int entries; 1630 1631 for (int i = 0; i < pfl->kpus; i++) { 1632 action = pfl->kpu[i].action; 1633 entries = pfl->kpu[i].action_entries; 1634 1635 for (int j = 0; j < entries; j++) 1636 npc_cn20k_translate_action_flags(&action[j]); 1637 } 1638 } 1639 1640 int npc_cn20k_apply_custom_kpu(struct rvu *rvu, 1641 struct npc_kpu_profile_adapter *profile) 1642 { 1643 const struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1644 size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1645 struct npc_kpu_profile_action *action; 1646 struct npc_kpu_profile_cam *cam; 1647 struct npc_kpu_fwdata *fw_kpu; 1648 size_t offset = 0; 1649 u16 kpu, entry; 1650 int entries; 1651 1652 hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); 1653 1654 if (rvu->kpu_fwdata_sz < hdr_sz) { 1655 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1656 return -EINVAL; 1657 } 1658 1659 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1660 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1661 fw->signature); 1662 return -EINVAL; 1663 } 1664 1665 /* Verify if the using known profile structure */ 1666 if (NPC_KPU_VER_MAJ(profile->version) > 1667 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1668 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1669 NPC_KPU_VER_MAJ(profile->version), 1670 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1671 return -EINVAL; 1672 } 1673 1674 /* Verify if profile is aligned with the required kernel changes */ 1675 if (NPC_KPU_VER_MIN(profile->version) < 1676 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1677 dev_warn(rvu->dev, 1678 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1679 NPC_KPU_VER_MAJ(profile->version), 1680 NPC_KPU_VER_MIN(profile->version), 1681 NPC_KPU_VER_PATCH(profile->version), 1682 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1683 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1684 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1685 return -EINVAL; 1686 } 1687 1688 /* Verify if profile fits the HW */ 1689 if (fw->kpus > rvu->hw->npc_kpus) { 1690 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus, 1691 rvu->hw->npc_kpus); 1692 return -EINVAL; 1693 } 1694 1695 /* Check if there is enough memory */ 1696 if (fw->kpus > profile->kpus) { 1697 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus, 1698 profile->kpus); 1699 return -EINVAL; 1700 } 1701 1702 profile->mcam_kex_prfl.mkex_extr = &fw->mkex; 1703 if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { 1704 dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", 1705 profile->mcam_kex_prfl.mkex_extr->mkex_sign); 1706 return -EINVAL; 1707 } 1708 1709 profile->custom = 1; 1710 profile->name = fw->name; 1711 profile->version = le64_to_cpu(fw->version); 1712 profile->lt_def = &fw->lt_def; 1713 1714 for (kpu = 0; kpu < fw->kpus; kpu++) { 1715 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1716 if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) 1717 dev_warn(rvu->dev, 1718 "Too many custom entries on KPU%d: %d > %d\n", 1719 kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1720 entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); 1721 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1722 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1723 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1724 offset += fw_kpu->entries * sizeof(*action); 1725 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1726 dev_warn(rvu->dev, 1727 "Profile size mismatch on KPU%i parsing.\n", 1728 kpu + 1); 1729 return -EINVAL; 1730 } 1731 1732 for (entry = 0; entry < entries; entry++) { 1733 profile->kpu[kpu].cam[entry] = cam[entry]; 1734 profile->kpu[kpu].action[entry] = action[entry]; 1735 npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]); 1736 } 1737 } 1738 1739 return 0; 1740 } 1741 1742 int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) 1743 { 1744 struct npc_subbank *sb; 1745 int bank_off, sb_id; 1746 1747 /* mcam_idx should be less than (2 * bank depth) */ 1748 if (mcam_idx >= npc_priv->bank_depth * 2) { 1749 dev_err(rvu->dev, "%s: bad params\n", 1750 __func__); 1751 return -EINVAL; 1752 } 1753 1754 /* find mcam offset per bank */ 1755 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1756 1757 /* Find subbank id */ 1758 sb_id = bank_off / npc_priv->subbank_depth; 1759 1760 /* Check if subbank id is more than maximum 1761 * number of subbanks available 1762 */ 1763 if (sb_id >= npc_priv->num_subbanks) { 1764 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1765 __func__, sb_id); 1766 return -EINVAL; 1767 } 1768 1769 sb = &npc_priv->sb[sb_id]; 1770 1771 *key_type = sb->key_type; 1772 1773 return 0; 1774 } 1775 1776 static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, 1777 u16 sub_off, u16 *mcam_idx) 1778 { 1779 int off, bot; 1780 1781 /* for x4 section, maximum allowed subbank index = 1782 * subsection depth - 1 1783 */ 1784 if (sb->key_type == NPC_MCAM_KEY_X4 && 1785 sub_off >= npc_priv->subbank_depth) { 1786 dev_err(rvu->dev, 1787 "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", 1788 __func__, sb->idx, sub_off); 1789 return -EINVAL; 1790 } 1791 1792 /* for x2 section, maximum allowed subbank index = 1793 * 2 * subsection depth - 1 1794 */ 1795 if (sb->key_type == NPC_MCAM_KEY_X2 && 1796 sub_off >= npc_priv->subbank_depth * 2) { 1797 dev_err(rvu->dev, 1798 "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", 1799 __func__, sb->idx, sub_off); 1800 return -EINVAL; 1801 } 1802 1803 /* Find subbank offset from respective subbank (w.r.t bank) */ 1804 off = sub_off & (npc_priv->subbank_depth - 1); 1805 1806 /* if subsection idx is in bank1, add bank depth, 1807 * which is part of sb->b1b 1808 */ 1809 bot = sub_off >= npc_priv->subbank_depth ? sb->b1b : sb->b0b; 1810 1811 *mcam_idx = bot + off; 1812 return 0; 1813 } 1814 1815 int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, 1816 struct npc_subbank **sb, 1817 int *sb_off) 1818 { 1819 int bank_off, sb_id; 1820 1821 /* mcam_idx should be less than (2 * bank depth) */ 1822 if (mcam_idx >= npc_priv->bank_depth * 2) { 1823 dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", 1824 __func__, mcam_idx); 1825 return -EINVAL; 1826 } 1827 1828 /* find mcam offset per bank */ 1829 bank_off = mcam_idx & (npc_priv->bank_depth - 1); 1830 1831 /* Find subbank id */ 1832 sb_id = bank_off / npc_priv->subbank_depth; 1833 1834 /* Check if subbank id is more than maximum 1835 * number of subbanks available 1836 */ 1837 if (sb_id >= npc_priv->num_subbanks) { 1838 dev_err(rvu->dev, "%s: invalid subbank %d\n", 1839 __func__, sb_id); 1840 return -EINVAL; 1841 } 1842 1843 *sb = &npc_priv->sb[sb_id]; 1844 1845 /* Subbank offset per bank */ 1846 *sb_off = bank_off % npc_priv->subbank_depth; 1847 1848 /* Index in a subbank should add subbank depth 1849 * if it is in bank1 1850 */ 1851 if (mcam_idx >= npc_priv->bank_depth) 1852 *sb_off += npc_priv->subbank_depth; 1853 1854 return 0; 1855 } 1856 1857 static int __npc_subbank_contig_alloc(struct rvu *rvu, 1858 struct npc_subbank *sb, 1859 int key_type, int sidx, 1860 int eidx, int prio, 1861 int count, int t, int b, 1862 unsigned long *bmap, 1863 u16 *save) 1864 { 1865 int k, offset, delta = 0; 1866 int cnt = 0, sbd; 1867 1868 sbd = npc_priv->subbank_depth; 1869 1870 if (sidx >= npc_priv->bank_depth) 1871 delta = sbd; 1872 1873 switch (prio) { 1874 case NPC_MCAM_LOWER_PRIO: 1875 case NPC_MCAM_ANY_PRIO: 1876 /* Find an area of size 'count' from sidx to eidx */ 1877 offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, 1878 count, 0); 1879 1880 if (offset >= sbd) { 1881 dev_err(rvu->dev, 1882 "%s: Could not find contiguous(%d) entries\n", 1883 __func__, count); 1884 return -EFAULT; 1885 } 1886 1887 dev_dbg(rvu->dev, 1888 "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", 1889 __func__, sidx, eidx, t, b, offset, 1890 count, delta); 1891 1892 for (cnt = 0; cnt < count; cnt++) 1893 save[cnt] = offset + cnt + delta; 1894 1895 break; 1896 1897 case NPC_MCAM_HIGHER_PRIO: 1898 /* Find an area of 'count' from eidx to sidx */ 1899 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1900 /* If an intermediate slot is not free, 1901 * reset the counter (cnt) to zero as 1902 * request is for contiguous. 1903 */ 1904 if (test_bit(k, bmap)) { 1905 cnt = 0; 1906 continue; 1907 } 1908 1909 save[cnt++] = k + delta; 1910 } 1911 break; 1912 } 1913 1914 /* Found 'count' number of free slots */ 1915 if (cnt == count) 1916 return 0; 1917 1918 dev_dbg(rvu->dev, 1919 "%s: Could not find contiguous(%d) entries in subbank=%u\n", 1920 __func__, count, sb->idx); 1921 return -EFAULT; 1922 } 1923 1924 static int __npc_subbank_non_contig_alloc(struct rvu *rvu, 1925 struct npc_subbank *sb, 1926 int key_type, int sidx, 1927 int eidx, int prio, 1928 int t, int b, 1929 unsigned long *bmap, 1930 int count, u16 *save, 1931 bool max_alloc, int *alloc_cnt) 1932 { 1933 unsigned long index; 1934 int cnt = 0, delta; 1935 int k, sbd; 1936 1937 sbd = npc_priv->subbank_depth; 1938 delta = sidx >= npc_priv->bank_depth ? sbd : 0; 1939 1940 switch (prio) { 1941 /* Find an area of size 'count' from sidx to eidx */ 1942 case NPC_MCAM_LOWER_PRIO: 1943 case NPC_MCAM_ANY_PRIO: 1944 index = find_next_zero_bit(bmap, sbd, sidx - b); 1945 if (index >= sbd) { 1946 dev_err(rvu->dev, 1947 "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", 1948 __func__, count, 1949 bitmap_weight(bmap, sbd), 1950 sb->idx); 1951 break; 1952 } 1953 1954 for (k = index; cnt < count && k <= (eidx - b); k++) { 1955 /* Skip used slots */ 1956 if (test_bit(k, bmap)) 1957 continue; 1958 1959 save[cnt++] = k + delta; 1960 } 1961 break; 1962 1963 /* Find an area of 'count' from eidx to sidx */ 1964 case NPC_MCAM_HIGHER_PRIO: 1965 for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { 1966 /* Skip used slots */ 1967 if (test_bit(k, bmap)) 1968 continue; 1969 1970 save[cnt++] = k + delta; 1971 } 1972 break; 1973 } 1974 1975 /* Update allocated 'cnt' to alloc_cnt */ 1976 *alloc_cnt = cnt; 1977 1978 /* Successfully allocated requested count slots */ 1979 if (cnt == count) 1980 return 0; 1981 1982 /* Allocation successful for cnt < count */ 1983 if (max_alloc && cnt > 0) 1984 return 0; 1985 1986 dev_dbg(rvu->dev, 1987 "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", 1988 __func__, count, sb->idx, cnt, max_alloc); 1989 1990 return -EFAULT; 1991 } 1992 1993 static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, 1994 int sb_off, unsigned long **bmap, 1995 int *off) 1996 { 1997 int sbd; 1998 1999 sbd = npc_priv->subbank_depth; 2000 2001 *off = sb_off & (sbd - 1); 2002 *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; 2003 } 2004 2005 /* set/clear bitmap */ 2006 static bool __npc_subbank_mark_slot(struct rvu *rvu, 2007 struct npc_subbank *sb, 2008 int sb_off, bool set) 2009 { 2010 unsigned long *bmap; 2011 int off; 2012 2013 /* if sb_off >= subbank.depth, then slots are in 2014 * bank1 2015 */ 2016 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2017 2018 dev_dbg(rvu->dev, 2019 "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", 2020 __func__, set, sb_off, sb->idx, off); 2021 2022 if (set) { 2023 /* Slot is already used */ 2024 if (test_bit(off, bmap)) 2025 return false; 2026 2027 sb->free_cnt--; 2028 set_bit(off, bmap); 2029 return true; 2030 } 2031 2032 /* Slot is already free */ 2033 if (!test_bit(off, bmap)) 2034 return false; 2035 2036 sb->free_cnt++; 2037 clear_bit(off, bmap); 2038 return true; 2039 } 2040 2041 static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) 2042 { 2043 int rc, blkaddr; 2044 2045 sb->flags = NPC_SUBBANK_FLAG_FREE; 2046 sb->key_type = 0; 2047 2048 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2049 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2050 2051 if (!xa_erase(&npc_priv->xa_sb_used, sb->arr_idx)) { 2052 dev_err(rvu->dev, 2053 "%s: Error to delete from xa_sb_used array\n", 2054 __func__); 2055 return -EFAULT; 2056 } 2057 2058 rc = xa_insert(&npc_priv->xa_sb_free, sb->arr_idx, 2059 xa_mk_value(sb->idx), GFP_KERNEL); 2060 if (rc) { 2061 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2062 xa_mk_value(sb->idx), GFP_KERNEL); 2063 if (rc) 2064 dev_err(rvu->dev, 2065 "%s: Failed to roll back sb(%u) arr_idx=%d\n", 2066 __func__, sb->idx, sb->arr_idx); 2067 2068 dev_err(rvu->dev, 2069 "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", 2070 __func__, sb->idx, sb->arr_idx); 2071 return rc; 2072 } 2073 2074 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2075 rvu_write64(rvu, blkaddr, 2076 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2077 NPC_MCAM_KEY_X2); 2078 2079 return rc; 2080 } 2081 2082 static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, 2083 int key_type) 2084 { 2085 int rc; 2086 2087 sb->flags = NPC_SUBBANK_FLAG_USED; 2088 sb->key_type = key_type; 2089 if (key_type == NPC_MCAM_KEY_X4) 2090 sb->free_cnt = npc_priv->subbank_depth; 2091 else 2092 sb->free_cnt = 2 * npc_priv->subbank_depth; 2093 2094 bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth); 2095 bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth); 2096 2097 if (!xa_erase(&npc_priv->xa_sb_free, sb->arr_idx)) { 2098 dev_err(rvu->dev, 2099 "%s: Error to delete from xa_sb_free array\n", 2100 __func__); 2101 return -EFAULT; 2102 } 2103 2104 rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx, 2105 xa_mk_value(sb->idx), GFP_KERNEL); 2106 if (rc) 2107 dev_err(rvu->dev, 2108 "%s: Error to add to xa_sb_used array\n", __func__); 2109 2110 return rc; 2111 } 2112 2113 static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, 2114 u16 sb_off) 2115 { 2116 bool deleted = false; 2117 unsigned long *bmap; 2118 int rc, off; 2119 2120 deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); 2121 if (!deleted) 2122 goto done; 2123 2124 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2125 2126 /* Check whether we can mark whole subbank as free */ 2127 if (sb->key_type == NPC_MCAM_KEY_X4) { 2128 if (sb->free_cnt < npc_priv->subbank_depth) 2129 goto done; 2130 } else { 2131 if (sb->free_cnt < 2 * npc_priv->subbank_depth) 2132 goto done; 2133 } 2134 2135 /* All slots in subbank are unused. Mark the subbank as free 2136 * and add to free pool 2137 */ 2138 rc = __npc_subbank_mark_free(rvu, sb); 2139 if (rc) 2140 dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); 2141 2142 done: 2143 return deleted; 2144 } 2145 2146 static int 2147 npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) 2148 { 2149 bool deleted; 2150 2151 mutex_lock(&sb->lock); 2152 deleted = __npc_subbank_free(rvu, sb, sb_off); 2153 mutex_unlock(&sb->lock); 2154 2155 return deleted ? 0 : -EFAULT; 2156 } 2157 2158 static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2159 int key_type, int ref, int limit, int prio, 2160 bool contig, int count, u16 *mcam_idx, 2161 int idx_sz, bool max_alloc, int *alloc_cnt) 2162 { 2163 int cnt, t, b, i, blkaddr; 2164 bool new_sub_bank = false; 2165 unsigned long *bmap; 2166 u16 *save = NULL; 2167 int sidx, eidx; 2168 bool diffbank; 2169 int bw, bfree; 2170 int rc = 0; 2171 bool ret; 2172 2173 /* Check if enough space is there to return requested number of 2174 * mcam indexes in case of contiguous allocation 2175 */ 2176 if (!max_alloc && count > idx_sz) { 2177 dev_err(rvu->dev, 2178 "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", 2179 __func__, count, idx_sz, sb->idx); 2180 return -ENOSPC; 2181 } 2182 2183 /* Allocation on multiple subbank is not supported by this function. 2184 * it means that ref and limit should be on same subbank. 2185 * 2186 * ref and limit values should be validated w.r.t prio as below. 2187 * say ref = 100, limit = 200, 2188 * if NPC_MCAM_LOWER_PRIO, allocate index 100 2189 * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. 2190 * if NPC_MCAM_ANY_PRIO, allocate index 100 2191 * 2192 * say ref = 200, limit = 100 2193 * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. 2194 * if NPC_MCAM_HIGHER_PRIO, allocate index 200 2195 * if NPC_MCAM_ANY_PRIO, allocate index 100 2196 * 2197 * Please note that NPC_MCAM_ANY_PRIO does not have any restriction 2198 * on "ref" and "limit" values. ie, ref > limit and limit > ref 2199 * are valid cases. 2200 */ 2201 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 2202 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 2203 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 2204 __func__, ref, limit); 2205 return -EINVAL; 2206 } 2207 2208 /* x4 indexes are from 0 to bank size as it combines two x2 banks */ 2209 if (key_type == NPC_MCAM_KEY_X4 && 2210 (ref >= npc_priv->bank_depth || limit >= npc_priv->bank_depth)) { 2211 dev_err(rvu->dev, 2212 "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", 2213 __func__, ref, limit); 2214 return -EINVAL; 2215 } 2216 2217 /* This function is called either bank0 or bank1 portion of a subbank. 2218 * so ref and limit should be on same bank. 2219 */ 2220 diffbank = !!((ref & npc_priv->bank_depth) ^ 2221 (limit & npc_priv->bank_depth)); 2222 if (diffbank) { 2223 dev_err(rvu->dev, 2224 "%s: request ref and limit should be from same bank\n", 2225 __func__); 2226 return -EINVAL; 2227 } 2228 2229 sidx = min_t(int, limit, ref); 2230 eidx = max_t(int, limit, ref); 2231 2232 /* Find total number of slots available; both used and free */ 2233 cnt = eidx - sidx + 1; 2234 if (contig && cnt < count) { 2235 dev_err(rvu->dev, 2236 "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", 2237 __func__, ref, limit, count); 2238 return -EINVAL; 2239 } 2240 2241 /* If subbank is free, check if requested number of indexes is less than 2242 * or equal to mcam entries available in the subbank if contig. 2243 */ 2244 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2245 if (contig && count > npc_priv->subbank_depth) { 2246 dev_err(rvu->dev, "%s: Less number of entries\n", 2247 __func__); 2248 return -ENOSPC; 2249 } 2250 2251 new_sub_bank = true; 2252 goto process; 2253 } 2254 2255 /* Flag should be set for all used subbanks */ 2256 WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), 2257 "Used flag is not set(%#x)\n", sb->flags); 2258 2259 /* If subbank key type does not match with requested key_type, 2260 * return error 2261 */ 2262 if (sb->key_type != key_type) { 2263 dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); 2264 return -EINVAL; 2265 } 2266 2267 process: 2268 /* if ref or limit >= npc_priv->bank_depth, index are in bank1. 2269 * else bank0. 2270 */ 2271 if (ref >= npc_priv->bank_depth) { 2272 bmap = sb->b1map; 2273 t = sb->b1t; 2274 b = sb->b1b; 2275 } else { 2276 bmap = sb->b0map; 2277 t = sb->b0t; 2278 b = sb->b0b; 2279 } 2280 2281 /* Calculate free slots */ 2282 bw = bitmap_weight(bmap, npc_priv->subbank_depth); 2283 bfree = npc_priv->subbank_depth - bw; 2284 2285 if (!bfree) { 2286 dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); 2287 return -ENOSPC; 2288 } 2289 2290 /* If request is for contiguous , then max we can allocate is 2291 * equal to subbank_depth 2292 */ 2293 if (contig && bfree < count) { 2294 dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); 2295 return -ENOSPC; 2296 } 2297 2298 /* 'save' array stores available indexes temporarily before 2299 * marking it as allocated 2300 */ 2301 save = kcalloc(count, sizeof(u16), GFP_KERNEL); 2302 if (!save) { 2303 rc = -ENOMEM; 2304 goto err1; 2305 } 2306 2307 if (contig) { 2308 rc = __npc_subbank_contig_alloc(rvu, sb, key_type, 2309 sidx, eidx, prio, 2310 count, t, b, 2311 bmap, save); 2312 /* contiguous allocation success means that 2313 * requested number of free slots got 2314 * allocated 2315 */ 2316 if (!rc) 2317 *alloc_cnt = count; 2318 2319 } else { 2320 rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, 2321 sidx, eidx, prio, 2322 t, b, bmap, 2323 count, save, 2324 max_alloc, alloc_cnt); 2325 } 2326 2327 if (rc) 2328 goto err1; 2329 2330 /* Mark new subbank bank as used */ 2331 if (new_sub_bank) { 2332 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2333 if (blkaddr < 0) { 2334 dev_err(rvu->dev, 2335 "%s: NPC block not implemented\n", __func__); 2336 rc = -EFAULT; 2337 goto err1; 2338 } 2339 2340 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2341 if (rc) { 2342 dev_err(rvu->dev, 2343 "%s: Error to mark subbank as used\n", 2344 __func__); 2345 goto err2; 2346 } 2347 2348 /* Configure section type to key_type */ 2349 rvu_write64(rvu, blkaddr, 2350 NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), 2351 key_type); 2352 } 2353 2354 for (i = 0; i < *alloc_cnt; i++) { 2355 rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], 2356 &mcam_idx[i]); 2357 if (rc) { 2358 dev_err(rvu->dev, 2359 "%s: Error to find mcam idx for %u\n", 2360 __func__, save[i]); 2361 /* TODO: handle err case gracefully */ 2362 goto err3; 2363 } 2364 2365 /* Mark all slots as used */ 2366 ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); 2367 if (!ret) { 2368 dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", 2369 __func__, mcam_idx[i]); 2370 rc = -EFAULT; 2371 goto err3; 2372 } 2373 } 2374 kfree(save); 2375 return 0; 2376 2377 err3: 2378 for (int j = 0; j < i; j++) 2379 __npc_subbank_mark_slot(rvu, sb, save[j], false); 2380 err2: 2381 if (new_sub_bank) 2382 __npc_subbank_mark_free(rvu, sb); 2383 err1: 2384 kfree(save); 2385 *alloc_cnt = 0; 2386 return rc; 2387 } 2388 2389 static int 2390 npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, 2391 int key_type, int ref, int limit, int prio, 2392 bool contig, int count, u16 *mcam_idx, 2393 int idx_sz, bool max_alloc, int *alloc_cnt) 2394 { 2395 int rc; 2396 2397 mutex_lock(&sb->lock); 2398 rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, 2399 contig, count, mcam_idx, idx_sz, 2400 max_alloc, alloc_cnt); 2401 mutex_unlock(&sb->lock); 2402 2403 return rc; 2404 } 2405 2406 static int 2407 npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) 2408 { 2409 int pcifunc, idx; 2410 void *map; 2411 2412 map = xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2413 if (!map) { 2414 dev_err(rvu->dev, 2415 "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", 2416 __func__, mcam_idx); 2417 return -EFAULT; 2418 } 2419 2420 pcifunc = xa_to_value(map); 2421 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2422 if (!map) { 2423 dev_err(rvu->dev, 2424 "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", 2425 __func__, pcifunc, mcam_idx); 2426 return -ESRCH; 2427 } 2428 2429 idx = xa_to_value(map); 2430 2431 map = xa_erase(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2432 if (!map) { 2433 dev_err(rvu->dev, 2434 "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", 2435 __func__, mcam_idx); 2436 return -EFAULT; 2437 } 2438 return 0; 2439 } 2440 2441 static int 2442 npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) 2443 { 2444 int rc, idx; 2445 void *map; 2446 2447 dev_dbg(rvu->dev, 2448 "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", 2449 __func__, mcam_idx, pcifunc); 2450 2451 rc = xa_insert(&npc_priv->xa_idx2pf_map, mcam_idx, 2452 xa_mk_value(pcifunc), GFP_KERNEL); 2453 2454 if (rc) { 2455 map = xa_load(&npc_priv->xa_idx2pf_map, mcam_idx); 2456 dev_err(rvu->dev, 2457 "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", 2458 __func__, mcam_idx, xa_to_value(map)); 2459 return -EFAULT; 2460 } 2461 2462 map = xa_load(&npc_priv->xa_pf_map, pcifunc); 2463 if (!map) { 2464 dev_err(rvu->dev, 2465 "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", 2466 __func__, pcifunc, mcam_idx); 2467 return -ESRCH; 2468 } 2469 2470 idx = xa_to_value(map); 2471 2472 rc = xa_insert(&npc_priv->xa_pf2idx_map[idx], mcam_idx, 2473 xa_mk_value(pcifunc), GFP_KERNEL); 2474 2475 if (rc) { 2476 map = xa_load(&npc_priv->xa_pf2idx_map[idx], mcam_idx); 2477 xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx); 2478 dev_err(rvu->dev, 2479 "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", 2480 __func__, mcam_idx, xa_to_value(map), idx); 2481 2482 return -EFAULT; 2483 } 2484 2485 return 0; 2486 } 2487 2488 static bool 2489 npc_subbank_suits(struct npc_subbank *sb, int key_type) 2490 { 2491 mutex_lock(&sb->lock); 2492 2493 if (!sb->key_type) { 2494 mutex_unlock(&sb->lock); 2495 return true; 2496 } 2497 2498 if (sb->key_type == key_type) { 2499 mutex_unlock(&sb->lock); 2500 return true; 2501 } 2502 2503 mutex_unlock(&sb->lock); 2504 return false; 2505 } 2506 2507 #define SB_ALIGN_UP(val) (((val) + npc_priv->subbank_depth) & \ 2508 ~((npc_priv->subbank_depth) - 1)) 2509 #define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv->subbank_depth) 2510 2511 static void npc_subbank_iter_down(struct rvu *rvu, 2512 int ref, int limit, 2513 int *cur_ref, int *cur_limit, 2514 bool *start, bool *stop) 2515 { 2516 int align; 2517 2518 *stop = false; 2519 2520 /* ALIGN_DOWN the limit to current subbank boundary bottom index */ 2521 if (*start) { 2522 *start = false; 2523 *cur_ref = ref; 2524 align = SB_ALIGN_DOWN(ref); 2525 if (align < limit) { 2526 *stop = true; 2527 *cur_limit = limit; 2528 return; 2529 } 2530 *cur_limit = align; 2531 return; 2532 } 2533 2534 *cur_ref = *cur_limit - 1; 2535 align = *cur_ref - npc_priv->subbank_depth + 1; 2536 if (align <= limit) { 2537 *stop = true; 2538 *cur_limit = limit; 2539 return; 2540 } 2541 2542 *cur_limit = align; 2543 } 2544 2545 static void npc_subbank_iter_up(struct rvu *rvu, 2546 int ref, int limit, 2547 int *cur_ref, int *cur_limit, 2548 bool *start, bool *stop) 2549 { 2550 int align; 2551 2552 *stop = false; 2553 2554 /* ALIGN_UP the limit to current subbank boundary top index */ 2555 if (*start) { 2556 *start = false; 2557 *cur_ref = ref; 2558 2559 /* Find next lower prio subbank's bottom index */ 2560 align = SB_ALIGN_UP(ref); 2561 2562 /* Crosses limit ? */ 2563 if (align - 1 > limit) { 2564 *stop = true; 2565 *cur_limit = limit; 2566 return; 2567 } 2568 2569 /* Current subbank's top index */ 2570 *cur_limit = align - 1; 2571 return; 2572 } 2573 2574 *cur_ref = *cur_limit + 1; 2575 align = *cur_ref + npc_priv->subbank_depth - 1; 2576 2577 if (align >= limit) { 2578 *stop = true; 2579 *cur_limit = limit; 2580 return; 2581 } 2582 2583 *cur_limit = align; 2584 } 2585 2586 static int 2587 npc_subbank_iter(struct rvu *rvu, int key_type, 2588 int ref, int limit, int prio, 2589 int *cur_ref, int *cur_limit, 2590 bool *start, bool *stop) 2591 { 2592 if (prio != NPC_MCAM_HIGHER_PRIO) 2593 npc_subbank_iter_up(rvu, ref, limit, 2594 cur_ref, cur_limit, 2595 start, stop); 2596 else 2597 npc_subbank_iter_down(rvu, ref, limit, 2598 cur_ref, cur_limit, 2599 start, stop); 2600 2601 /* limit and ref should < bank_depth for x4 */ 2602 if (key_type == NPC_MCAM_KEY_X4) { 2603 if (*cur_ref >= npc_priv->bank_depth) 2604 return -EINVAL; 2605 2606 if (*cur_limit >= npc_priv->bank_depth) 2607 return -EINVAL; 2608 } 2609 /* limit and ref should < 2 * bank_depth, for x2 */ 2610 if (*cur_ref >= 2 * npc_priv->bank_depth) 2611 return -EINVAL; 2612 2613 if (*cur_limit >= 2 * npc_priv->bank_depth) 2614 return -EINVAL; 2615 2616 return 0; 2617 } 2618 2619 static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, 2620 bool maps_del) 2621 { 2622 struct npc_subbank *sb; 2623 u16 vidx, midx; 2624 int sb_off, i; 2625 bool ret; 2626 int rc; 2627 2628 /* Check if we can dealloc indexes properly ? */ 2629 for (i = 0; i < count; i++) { 2630 rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), 2631 &sb, &sb_off); 2632 if (rc) { 2633 dev_err(rvu->dev, 2634 "Failed to free mcam idx=%u\n", mcam_idx[i]); 2635 return rc; 2636 } 2637 } 2638 2639 for (i = 0; i < count; i++) { 2640 if (npc_is_vidx(mcam_idx[i])) { 2641 vidx = mcam_idx[i]; 2642 midx = npc_vidx2idx(vidx); 2643 } else { 2644 midx = mcam_idx[i]; 2645 vidx = npc_idx2vidx(midx); 2646 } 2647 2648 if (midx >= npc_priv->bank_depth * npc_priv->num_banks) { 2649 dev_err(rvu->dev, 2650 "%s: Invalid mcam_idx=%u cannot be deleted\n", 2651 __func__, mcam_idx[i]); 2652 return -EINVAL; 2653 } 2654 2655 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 2656 &sb, &sb_off); 2657 if (rc) { 2658 dev_err(rvu->dev, 2659 "%s: Failed to find subbank info for vidx=%u\n", 2660 __func__, vidx); 2661 return rc; 2662 } 2663 2664 ret = npc_subbank_free(rvu, sb, sb_off); 2665 if (ret) { 2666 dev_err(rvu->dev, 2667 "%s: Failed to find subbank info for vidx=%u\n", 2668 __func__, vidx); 2669 return -EINVAL; 2670 } 2671 2672 if (!maps_del) 2673 continue; 2674 2675 rc = npc_del_from_pf_maps(rvu, midx); 2676 if (rc) 2677 return rc; 2678 2679 /* If there is no vidx mapping; continue */ 2680 if (vidx == midx) 2681 continue; 2682 2683 rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); 2684 if (rc) 2685 return rc; 2686 } 2687 2688 return 0; 2689 } 2690 2691 static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, 2692 int ref, int limit, int prio, 2693 bool contig, int count, 2694 u16 *mcam_idx) 2695 { 2696 struct npc_subbank *sb; 2697 unsigned long *bmap; 2698 int sb_off, off, rc; 2699 int cnt = 0; 2700 bool bitset; 2701 2702 if (prio != NPC_MCAM_HIGHER_PRIO) { 2703 while (ref <= limit) { 2704 /* Calculate subbank and subbank index */ 2705 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2706 &sb, &sb_off); 2707 if (rc) 2708 goto err; 2709 2710 /* If subbank is not suitable for requested key type 2711 * restart search from next subbank 2712 */ 2713 if (!npc_subbank_suits(sb, key_type)) { 2714 ref = SB_ALIGN_UP(ref); 2715 if (contig) { 2716 rc = npc_idx_free(rvu, mcam_idx, 2717 cnt, false); 2718 if (rc) 2719 return rc; 2720 cnt = 0; 2721 } 2722 continue; 2723 } 2724 2725 mutex_lock(&sb->lock); 2726 2727 /* If subbank is free; mark it as used */ 2728 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2729 rc = __npc_subbank_mark_used(rvu, sb, 2730 key_type); 2731 if (rc) { 2732 mutex_unlock(&sb->lock); 2733 dev_err(rvu->dev, 2734 "%s:Error to add to use array\n", 2735 __func__); 2736 goto err; 2737 } 2738 } 2739 2740 /* Find correct bmap */ 2741 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2742 2743 /* if bit is already set, reset 'cnt' */ 2744 bitset = test_bit(off, bmap); 2745 if (bitset) { 2746 mutex_unlock(&sb->lock); 2747 if (contig) { 2748 rc = npc_idx_free(rvu, mcam_idx, 2749 cnt, false); 2750 if (rc) 2751 return rc; 2752 cnt = 0; 2753 } 2754 2755 ref++; 2756 continue; 2757 } 2758 2759 set_bit(off, bmap); 2760 sb->free_cnt--; 2761 mcam_idx[cnt++] = ref; 2762 mutex_unlock(&sb->lock); 2763 2764 if (cnt == count) 2765 return 0; 2766 ref++; 2767 } 2768 2769 /* Could not allocate request count slots */ 2770 goto err; 2771 } 2772 while (ref >= limit) { 2773 rc = npc_mcam_idx_2_subbank_idx(rvu, ref, 2774 &sb, &sb_off); 2775 if (rc) 2776 goto err; 2777 2778 if (!npc_subbank_suits(sb, key_type)) { 2779 ref = SB_ALIGN_DOWN(ref) - 1; 2780 if (contig) { 2781 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2782 if (rc) 2783 return rc; 2784 2785 cnt = 0; 2786 } 2787 continue; 2788 } 2789 2790 mutex_lock(&sb->lock); 2791 2792 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 2793 rc = __npc_subbank_mark_used(rvu, sb, key_type); 2794 if (rc) { 2795 mutex_unlock(&sb->lock); 2796 dev_err(rvu->dev, 2797 "%s:Error to add to use array\n", 2798 __func__); 2799 goto err; 2800 } 2801 } 2802 2803 __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); 2804 bitset = test_bit(off, bmap); 2805 if (bitset) { 2806 mutex_unlock(&sb->lock); 2807 if (contig) { 2808 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2809 if (rc) 2810 return rc; 2811 2812 cnt = 0; 2813 } 2814 ref--; 2815 continue; 2816 } 2817 2818 mcam_idx[cnt++] = ref; 2819 sb->free_cnt--; 2820 set_bit(off, bmap); 2821 mutex_unlock(&sb->lock); 2822 2823 if (cnt == count) 2824 return 0; 2825 ref--; 2826 } 2827 2828 err: 2829 rc = npc_idx_free(rvu, mcam_idx, cnt, false); 2830 if (rc) 2831 dev_err(rvu->dev, 2832 "%s: Error happened while freeing cnt=%u indexes\n", 2833 __func__, cnt); 2834 2835 return -ENOSPC; 2836 } 2837 2838 static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, 2839 int key_type) 2840 { 2841 int cnt, spd; 2842 2843 spd = npc_priv->subbank_depth; 2844 mutex_lock(&sb->lock); 2845 2846 if (sb->flags & NPC_SUBBANK_FLAG_FREE) 2847 cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; 2848 else 2849 cnt = sb->free_cnt; 2850 2851 mutex_unlock(&sb->lock); 2852 return cnt; 2853 } 2854 2855 static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, 2856 int ref, int limit, int prio, 2857 bool contig, int count, 2858 u16 *mcam_idx) 2859 { 2860 struct npc_subbank *sb1, *sb2; 2861 bool max_alloc, start, stop; 2862 int r, l, sb_idx1, sb_idx2; 2863 int tot = 0, rc; 2864 int alloc_cnt; 2865 2866 max_alloc = !contig; 2867 2868 start = true; 2869 stop = false; 2870 2871 /* Loop until we cross the ref/limit boundary */ 2872 while (!stop) { 2873 rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, 2874 &r, &l, &start, &stop); 2875 2876 dev_dbg(rvu->dev, 2877 "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", 2878 __func__, ref, limit, r, l, 2879 start, stop, tot, count, rc); 2880 2881 if (rc) 2882 goto err; 2883 2884 /* Find subbank and subbank index for ref */ 2885 rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, 2886 &sb_idx1); 2887 if (rc) 2888 goto err; 2889 2890 dev_dbg(rvu->dev, 2891 "%s: ref subbank=%d off=%d\n", 2892 __func__, sb1->idx, sb_idx1); 2893 2894 /* Skip subbank if it is not available for the keytype */ 2895 if (!npc_subbank_suits(sb1, key_type)) { 2896 dev_dbg(rvu->dev, 2897 "%s: not suitable sb=%d key_type=%d\n", 2898 __func__, sb1->idx, key_type); 2899 continue; 2900 } 2901 2902 /* Find subbank and subbank index for limit */ 2903 rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, 2904 &sb_idx2); 2905 if (rc) 2906 goto err; 2907 2908 dev_dbg(rvu->dev, 2909 "%s: limit subbank=%d off=%d\n", 2910 __func__, sb_idx1, sb_idx2); 2911 2912 /* subbank of ref and limit should be same */ 2913 if (sb1 != sb2) { 2914 dev_err(rvu->dev, 2915 "%s: l(%d) and r(%d) are not in same subbank\n", 2916 __func__, r, l); 2917 goto err; 2918 } 2919 2920 if (contig && 2921 npc_subbank_free_cnt(rvu, sb1, key_type) < count) { 2922 dev_dbg(rvu->dev, "%s: less count =%d\n", 2923 __func__, 2924 npc_subbank_free_cnt(rvu, sb1, key_type)); 2925 continue; 2926 } 2927 2928 /* Try in one bank of a subbank */ 2929 alloc_cnt = 0; 2930 rc = npc_subbank_alloc(rvu, sb1, key_type, 2931 r, l, prio, contig, 2932 count - tot, mcam_idx + tot, 2933 count - tot, max_alloc, 2934 &alloc_cnt); 2935 2936 tot += alloc_cnt; 2937 2938 dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", 2939 __func__, tot, alloc_cnt); 2940 2941 if (!rc && count == tot) 2942 return 0; 2943 } 2944 err: 2945 dev_dbg(rvu->dev, "%s: Error to allocate\n", 2946 __func__); 2947 2948 /* non contiguous allocation fails. We need to do clean up */ 2949 if (max_alloc) { 2950 rc = npc_idx_free(rvu, mcam_idx, tot, false); 2951 if (rc) 2952 dev_err(rvu->dev, 2953 "%s: failed to free %u indexes\n", 2954 __func__, tot); 2955 } 2956 2957 return -EFAULT; 2958 } 2959 2960 /* Minimize allocation from bottom and top subbanks for noref allocations. 2961 * Default allocations are ref based, and will be allocated from top 2962 * subbanks (least priority subbanks). Since default allocation is at very 2963 * early stage of kernel netdev probes, this subbanks will be moved to 2964 * used subbanks list. This will pave a way for noref allocation from these 2965 * used subbanks. Skip allocation for these top and bottom, and try free 2966 * bank next. If none slot is available, come back and search in these 2967 * subbanks. 2968 */ 2969 2970 static int npc_subbank_restricted_idxs[2]; 2971 static bool restrict_valid = true; 2972 2973 static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) 2974 { 2975 int i; 2976 2977 if (!restrict_valid) 2978 return false; 2979 2980 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { 2981 if (index == npc_subbank_restricted_idxs[i]) 2982 return true; 2983 } 2984 2985 return false; 2986 } 2987 2988 static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, 2989 int count, u16 *mcam_idx) 2990 { 2991 struct npc_subbank *sb; 2992 unsigned long index; 2993 int tot = 0, rc; 2994 bool max_alloc; 2995 int alloc_cnt; 2996 int idx, i; 2997 void *val; 2998 2999 max_alloc = !contig; 3000 3001 /* Check used subbanks for free slots */ 3002 xa_for_each(&npc_priv->xa_sb_used, index, val) { 3003 idx = xa_to_value(val); 3004 3005 /* Minimize allocation from restricted subbanks 3006 * in noref allocations. 3007 */ 3008 if (npc_subbank_restrict_usage(rvu, idx)) 3009 continue; 3010 3011 sb = &npc_priv->sb[idx]; 3012 3013 /* Skip if not suitable subbank */ 3014 if (!npc_subbank_suits(sb, key_type)) 3015 continue; 3016 3017 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3018 continue; 3019 3020 /* try in bank 0. Try passing ref and limit equal to 3021 * subbank boundaries 3022 */ 3023 alloc_cnt = 0; 3024 rc = npc_subbank_alloc(rvu, sb, key_type, 3025 sb->b0b, sb->b0t, 0, 3026 contig, count - tot, 3027 mcam_idx + tot, 3028 count - tot, 3029 max_alloc, &alloc_cnt); 3030 3031 /* Non contiguous allocation may allocate less than 3032 * requested 'count'. 3033 */ 3034 tot += alloc_cnt; 3035 3036 dev_dbg(rvu->dev, 3037 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3038 __func__, alloc_cnt, sb->idx, tot, count); 3039 3040 /* Successfully allocated */ 3041 if (!rc && count == tot) 3042 return 0; 3043 3044 /* x4 entries can be allocated from bank 0 only */ 3045 if (key_type == NPC_MCAM_KEY_X4) 3046 continue; 3047 3048 /* try in bank 1 for x2 */ 3049 alloc_cnt = 0; 3050 rc = npc_subbank_alloc(rvu, sb, key_type, 3051 sb->b1b, sb->b1t, 0, 3052 contig, count - tot, 3053 mcam_idx + tot, 3054 count - tot, max_alloc, 3055 &alloc_cnt); 3056 3057 tot += alloc_cnt; 3058 3059 dev_dbg(rvu->dev, 3060 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3061 __func__, alloc_cnt, sb->idx, tot, count); 3062 3063 if (!rc && count == tot) 3064 return 0; 3065 } 3066 3067 /* Allocate in free subbanks */ 3068 xa_for_each(&npc_priv->xa_sb_free, index, val) { 3069 idx = xa_to_value(val); 3070 sb = &npc_priv->sb[idx]; 3071 3072 /* Minimize allocation from restricted subbanks 3073 * in noref allocations. 3074 */ 3075 if (npc_subbank_restrict_usage(rvu, idx)) 3076 continue; 3077 3078 if (!npc_subbank_suits(sb, key_type)) 3079 continue; 3080 3081 /* try in bank 0 */ 3082 alloc_cnt = 0; 3083 rc = npc_subbank_alloc(rvu, sb, key_type, 3084 sb->b0b, sb->b0t, 0, 3085 contig, count - tot, 3086 mcam_idx + tot, 3087 count - tot, 3088 max_alloc, &alloc_cnt); 3089 3090 tot += alloc_cnt; 3091 3092 dev_dbg(rvu->dev, 3093 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3094 __func__, alloc_cnt, sb->idx, tot, count); 3095 3096 /* Successfully allocated */ 3097 if (!rc && count == tot) 3098 return 0; 3099 3100 /* x4 entries can be allocated from bank 0 only */ 3101 if (key_type == NPC_MCAM_KEY_X4) 3102 continue; 3103 3104 /* try in bank 1 for x2 */ 3105 alloc_cnt = 0; 3106 rc = npc_subbank_alloc(rvu, sb, 3107 key_type, sb->b1b, sb->b1t, 0, 3108 contig, count - tot, 3109 mcam_idx + tot, count - tot, 3110 max_alloc, &alloc_cnt); 3111 3112 tot += alloc_cnt; 3113 3114 dev_dbg(rvu->dev, 3115 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3116 __func__, alloc_cnt, sb->idx, tot, count); 3117 3118 if (!rc && count == tot) 3119 return 0; 3120 } 3121 3122 /* Allocate from restricted subbanks */ 3123 for (i = 0; restrict_valid && 3124 (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { 3125 idx = npc_subbank_restricted_idxs[i]; 3126 sb = &npc_priv->sb[idx]; 3127 3128 /* Skip if not suitable subbank */ 3129 if (!npc_subbank_suits(sb, key_type)) 3130 continue; 3131 3132 if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) 3133 continue; 3134 3135 /* try in bank 0. Try passing ref and limit equal to 3136 * subbank boundaries 3137 */ 3138 alloc_cnt = 0; 3139 rc = npc_subbank_alloc(rvu, sb, key_type, 3140 sb->b0b, sb->b0t, 0, 3141 contig, count - tot, 3142 mcam_idx + tot, 3143 count - tot, 3144 max_alloc, &alloc_cnt); 3145 3146 /* Non contiguous allocation may allocate less than 3147 * requested 'count'. 3148 */ 3149 tot += alloc_cnt; 3150 3151 dev_dbg(rvu->dev, 3152 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3153 __func__, alloc_cnt, sb->idx, tot, count); 3154 3155 /* Successfully allocated */ 3156 if (!rc && count == tot) 3157 return 0; 3158 3159 /* x4 entries can be allocated from bank 0 only */ 3160 if (key_type == NPC_MCAM_KEY_X4) 3161 continue; 3162 3163 /* try in bank 1 for x2 */ 3164 alloc_cnt = 0; 3165 rc = npc_subbank_alloc(rvu, sb, key_type, 3166 sb->b1b, sb->b1t, 0, 3167 contig, count - tot, 3168 mcam_idx + tot, 3169 count - tot, max_alloc, 3170 &alloc_cnt); 3171 3172 tot += alloc_cnt; 3173 3174 dev_dbg(rvu->dev, 3175 "%s: Allocated %d from subbank %d, tot=%d count=%d\n", 3176 __func__, alloc_cnt, sb->idx, tot, count); 3177 3178 if (!rc && count == tot) 3179 return 0; 3180 } 3181 3182 /* non contiguous allocation fails. We need to do clean up */ 3183 if (max_alloc) 3184 npc_idx_free(rvu, mcam_idx, tot, false); 3185 3186 dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", 3187 __func__); 3188 3189 return -EFAULT; 3190 } 3191 3192 int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) 3193 { 3194 return npc_idx_free(rvu, mcam_idx, count, true); 3195 } 3196 3197 int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, 3198 int prio, u16 *mcam_idx, int ref, int limit, 3199 bool contig, int count, bool virt) 3200 { 3201 bool defrag_candidate = false; 3202 int i, eidx, rc, bd; 3203 bool ref_valid; 3204 u16 vidx; 3205 3206 bd = npc_priv->bank_depth; 3207 3208 /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 3209 * Here user wants to allocate 0th entry 3210 */ 3211 if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && 3212 count == 1) { 3213 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3214 prio, contig, count, mcam_idx); 3215 3216 if (rc) 3217 return rc; 3218 goto add2map; 3219 } 3220 3221 ref_valid = !!(limit || ref); 3222 defrag_candidate = !ref_valid && !contig && virt; 3223 if (!ref_valid) { 3224 if (contig && count > npc_priv->subbank_depth) 3225 goto try_noref_multi_subbank; 3226 3227 rc = npc_subbank_noref_alloc(rvu, key_type, contig, 3228 count, mcam_idx); 3229 if (!rc) 3230 goto add2map; 3231 3232 try_noref_multi_subbank: 3233 eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; 3234 3235 if (prio == NPC_MCAM_HIGHER_PRIO) 3236 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3237 eidx, 0, 3238 NPC_MCAM_HIGHER_PRIO, 3239 contig, count, 3240 mcam_idx); 3241 else 3242 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3243 0, eidx, 3244 NPC_MCAM_LOWER_PRIO, 3245 contig, count, 3246 mcam_idx); 3247 3248 if (!rc) 3249 goto add2map; 3250 3251 return rc; 3252 } 3253 3254 if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || 3255 (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { 3256 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3257 __func__, ref, limit); 3258 return -EINVAL; 3259 } 3260 3261 if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || 3262 (key_type == NPC_MCAM_KEY_X2 && 3263 (ref >= 2 * bd || limit >= 2 * bd))) { 3264 dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", 3265 __func__, ref, limit); 3266 return -EINVAL; 3267 } 3268 3269 if (contig && count > npc_priv->subbank_depth) 3270 goto try_ref_multi_subbank; 3271 3272 rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, 3273 prio, contig, count, mcam_idx); 3274 if (!rc) 3275 goto add2map; 3276 3277 try_ref_multi_subbank: 3278 rc = npc_multi_subbank_ref_alloc(rvu, key_type, 3279 ref, limit, prio, 3280 contig, count, mcam_idx); 3281 if (!rc) 3282 goto add2map; 3283 3284 return rc; 3285 3286 add2map: 3287 for (i = 0; i < count; i++) { 3288 rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); 3289 if (rc) 3290 goto err; 3291 3292 if (!defrag_candidate) 3293 continue; 3294 3295 rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); 3296 if (rc) { 3297 npc_del_from_pf_maps(rvu, mcam_idx[i]); 3298 goto err; 3299 } 3300 3301 /* Return vidx to caller */ 3302 mcam_idx[i] = vidx; 3303 } 3304 3305 return 0; 3306 err: 3307 for (int j = 0; j < i; j++) { 3308 npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); 3309 3310 if (!defrag_candidate) 3311 continue; 3312 3313 npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); 3314 } 3315 3316 return rc; 3317 3318 } 3319 3320 void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, 3321 int *x4_free, int *sb_free) 3322 { 3323 struct npc_subbank *sb; 3324 int i; 3325 3326 /* Reset all stats to zero */ 3327 *x2_free = 0; 3328 *x4_free = 0; 3329 *sb_free = 0; 3330 3331 for (i = 0; i < npc_priv->num_subbanks; i++) { 3332 sb = &npc_priv->sb[i]; 3333 mutex_lock(&sb->lock); 3334 3335 /* Count number of free subbanks */ 3336 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 3337 (*sb_free)++; 3338 goto next; 3339 } 3340 3341 /* Sumup x4 free count */ 3342 if (sb->key_type == NPC_MCAM_KEY_X4) { 3343 (*x4_free) += sb->free_cnt; 3344 goto next; 3345 } 3346 3347 /* Sumup x2 free counts */ 3348 (*x2_free) += sb->free_cnt; 3349 next: 3350 mutex_unlock(&sb->lock); 3351 } 3352 } 3353 3354 int 3355 rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, 3356 struct msg_req *req, 3357 struct npc_cn20k_get_fcnt_rsp *rsp) 3358 { 3359 npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, 3360 &rsp->free_x4, &rsp->free_subbanks); 3361 return 0; 3362 } 3363 3364 int 3365 rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, 3366 struct msg_req *req, 3367 struct npc_cn20k_get_kex_cfg_rsp *rsp) 3368 { 3369 int extr, lt; 3370 3371 rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); 3372 rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); 3373 3374 /* Get EXTRACTOR LID */ 3375 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3376 rsp->intf_extr_lid[NIX_INTF_RX][extr] = 3377 CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); 3378 rsp->intf_extr_lid[NIX_INTF_TX][extr] = 3379 CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); 3380 } 3381 3382 /* Get EXTRACTOR LTYPE */ 3383 for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { 3384 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3385 rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = 3386 CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); 3387 rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = 3388 CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); 3389 } 3390 } 3391 3392 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3393 return 0; 3394 } 3395 3396 static u32 *subbank_srch_order; 3397 3398 static void npc_populate_restricted_idxs(int num_subbanks) 3399 { 3400 npc_subbank_restricted_idxs[0] = num_subbanks - 1; 3401 npc_subbank_restricted_idxs[1] = 0; 3402 } 3403 3404 static int npc_create_srch_order(int cnt) 3405 { 3406 int val = 0; 3407 3408 subbank_srch_order = kcalloc(cnt, sizeof(u32), 3409 GFP_KERNEL); 3410 if (!subbank_srch_order) 3411 return -ENOMEM; 3412 3413 /* cnt(subbank depth) is always a power of 2. There is a check in 3414 * npc_priv_init() to check the same. 3415 */ 3416 for (int i = 0; i < cnt; i += 2) { 3417 subbank_srch_order[i] = cnt / 2 - val - 1; 3418 subbank_srch_order[i + 1] = cnt / 2 + 1 + val; 3419 val++; 3420 } 3421 3422 subbank_srch_order[cnt - 1] = cnt / 2; 3423 return 0; 3424 } 3425 3426 static int npc_subbanks_srch_order_init(struct rvu *rvu) 3427 { 3428 struct npc_subbank *sb; 3429 int sb_idx; 3430 int i, j; 3431 int rc; 3432 3433 for (i = 0; i < npc_priv->num_subbanks; i++) { 3434 sb_idx = subbank_srch_order[i]; 3435 sb = &npc_priv->sb[sb_idx]; 3436 sb->arr_idx = i; 3437 3438 dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", 3439 __func__, sb->idx, sb->arr_idx); 3440 3441 rc = xa_err(xa_store(&npc_priv->xa_sb_free, sb->arr_idx, 3442 xa_mk_value(sb->idx), GFP_KERNEL)); 3443 if (rc) { 3444 dev_err(rvu->dev, 3445 "%s: xa_store(xa_sb_free) failed at slot %d (sb=%d): %d\n", 3446 __func__, i, sb_idx, rc); 3447 for (j = 0; j < i; j++) 3448 xa_erase(&npc_priv->xa_sb_free, j); 3449 return rc; 3450 } 3451 } 3452 3453 return 0; 3454 } 3455 3456 static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) 3457 { 3458 mutex_init(&sb->lock); 3459 3460 sb->b0b = idx * npc_priv->subbank_depth; 3461 sb->b0t = sb->b0b + npc_priv->subbank_depth - 1; 3462 3463 sb->b1b = npc_priv->bank_depth + idx * npc_priv->subbank_depth; 3464 sb->b1t = sb->b1b + npc_priv->subbank_depth - 1; 3465 3466 sb->flags = NPC_SUBBANK_FLAG_FREE; 3467 sb->idx = idx; 3468 } 3469 3470 static int npc_pcifunc_map_create(struct rvu *rvu) 3471 { 3472 int pf, vf, numvfs; 3473 int cnt = 0; 3474 u16 pcifunc; 3475 u64 cfg; 3476 3477 for (pf = 0; pf < rvu->hw->total_pfs; pf++) { 3478 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3479 numvfs = (cfg >> 12) & 0xFF; 3480 3481 /* Skip not enabled PFs */ 3482 if (!(cfg & BIT_ULL(20))) 3483 goto chk_vfs; 3484 3485 /* If Admin function, check on VFs */ 3486 if (cfg & BIT_ULL(21)) 3487 goto chk_vfs; 3488 3489 pcifunc = pf << 9; 3490 3491 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3492 xa_mk_value(cnt), GFP_KERNEL); 3493 3494 cnt++; 3495 3496 chk_vfs: 3497 for (vf = 0; vf < numvfs; vf++) { 3498 pcifunc = (pf << 9) | (vf + 1); 3499 3500 xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc, 3501 xa_mk_value(cnt), GFP_KERNEL); 3502 cnt++; 3503 } 3504 } 3505 3506 return cnt; 3507 } 3508 3509 struct npc_defrag_node { 3510 u8 idx; 3511 u8 key_type; 3512 bool valid; 3513 bool refs; 3514 u16 free_cnt; 3515 u16 vidx_cnt; 3516 u16 *vidx; 3517 struct list_head list; 3518 }; 3519 3520 static bool npc_defrag_skip_restricted_sb(int sb_id) 3521 { 3522 int i; 3523 3524 if (!restrict_valid) 3525 return false; 3526 3527 for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) 3528 if (sb_id == npc_subbank_restricted_idxs[i]) 3529 return true; 3530 return false; 3531 } 3532 3533 /* Find subbank with minimum number of virtual indexes */ 3534 static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) 3535 { 3536 struct npc_defrag_node *node, *tnode = NULL; 3537 int min = INT_MAX; 3538 3539 list_for_each_entry(node, lh, list) { 3540 if (!node->valid) 3541 continue; 3542 3543 /* if subbank has ref allocated mcam indexes, that subbank 3544 * is not a good candidate to move out indexes. 3545 */ 3546 if (node->refs) 3547 continue; 3548 3549 if (min > node->vidx_cnt) { 3550 min = node->vidx_cnt; 3551 tnode = node; 3552 } 3553 } 3554 3555 return tnode; 3556 } 3557 3558 /* Find subbank with maximum number of free spaces */ 3559 static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) 3560 { 3561 struct npc_defrag_node *node, *tnode = NULL; 3562 int max = INT_MIN; 3563 3564 list_for_each_entry(node, lh, list) { 3565 if (!node->valid) 3566 continue; 3567 3568 if (max < node->free_cnt) { 3569 max = node->free_cnt; 3570 tnode = node; 3571 } 3572 } 3573 3574 return tnode; 3575 } 3576 3577 static int npc_defrag_alloc_free_slots(struct rvu *rvu, 3578 struct npc_defrag_node *f, 3579 int cnt, u16 *save) 3580 { 3581 int alloc_cnt1, alloc_cnt2; 3582 struct npc_subbank *sb; 3583 int rc, sb_off, i, err; 3584 bool deleted; 3585 3586 sb = &npc_priv->sb[f->idx]; 3587 3588 alloc_cnt1 = 0; 3589 alloc_cnt2 = 0; 3590 3591 rc = __npc_subbank_alloc(rvu, sb, 3592 f->key_type, sb->b0b, 3593 sb->b0t, 3594 NPC_MCAM_LOWER_PRIO, 3595 false, cnt, save, cnt, true, 3596 &alloc_cnt1); 3597 3598 /* X4 entries only occupy bank 0 (b0b..b0t); see npc_subbank_idx_2_mcam_idx(). 3599 * X2 uses both halves of the subbank, so spill into bank 1 if needed. 3600 */ 3601 if (alloc_cnt1 < cnt && f->key_type == NPC_MCAM_KEY_X2) { 3602 rc = __npc_subbank_alloc(rvu, sb, 3603 f->key_type, sb->b1b, 3604 sb->b1t, 3605 NPC_MCAM_LOWER_PRIO, 3606 false, cnt - alloc_cnt1, 3607 save + alloc_cnt1, 3608 cnt - alloc_cnt1, 3609 true, &alloc_cnt2); 3610 } 3611 3612 if (alloc_cnt1 + alloc_cnt2 != cnt) { 3613 dev_err(rvu->dev, 3614 "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", 3615 __func__, cnt, alloc_cnt1, alloc_cnt2); 3616 rc = -ENOSPC; 3617 goto fail_free_alloc; 3618 } 3619 3620 return 0; 3621 3622 fail_free_alloc: 3623 for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { 3624 err = npc_mcam_idx_2_subbank_idx(rvu, save[i], 3625 &sb, &sb_off); 3626 if (err) { 3627 dev_err(rvu->dev, 3628 "%s: Error to find subbank for mcam idx=%u\n", 3629 __func__, save[i]); 3630 break; 3631 } 3632 3633 deleted = __npc_subbank_free(rvu, sb, sb_off); 3634 if (!deleted) { 3635 dev_err(rvu->dev, 3636 "%s: Error to free mcam idx=%u\n", 3637 __func__, save[i]); 3638 break; 3639 } 3640 } 3641 3642 return rc; 3643 } 3644 3645 static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, 3646 u16 new_midx, u16 vidx) 3647 { 3648 struct npc_defrag_show_node *node; 3649 3650 node = kcalloc(1, sizeof(*node), GFP_KERNEL); 3651 if (!node) 3652 return -ENOMEM; 3653 3654 node->old_midx = old_midx; 3655 node->new_midx = new_midx; 3656 node->vidx = vidx; 3657 INIT_LIST_HEAD(&node->list); 3658 3659 mutex_lock(&npc_priv->lock); 3660 list_add_tail(&node->list, &npc_priv->defrag_lh); 3661 mutex_unlock(&npc_priv->lock); 3662 3663 return 0; 3664 } 3665 3666 static 3667 int npc_defrag_move_vdx_to_free(struct rvu *rvu, 3668 struct npc_defrag_node *f, 3669 struct npc_defrag_node *v, 3670 int cnt, u16 *save) 3671 { 3672 u16 new_midx, old_midx, vidx, target_pf; 3673 struct npc_mcam *mcam = &rvu->hw->mcam; 3674 struct rvu_npc_mcam_rule *rule, *tmp; 3675 int i, vidx_cnt, rc, sb_off; 3676 struct npc_subbank *sb; 3677 bool deleted; 3678 u16 pcifunc; 3679 int blkaddr; 3680 void *map; 3681 u8 bank; 3682 u16 midx; 3683 u64 stats; 3684 3685 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3686 3687 vidx_cnt = v->vidx_cnt; 3688 for (i = 0; i < cnt; i++) { 3689 vidx = v->vidx[vidx_cnt - i - 1]; 3690 old_midx = npc_vidx2idx(vidx); 3691 new_midx = save[cnt - i - 1]; 3692 3693 dev_dbg(rvu->dev, 3694 "%s: Moving %u ---> %u (vidx=%u)\n", 3695 __func__, 3696 old_midx, new_midx, vidx); 3697 3698 rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); 3699 if (rc) 3700 dev_err(rvu->dev, 3701 "%s: Error happened to add to show list vidx=%u\n", 3702 __func__, vidx); 3703 3704 /* Modify vidx to point to new mcam idx */ 3705 rc = npc_vidx_maps_modify(rvu, vidx, new_midx); 3706 if (rc) 3707 return rc; 3708 3709 midx = old_midx % mcam->banksize; 3710 bank = old_midx / mcam->banksize; 3711 stats = rvu_read64(rvu, blkaddr, 3712 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, 3713 bank)); 3714 3715 /* If bug happened during copy/enable mcam, then there is a bug in allocation 3716 * algorithm itself. There is no point in rewinding and returning, as it 3717 * will face further issue. Return error after printing error 3718 */ 3719 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) { 3720 dev_err(rvu->dev, 3721 "%s: Error happened while disabling old_mid=%u\n", 3722 __func__, old_midx); 3723 return -EFAULT; 3724 } 3725 3726 if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) { 3727 dev_err(rvu->dev, 3728 "%s: Error happened while copying old_midx=%u new_midx=%u\n", 3729 __func__, old_midx, new_midx); 3730 return -EFAULT; 3731 } 3732 3733 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) { 3734 dev_err(rvu->dev, 3735 "%s: Error happened while enabling new_mid=%u\n", 3736 __func__, new_midx); 3737 return -EFAULT; 3738 } 3739 3740 midx = new_midx % mcam->banksize; 3741 bank = new_midx / mcam->banksize; 3742 rvu_write64(rvu, blkaddr, 3743 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), 3744 stats); 3745 3746 /* Free the old mcam idx */ 3747 rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, 3748 &sb, &sb_off); 3749 if (rc) { 3750 dev_err(rvu->dev, 3751 "%s: Unable to calculate subbank off for mcamidx=%u\n", 3752 __func__, old_midx); 3753 return rc; 3754 } 3755 3756 deleted = __npc_subbank_free(rvu, sb, sb_off); 3757 if (!deleted) { 3758 dev_err(rvu->dev, 3759 "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", 3760 __func__, old_midx, sb->idx, sb_off); 3761 return -EFAULT; 3762 } 3763 3764 /* save pcifunc */ 3765 map = xa_load(&npc_priv->xa_idx2pf_map, old_midx); 3766 pcifunc = xa_to_value(map); 3767 3768 /* delete from pf maps */ 3769 rc = npc_del_from_pf_maps(rvu, old_midx); 3770 if (rc) { 3771 dev_err(rvu->dev, 3772 "%s: Failed to delete pf maps for mcamidx=%u\n", 3773 __func__, old_midx); 3774 return rc; 3775 } 3776 3777 /* add new mcam_idx to pf map */ 3778 rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); 3779 if (rc) { 3780 dev_err(rvu->dev, 3781 "%s: Failed to add pf maps for mcamidx=%u\n", 3782 __func__, new_midx); 3783 return rc; 3784 } 3785 3786 /* Remove from mcam maps */ 3787 mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; 3788 mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; 3789 npc_mcam_clear_bit(mcam, old_midx); 3790 3791 mcam->entry2pfvf_map[new_midx] = pcifunc; 3792 /* Counter is not preserved */ 3793 mcam->entry2cntr_map[new_midx] = new_midx; 3794 target_pf = mcam->entry2target_pffunc[old_midx]; 3795 mcam->entry2target_pffunc[new_midx] = target_pf; 3796 mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP; 3797 3798 npc_mcam_set_bit(mcam, new_midx); 3799 3800 /* Note: list order is not functionally required for mcam_rules */ 3801 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 3802 if (rule->entry != old_midx) 3803 continue; 3804 3805 rule->entry = new_midx; 3806 break; 3807 } 3808 3809 /* Mark as invalid */ 3810 v->vidx[vidx_cnt - i - 1] = -1; 3811 save[cnt - i - 1] = -1; 3812 3813 f->free_cnt--; 3814 v->vidx_cnt--; 3815 } 3816 3817 return 0; 3818 } 3819 3820 static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) 3821 { 3822 struct npc_defrag_node *v = NULL; 3823 struct npc_defrag_node *f = NULL; 3824 int rc = 0, cnt; 3825 u16 *save; 3826 3827 while (1) { 3828 /* Find subbank with minimum vidx */ 3829 if (!v) { 3830 v = npc_subbank_min_vidx(lh); 3831 if (!v) 3832 break; 3833 } 3834 3835 /* Find subbank with maximum free slots */ 3836 if (!f) { 3837 f = npc_subbank_max_free(lh); 3838 if (!f) 3839 break; 3840 } 3841 3842 if (!v->vidx_cnt) { 3843 list_del_init(&v->list); 3844 v = NULL; 3845 continue; 3846 } 3847 3848 if (!f->free_cnt) { 3849 list_del_init(&f->list); 3850 f = NULL; 3851 continue; 3852 } 3853 3854 /* If both subbanks are same, choose vidx and 3855 * search for free list again 3856 */ 3857 if (f == v) { 3858 list_del_init(&f->list); 3859 f = NULL; 3860 continue; 3861 } 3862 3863 /* Calculate minimum free slots needs to be allocated */ 3864 cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : 3865 f->free_cnt; 3866 3867 dev_dbg(rvu->dev, 3868 "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", 3869 __func__, cnt, f->free_cnt, f->idx, 3870 v->vidx_cnt, v->idx); 3871 3872 /* Allocate an array to store newly allocated 3873 * free slots (mcam indexes) 3874 */ 3875 save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); 3876 if (!save) { 3877 rc = -ENOMEM; 3878 goto err; 3879 } 3880 3881 /* Alloc free slots for existing vidx */ 3882 rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); 3883 if (rc) { 3884 kfree(save); 3885 goto err; 3886 } 3887 3888 /* Move vidx to free slots; update pf_map and vidx maps, 3889 * and free existing vidx mcam slots 3890 */ 3891 rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); 3892 if (rc) { 3893 kfree(save); 3894 goto err; 3895 } 3896 3897 kfree(save); 3898 3899 if (!f->free_cnt) { 3900 list_del_init(&f->list); 3901 f = NULL; 3902 } 3903 3904 if (!v->vidx_cnt) { 3905 list_del_init(&v->list); 3906 v = NULL; 3907 } 3908 } 3909 3910 err: 3911 /* Whole defragmentation process is done within locks. if there 3912 * is an error, it would be hard to roll back as index remove/add 3913 * can fail again if it failed before. This would mean that there 3914 * is bug in the index management algorithm. 3915 * Return from here than rolling back. 3916 */ 3917 return rc; 3918 } 3919 3920 static void npc_defrag_list_clear(void) 3921 { 3922 struct npc_defrag_show_node *node, *next; 3923 3924 mutex_lock(&npc_priv->lock); 3925 list_for_each_entry_safe(node, next, &npc_priv->defrag_lh, list) { 3926 list_del_init(&node->list); 3927 kfree(node); 3928 } 3929 3930 mutex_unlock(&npc_priv->lock); 3931 } 3932 3933 static void npc_lock_all_subbank(void) 3934 { 3935 int i; 3936 3937 for (i = 0; i < npc_priv->num_subbanks; i++) 3938 mutex_lock(&npc_priv->sb[i].lock); 3939 } 3940 3941 static void npc_unlock_all_subbank(void) 3942 { 3943 int i; 3944 3945 for (i = npc_priv->num_subbanks - 1; i >= 0; i--) 3946 mutex_unlock(&npc_priv->sb[i].lock); 3947 } 3948 3949 int npc_cn20k_search_order_set(struct rvu *rvu, 3950 u64 narr[MAX_NUM_SUB_BANKS], int cnt) 3951 { 3952 struct npc_mcam *mcam = &rvu->hw->mcam; 3953 int rsrc[2][MAX_NUM_SUB_BANKS] = { }; 3954 u8 save[MAX_NUM_SUB_BANKS] = { }; 3955 struct npc_subbank *sb; 3956 struct xarray *xa; 3957 int prio, rc, err; 3958 int sb_idx; 3959 enum { 3960 FREE = 0, 3961 USED = 1, 3962 }; 3963 3964 if (cnt != npc_priv->num_subbanks) { 3965 dev_err(rvu->dev, "Number of entries(%u) != %u\n", 3966 cnt, npc_priv->num_subbanks); 3967 return -EINVAL; 3968 } 3969 3970 mutex_lock(&mcam->lock); 3971 npc_lock_all_subbank(); 3972 3973 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 3974 sb = &npc_priv->sb[sb_idx]; 3975 save[sb->idx] = sb->arr_idx; 3976 } 3977 3978 for (prio = 0; prio < cnt; prio++) { 3979 sb_idx = narr[prio]; 3980 sb = &npc_priv->sb[sb_idx]; 3981 3982 if (sb->flags & NPC_SUBBANK_FLAG_USED) 3983 xa = &npc_priv->xa_sb_used; 3984 else 3985 xa = &npc_priv->xa_sb_free; 3986 3987 rc = xa_err(xa_store(xa, prio, 3988 xa_mk_value(sb_idx), GFP_KERNEL)); 3989 if (rc) { 3990 dev_err(rvu->dev, 3991 "Setting arr_idx=%d for sb=%d failed\n", 3992 sb->arr_idx, sb_idx); 3993 goto fail; 3994 } 3995 3996 if (sb->flags & NPC_SUBBANK_FLAG_USED) { 3997 rsrc[USED][sb->arr_idx] -= 1; 3998 rsrc[USED][prio] += 1; 3999 } else { 4000 rsrc[FREE][sb->arr_idx] -= 1; 4001 rsrc[FREE][prio] += 1; 4002 } 4003 4004 sb->arr_idx = prio; 4005 } 4006 4007 for (prio = 0; prio < cnt; prio++) { 4008 if (rsrc[FREE][prio] == -1) 4009 xa_erase(&npc_priv->xa_sb_free, prio); 4010 4011 if (rsrc[USED][prio] == -1) 4012 xa_erase(&npc_priv->xa_sb_used, prio); 4013 } 4014 4015 for (int i = 0; i < cnt; i++) 4016 subbank_srch_order[i] = (u32)narr[i]; 4017 4018 restrict_valid = false; 4019 4020 npc_unlock_all_subbank(); 4021 mutex_unlock(&mcam->lock); 4022 4023 return 0; 4024 4025 fail: 4026 for (prio = 0; prio < cnt; prio++) { 4027 if (rsrc[FREE][prio] == 1) 4028 xa_erase(&npc_priv->xa_sb_free, prio); 4029 4030 if (rsrc[USED][prio] == 1) 4031 xa_erase(&npc_priv->xa_sb_used, prio); 4032 } 4033 4034 for (sb_idx = 0; sb_idx < cnt; sb_idx++) { 4035 sb = &npc_priv->sb[sb_idx]; 4036 sb->arr_idx = save[sb_idx]; 4037 4038 if (sb->flags & NPC_SUBBANK_FLAG_USED) 4039 xa = &npc_priv->xa_sb_used; 4040 else 4041 xa = &npc_priv->xa_sb_free; 4042 4043 /* Since the entry already exists, xa_store() replaces 4044 * the value without a kmalloc(), making failure highly unlikely. 4045 */ 4046 err = xa_err(xa_store(xa, sb->arr_idx, 4047 xa_mk_value(sb->idx), GFP_KERNEL)); 4048 WARN(!!err, "Failed to rollback sb=%u idx=%u\n", 4049 sb->idx, sb->arr_idx); 4050 } 4051 4052 npc_unlock_all_subbank(); 4053 mutex_unlock(&mcam->lock); 4054 4055 return rc; 4056 } 4057 4058 const u32 *npc_cn20k_search_order_get(bool *restricted_order, u32 *sz) 4059 { 4060 *restricted_order = restrict_valid; 4061 *sz = npc_priv->num_subbanks; 4062 return subbank_srch_order; 4063 } 4064 4065 /* Only non-ref non-contigous mcam indexes 4066 * are picked for defrag process 4067 */ 4068 int npc_cn20k_defrag(struct rvu *rvu) 4069 { 4070 struct npc_mcam *mcam = &rvu->hw->mcam; 4071 struct npc_defrag_node *node, *tnode; 4072 struct list_head x4lh, x2lh, *lh; 4073 int rc = 0, i, sb_off, tot; 4074 struct npc_subbank *sb; 4075 unsigned long index; 4076 void *map; 4077 u16 midx; 4078 4079 /* Free previous show list */ 4080 npc_defrag_list_clear(); 4081 4082 INIT_LIST_HEAD(&x4lh); 4083 INIT_LIST_HEAD(&x2lh); 4084 4085 node = kcalloc(npc_priv->num_subbanks, sizeof(*node), GFP_KERNEL); 4086 if (!node) 4087 return -ENOMEM; 4088 4089 /* Lock mcam */ 4090 mutex_lock(&mcam->lock); 4091 npc_lock_all_subbank(); 4092 4093 /* Fill in node with subbank properties */ 4094 for (i = 0; i < npc_priv->num_subbanks; i++) { 4095 sb = &npc_priv->sb[i]; 4096 4097 node[i].idx = i; 4098 node[i].key_type = sb->key_type; 4099 node[i].free_cnt = sb->free_cnt; 4100 node[i].vidx = kcalloc(npc_priv->subbank_depth * 2, 4101 sizeof(*node[i].vidx), 4102 GFP_KERNEL); 4103 if (!node[i].vidx) { 4104 rc = -ENOMEM; 4105 goto free_vidx; 4106 } 4107 4108 /* If subbank is empty, dont include it in defrag 4109 * process 4110 */ 4111 if (sb->flags & NPC_SUBBANK_FLAG_FREE) { 4112 node[i].valid = false; 4113 continue; 4114 } 4115 4116 if (npc_defrag_skip_restricted_sb(i)) { 4117 node[i].valid = false; 4118 continue; 4119 } 4120 4121 node[i].valid = true; 4122 INIT_LIST_HEAD(&node[i].list); 4123 4124 /* Add node to x2 or x4 list */ 4125 lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; 4126 list_add_tail(&node[i].list, lh); 4127 } 4128 4129 /* Filling vidx[] array with all vidx in that subbank */ 4130 xa_for_each_start(&npc_priv->xa_vidx2idx_map, index, map, 4131 npc_priv->bank_depth * 2) { 4132 midx = xa_to_value(map); 4133 rc = npc_mcam_idx_2_subbank_idx(rvu, midx, 4134 &sb, &sb_off); 4135 if (rc) { 4136 dev_err(rvu->dev, 4137 "%s: Error to get mcam_idx for vidx=%lu\n", 4138 __func__, index); 4139 goto free_vidx; 4140 } 4141 4142 tnode = &node[sb->idx]; 4143 tnode->vidx[tnode->vidx_cnt] = index; 4144 tnode->vidx_cnt++; 4145 } 4146 4147 /* Mark all subbank which has ref allocation */ 4148 for (i = 0; i < npc_priv->num_subbanks; i++) { 4149 tnode = &node[i]; 4150 4151 if (!tnode->valid) 4152 continue; 4153 4154 tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? 4155 npc_priv->subbank_depth * 2 : npc_priv->subbank_depth; 4156 4157 if (node[i].vidx_cnt != tot - tnode->free_cnt) 4158 tnode->refs = true; 4159 } 4160 4161 rc = npc_defrag_process(rvu, &x2lh); 4162 if (rc) 4163 goto free_vidx; 4164 4165 rc = npc_defrag_process(rvu, &x4lh); 4166 if (rc) 4167 goto free_vidx; 4168 4169 free_vidx: 4170 npc_unlock_all_subbank(); 4171 mutex_unlock(&mcam->lock); 4172 for (i = 0; i < npc_priv->num_subbanks; i++) 4173 kfree(node[i].vidx); 4174 kfree(node); 4175 return rc; 4176 } 4177 4178 int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, 4179 struct msg_rsp *rsp) 4180 { 4181 return npc_cn20k_defrag(rvu); 4182 } 4183 4184 int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, 4185 u16 *mcast, u16 *promisc, u16 *ucast) 4186 { 4187 u16 *ptr[4] = {promisc, mcast, bcast, ucast}; 4188 unsigned long idx; 4189 bool set = false; 4190 void *val; 4191 int i, j; 4192 4193 for (i = 0; i < ARRAY_SIZE(ptr); i++) { 4194 if (!ptr[i]) 4195 continue; 4196 4197 *ptr[i] = USHRT_MAX; 4198 } 4199 4200 if (!npc_priv) 4201 return 0; 4202 4203 if (is_lbk_vf(rvu, pcifunc)) { 4204 if (!ptr[0]) 4205 return -EINVAL; 4206 4207 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4208 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4209 if (!val) { 4210 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4211 __func__, 4212 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4213 pcifunc); 4214 4215 return -ESRCH; 4216 } 4217 4218 *ptr[0] = xa_to_value(val); 4219 return 0; 4220 } 4221 4222 if (is_vf(pcifunc)) { 4223 if (!ptr[3]) 4224 return -EINVAL; 4225 4226 idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4227 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4228 if (!val) { 4229 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4230 __func__, 4231 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4232 pcifunc); 4233 4234 return -ESRCH; 4235 } 4236 4237 *ptr[3] = xa_to_value(val); 4238 return 0; 4239 } 4240 4241 for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, 4242 j++) { 4243 if (!ptr[j]) 4244 continue; 4245 4246 idx = NPC_DFT_RULE_ID_MK(pcifunc, i); 4247 val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx); 4248 if (!val) { 4249 pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", 4250 __func__, 4251 npc_dft_rule_name[i], pcifunc); 4252 4253 continue; 4254 } 4255 4256 *ptr[j] = xa_to_value(val); 4257 set = true; 4258 } 4259 4260 return set ? 0 : -ESRCH; 4261 } 4262 4263 int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, 4264 struct npc_get_pfl_info_rsp *rsp) 4265 { 4266 if (!is_cn20k(rvu->pdev)) { 4267 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4268 return -EOPNOTSUPP; 4269 } 4270 4271 rsp->kw_type = npc_priv->kw; 4272 rsp->x4_slots = npc_priv->bank_depth; 4273 return 0; 4274 } 4275 4276 int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, 4277 struct npc_get_num_kws_req *req, 4278 struct npc_get_num_kws_rsp *rsp) 4279 { 4280 u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4281 u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; 4282 struct rvu_npc_mcam_rule dummy = { 0 }; 4283 struct mcam_entry_mdata mdata = { }; 4284 struct npc_install_flow_req *fl; 4285 int i, cnt = 0, blkaddr; 4286 4287 if (!is_cn20k(rvu->pdev)) { 4288 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4289 return -EOPNOTSUPP; 4290 } 4291 4292 fl = &req->fl; 4293 4294 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4295 if (blkaddr < 0) { 4296 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4297 return NPC_MCAM_INVALID_REQ; 4298 } 4299 4300 mdata.kw = kw; 4301 mdata.kw_mask = kw_mask; 4302 4303 npc_update_flow(rvu, &mdata, fl->features, &fl->packet, 4304 &fl->mask, &dummy, fl->intf, blkaddr); 4305 4306 /* Find the most significant word valid. Traverse from 4307 * MSB to LSB, check if cam0 or cam1 is set 4308 */ 4309 for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { 4310 if (kw[i] || kw_mask[i]) { 4311 cnt = i + 1; 4312 break; 4313 } 4314 } 4315 4316 rsp->kws = cnt; 4317 4318 return 0; 4319 } 4320 4321 int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, 4322 struct npc_get_dft_rl_idxs_rsp *rsp) 4323 { 4324 u16 bcast, mcast, promisc, ucast; 4325 u16 pcifunc; 4326 int rc; 4327 4328 if (!is_cn20k(rvu->pdev)) { 4329 dev_err(rvu->dev, "Mbox support is only for cn20k\n"); 4330 return -EOPNOTSUPP; 4331 } 4332 4333 pcifunc = req->hdr.pcifunc; 4334 4335 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 4336 &promisc, &ucast); 4337 if (rc) 4338 return rc; 4339 4340 rsp->bcast = bcast; 4341 rsp->mcast = mcast; 4342 rsp->promisc = promisc; 4343 rsp->ucast = ucast; 4344 return 0; 4345 } 4346 4347 bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) 4348 { 4349 return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || 4350 is_lbk_vf(rvu, pcifunc); 4351 } 4352 4353 void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) 4354 { 4355 struct npc_mcam *mcam = &rvu->hw->mcam; 4356 u16 ptr[4] = {[0 ... 3] = USHRT_MAX}; 4357 struct rvu_npc_mcam_rule *rule, *tmp; 4358 unsigned long index; 4359 int blkaddr, rc, i; 4360 void *map; 4361 4362 if (!npc_priv) 4363 return; 4364 4365 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4366 dev_dbg(rvu->dev, 4367 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4368 __func__, pcifunc); 4369 return; 4370 } 4371 4372 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 4373 &ptr[2], &ptr[3]); 4374 if (rc) 4375 return; 4376 4377 /* LBK */ 4378 if (is_lbk_vf(rvu, pcifunc)) { 4379 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4380 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4381 if (!map) 4382 dev_dbg(rvu->dev, 4383 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4384 __func__, 4385 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4386 pcifunc); 4387 4388 goto free_rules; 4389 } 4390 4391 /* VF */ 4392 if (is_vf(pcifunc)) { 4393 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4394 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4395 if (!map) 4396 dev_dbg(rvu->dev, 4397 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4398 __func__, 4399 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4400 pcifunc); 4401 4402 goto free_rules; 4403 } 4404 4405 /* PF */ 4406 for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { 4407 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4408 map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4409 if (!map) 4410 dev_dbg(rvu->dev, 4411 "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", 4412 __func__, npc_dft_rule_name[i], 4413 pcifunc); 4414 } 4415 4416 free_rules: 4417 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4418 if (blkaddr < 0) 4419 return; 4420 for (int i = 0; i < 4; i++) { 4421 if (ptr[i] == USHRT_MAX) 4422 continue; 4423 4424 mutex_lock(&mcam->lock); 4425 npc_mcam_clear_bit(mcam, ptr[i]); 4426 mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP; 4427 npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false); 4428 mcam->entry2target_pffunc[ptr[i]] = 0x0; 4429 mutex_unlock(&mcam->lock); 4430 4431 rc = npc_cn20k_idx_free(rvu, &ptr[i], 1); 4432 if (rc) { 4433 /* Non recoverable error. Let us WARN and return. Keep system alive to 4434 * enable debugging 4435 */ 4436 WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n", 4437 __func__, pcifunc, ptr[i]); 4438 return; 4439 } 4440 } 4441 4442 mutex_lock(&mcam->lock); 4443 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 4444 for (int i = 0; i < 4; i++) { 4445 if (ptr[i] != rule->entry) 4446 continue; 4447 4448 list_del(&rule->list); 4449 kfree(rule); 4450 break; 4451 } 4452 } 4453 mutex_unlock(&mcam->lock); 4454 } 4455 4456 int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) 4457 { 4458 struct npc_mcam_free_entry_req free_req = { 0 }; 4459 u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; 4460 struct npc_mcam_alloc_entry_req req = { 0 }; 4461 struct npc_mcam_alloc_entry_rsp rsp = { 0 }; 4462 int ret, eidx, i, k, pf, cnt; 4463 struct rvu_pfvf *pfvf; 4464 unsigned long index; 4465 struct msg_rsp free_rsp; 4466 u16 b, m, p, u; 4467 4468 if (!npc_priv) 4469 return 0; 4470 4471 if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { 4472 dev_dbg(rvu->dev, 4473 "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", 4474 __func__, pcifunc); 4475 return 0; 4476 } 4477 4478 /* Check if default rules are already alloced for this pcifunc */ 4479 ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); 4480 if (!ret) { 4481 dev_dbg(rvu->dev, 4482 "%s: default rules are already installed (pcifunc=%#x)\n", 4483 __func__, pcifunc); 4484 dev_dbg(rvu->dev, 4485 "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", 4486 __func__, b, m, p, u); 4487 return 0; 4488 } 4489 4490 /* Set ref index as lowest priority index */ 4491 eidx = 2 * npc_priv->bank_depth - 1; 4492 4493 /* Install only UCAST for VF */ 4494 cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); 4495 4496 /* For VF pcifunc, allocate default mcam indexes by taking 4497 * ref as PF's ucast index. 4498 */ 4499 if (is_vf(pcifunc)) { 4500 pf = rvu_get_pf(rvu->pdev, pcifunc); 4501 pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; 4502 4503 /* Get PF's ucast entry index */ 4504 ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, 4505 NULL, NULL, &pf_ucast); 4506 4507 /* There is no PF rules installed; and VF installation comes 4508 * first. PF may come later. 4509 * TODO: Install PF rules before installing VF rules. 4510 */ 4511 4512 /* Set PF's ucast as ref entry */ 4513 if (!ret) 4514 eidx = pf_ucast; 4515 } 4516 4517 pfvf = rvu_get_pfvf(rvu, pcifunc); 4518 pfvf->hw_prio = NPC_DFT_RULE_PRIO; 4519 4520 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4521 req.kw_type = NPC_MCAM_KEY_X4; 4522 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4523 } else { 4524 req.kw_type = NPC_MCAM_KEY_X2; 4525 req.ref_entry = eidx; 4526 } 4527 4528 req.contig = false; 4529 req.ref_prio = NPC_MCAM_HIGHER_PRIO; 4530 req.count = cnt; 4531 req.hdr.pcifunc = pcifunc; 4532 4533 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4534 4535 /* successfully allocated index */ 4536 if (!ret) { 4537 /* Copy indexes to local array */ 4538 for (i = 0; i < cnt; i++) 4539 mcam_idx[i] = rsp.entry_list[i]; 4540 4541 goto chk_sanity; 4542 } 4543 4544 /* If there is no slots available and request is for PF, 4545 * return error. 4546 */ 4547 if (!is_vf(pcifunc)) { 4548 dev_err(rvu->dev, 4549 "%s: Default index allocation failed for pcifunc=%#x\n", 4550 __func__, pcifunc); 4551 return ret; 4552 } 4553 4554 /* We could not find an index with higher priority index for VF. 4555 * Find rule with lower priority index and set hardware priority 4556 * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) 4557 */ 4558 req.contig = false; 4559 req.count = cnt; 4560 req.hdr.pcifunc = pcifunc; 4561 req.ref_prio = NPC_MCAM_LOWER_PRIO; 4562 4563 if (npc_priv->kw == NPC_MCAM_KEY_X4) { 4564 req.kw_type = NPC_MCAM_KEY_X4; 4565 req.ref_entry = eidx & (npc_priv->bank_depth - 1); 4566 } else { 4567 req.kw_type = NPC_MCAM_KEY_X2; 4568 req.ref_entry = eidx; 4569 } 4570 4571 ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); 4572 if (ret) { 4573 dev_err(rvu->dev, 4574 "%s: Default index allocation failed for pcifunc=%#x\n", 4575 __func__, pcifunc); 4576 return ret; 4577 } 4578 4579 /* Copy indexes to local array */ 4580 for (i = 0; i < cnt; i++) 4581 mcam_idx[i] = rsp.entry_list[i]; 4582 4583 pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; 4584 4585 chk_sanity: 4586 /* LBK */ 4587 if (is_lbk_vf(rvu, pcifunc)) { 4588 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); 4589 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4590 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4591 if (ret) { 4592 dev_err(rvu->dev, 4593 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4594 __func__, 4595 npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], 4596 pcifunc); 4597 goto err; 4598 } 4599 4600 goto done; 4601 } 4602 4603 /* VF */ 4604 if (is_vf(pcifunc)) { 4605 index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); 4606 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4607 xa_mk_value(mcam_idx[0]), GFP_KERNEL); 4608 if (ret) { 4609 dev_err(rvu->dev, 4610 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4611 __func__, 4612 npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], 4613 pcifunc); 4614 goto err; 4615 } 4616 4617 goto done; 4618 } 4619 4620 /* PF */ 4621 for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && 4622 k < cnt; i++, k++) { 4623 index = NPC_DFT_RULE_ID_MK(pcifunc, i); 4624 ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index, 4625 xa_mk_value(mcam_idx[k]), GFP_KERNEL); 4626 if (ret) { 4627 dev_err(rvu->dev, 4628 "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", 4629 __func__, npc_dft_rule_name[i], 4630 pcifunc); 4631 for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { 4632 index = NPC_DFT_RULE_ID_MK(pcifunc, p); 4633 xa_erase(&npc_priv->xa_pf2dfl_rmap, index); 4634 } 4635 goto err; 4636 } 4637 } 4638 4639 done: 4640 return 0; 4641 4642 err: 4643 free_req.hdr.pcifunc = pcifunc; 4644 free_req.all = 1; 4645 ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); 4646 if (ret) 4647 dev_err(rvu->dev, 4648 "%s: Error deleting default entries (pcifunc=%#x\n", 4649 __func__, pcifunc); 4650 4651 return -EFAULT; 4652 } 4653 4654 static int npc_priv_init(struct rvu *rvu) 4655 { 4656 struct npc_mcam *mcam = &rvu->hw->mcam; 4657 int blkaddr, num_banks, bank_depth; 4658 int num_subbanks, subbank_depth; 4659 u64 npc_const1, npc_const2 = 0; 4660 struct npc_subbank *sb; 4661 int ret = -ENOMEM; 4662 u64 cfg; 4663 int i; 4664 4665 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4666 if (blkaddr < 0) { 4667 dev_err(rvu->dev, "%s: NPC block not implemented\n", 4668 __func__); 4669 return -ENODEV; 4670 } 4671 4672 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 4673 if (npc_const1 & BIT_ULL(63)) 4674 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 4675 4676 num_banks = mcam->banks; 4677 if (!num_banks || num_banks > MAX_NUM_BANKS) { 4678 dev_err(rvu->dev, 4679 "Number of banks(%u) is invalid\n", num_banks); 4680 return -EINVAL; 4681 } 4682 4683 num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); 4684 if (!num_subbanks || num_subbanks > MAX_NUM_SUB_BANKS) { 4685 dev_err(rvu->dev, 4686 "Number of subbanks is invalid %u\n", 4687 num_subbanks); 4688 return -EFAULT; 4689 } 4690 4691 if (num_subbanks & (num_subbanks - 1)) { 4692 dev_err(rvu->dev, 4693 "subbanks cnt(%u) should be a power of 2\n", 4694 num_subbanks); 4695 return -EINVAL; 4696 } 4697 4698 bank_depth = mcam->banksize; 4699 if (!bank_depth || bank_depth % num_subbanks) { 4700 dev_err(rvu->dev, 4701 "Bank depth(%u) should be a multiple of num_subbanks(%u)\n", 4702 bank_depth, num_subbanks); 4703 return -EINVAL; 4704 } 4705 4706 subbank_depth = bank_depth / num_subbanks; 4707 if (!subbank_depth || subbank_depth > MAX_SUBBANK_DEPTH) { 4708 dev_err(rvu->dev, 4709 "Invalid subbank depth %u\n", 4710 subbank_depth); 4711 return -EINVAL; 4712 } 4713 4714 npc_priv = kcalloc(1, sizeof(*npc_priv), GFP_KERNEL); 4715 if (!npc_priv) 4716 return -ENOMEM; 4717 4718 npc_priv->num_banks = num_banks; 4719 npc_priv->num_subbanks = num_subbanks; 4720 npc_priv->bank_depth = bank_depth; 4721 npc_priv->subbank_depth = subbank_depth; 4722 4723 /* Get kex configured key size */ 4724 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); 4725 npc_priv->kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); 4726 4727 dev_info(rvu->dev, 4728 "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", 4729 num_banks, bank_depth, num_subbanks, subbank_depth, 4730 npc_kw_name[npc_priv->kw]); 4731 4732 npc_priv->sb = kcalloc(num_subbanks, sizeof(struct npc_subbank), 4733 GFP_KERNEL); 4734 if (!npc_priv->sb) 4735 goto fail1; 4736 4737 xa_init_flags(&npc_priv->xa_sb_used, XA_FLAGS_ALLOC); 4738 xa_init_flags(&npc_priv->xa_sb_free, XA_FLAGS_ALLOC); 4739 xa_init_flags(&npc_priv->xa_idx2pf_map, XA_FLAGS_ALLOC); 4740 xa_init_flags(&npc_priv->xa_pf_map, XA_FLAGS_ALLOC); 4741 xa_init_flags(&npc_priv->xa_pf2dfl_rmap, XA_FLAGS_ALLOC); 4742 xa_init_flags(&npc_priv->xa_idx2vidx_map, XA_FLAGS_ALLOC); 4743 xa_init_flags(&npc_priv->xa_vidx2idx_map, XA_FLAGS_ALLOC); 4744 4745 if (npc_create_srch_order(num_subbanks)) 4746 goto fail2; 4747 4748 npc_populate_restricted_idxs(num_subbanks); 4749 4750 /* Initialize subbanks */ 4751 for (i = 0, sb = npc_priv->sb; i < num_subbanks; i++, sb++) 4752 npc_subbank_init(rvu, sb, i); 4753 4754 ret = npc_subbanks_srch_order_init(rvu); 4755 if (ret) 4756 goto fail3; 4757 4758 /* Get number of pcifuncs in the system */ 4759 npc_priv->pf_cnt = npc_pcifunc_map_create(rvu); 4760 npc_priv->xa_pf2idx_map = kcalloc(npc_priv->pf_cnt, 4761 sizeof(struct xarray), 4762 GFP_KERNEL); 4763 if (!npc_priv->xa_pf2idx_map) { 4764 ret = -ENOMEM; 4765 goto fail3; 4766 } 4767 4768 for (i = 0; i < npc_priv->pf_cnt; i++) 4769 xa_init_flags(&npc_priv->xa_pf2idx_map[i], XA_FLAGS_ALLOC); 4770 4771 INIT_LIST_HEAD(&npc_priv->defrag_lh); 4772 mutex_init(&npc_priv->lock); 4773 4774 return 0; 4775 4776 fail3: 4777 kfree(subbank_srch_order); 4778 subbank_srch_order = NULL; 4779 4780 fail2: 4781 xa_destroy(&npc_priv->xa_sb_used); 4782 xa_destroy(&npc_priv->xa_sb_free); 4783 xa_destroy(&npc_priv->xa_idx2pf_map); 4784 xa_destroy(&npc_priv->xa_pf_map); 4785 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4786 xa_destroy(&npc_priv->xa_idx2vidx_map); 4787 xa_destroy(&npc_priv->xa_vidx2idx_map); 4788 kfree(npc_priv->sb); 4789 npc_priv->sb = NULL; 4790 fail1: 4791 kfree(npc_priv); 4792 npc_priv = NULL; 4793 return ret; 4794 } 4795 4796 void npc_cn20k_deinit(struct rvu *rvu) 4797 { 4798 int i; 4799 4800 if (!npc_priv) 4801 return; 4802 4803 xa_destroy(&npc_priv->xa_sb_used); 4804 xa_destroy(&npc_priv->xa_sb_free); 4805 xa_destroy(&npc_priv->xa_idx2pf_map); 4806 xa_destroy(&npc_priv->xa_pf_map); 4807 xa_destroy(&npc_priv->xa_pf2dfl_rmap); 4808 xa_destroy(&npc_priv->xa_idx2vidx_map); 4809 xa_destroy(&npc_priv->xa_vidx2idx_map); 4810 4811 for (i = 0; i < npc_priv->pf_cnt; i++) 4812 xa_destroy(&npc_priv->xa_pf2idx_map[i]); 4813 4814 kfree(npc_priv->xa_pf2idx_map); 4815 /* No need to destroy mutex lock as it is 4816 * part of subbank structure 4817 */ 4818 kfree(npc_priv->sb); 4819 kfree(subbank_srch_order); 4820 bitmap_clear(npc_priv->en_map, 0, MAX_NUM_BANKS * MAX_NUM_SUB_BANKS * 4821 MAX_SUBBANK_DEPTH); 4822 npc_defrag_list_clear(); 4823 kfree(npc_priv); 4824 npc_priv = NULL; 4825 } 4826 4827 static int npc_setup_mcam_section(struct rvu *rvu, int key_type) 4828 { 4829 int blkaddr, sec; 4830 4831 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4832 if (blkaddr < 0) { 4833 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4834 return -ENODEV; 4835 } 4836 4837 for (sec = 0; sec < npc_priv->num_subbanks; sec++) 4838 rvu_write64(rvu, blkaddr, 4839 NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); 4840 4841 return 0; 4842 } 4843 4844 int npc_cn20k_init(struct rvu *rvu) 4845 { 4846 int err; 4847 4848 err = npc_priv_init(rvu); 4849 if (err) { 4850 dev_err(rvu->dev, "%s: Error to init\n", 4851 __func__); 4852 return err; 4853 } 4854 4855 err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); 4856 if (err) { 4857 dev_err(rvu->dev, "%s: mcam section configuration failure\n", 4858 __func__); 4859 goto fail; 4860 } 4861 4862 return 0; 4863 4864 fail: 4865 npc_cn20k_deinit(rvu); 4866 return err; 4867 } 4868