1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 #include "rvu_npc_hash.h" 19 #include "cn20k/npc.h" 20 #include "rvu_npc.h" 21 #include "cn20k/reg.h" 22 23 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 24 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 25 26 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 27 #define NPC_HW_TSTAMP_OFFSET 8ULL 28 #define NPC_KEX_CHAN_MASK 0xFFFULL 29 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 30 31 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 32 33 static const char def_pfl_name[] = "default"; 34 35 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 36 int blkaddr, u16 pcifunc); 37 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 38 u16 pcifunc); 39 40 bool is_npc_intf_tx(u8 intf) 41 { 42 return !!(intf & 0x1); 43 } 44 45 bool is_npc_intf_rx(u8 intf) 46 { 47 return !(intf & 0x1); 48 } 49 50 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 51 { 52 struct rvu_hwinfo *hw = rvu->hw; 53 54 return intf < hw->npc_intfs; 55 } 56 57 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 58 { 59 /* Due to a HW issue in these silicon versions, parse nibble enable 60 * configuration has to be identical for both Rx and Tx interfaces. 61 */ 62 if (is_rvu_96xx_B0(rvu)) 63 return nibble_ena; 64 return 0; 65 } 66 67 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 68 { 69 int blkaddr; 70 u64 val = 0; 71 72 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 73 if (blkaddr < 0) 74 return; 75 76 /* Config CPI base for the PKIND */ 77 val = pkind | 1ULL << 62; 78 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 79 } 80 81 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 82 { 83 struct npc_pkind *pkind = &rvu->hw->pkind; 84 u32 map; 85 int i; 86 87 for (i = 0; i < pkind->rsrc.max; i++) { 88 map = pkind->pfchan_map[i]; 89 if (((map >> 16) & 0x3F) == pf) 90 return i; 91 } 92 return -1; 93 } 94 95 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 96 97 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 98 { 99 int pkind, blkaddr; 100 u64 val; 101 102 pkind = rvu_npc_get_pkind(rvu, pf); 103 if (pkind < 0) { 104 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 105 return -EINVAL; 106 } 107 108 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 109 if (blkaddr < 0) { 110 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 111 return -EINVAL; 112 } 113 114 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 115 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 116 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 117 if (enable) 118 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 119 NPC_HW_TSTAMP_OFFSET); 120 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 121 122 return 0; 123 } 124 125 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 126 int nixlf) 127 { 128 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 129 struct rvu *rvu = hw->rvu; 130 int blkaddr = 0, max = 0; 131 struct rvu_block *block; 132 struct rvu_pfvf *pfvf; 133 134 pfvf = rvu_get_pfvf(rvu, pcifunc); 135 /* Given a PF/VF and NIX LF number calculate the unicast mcam 136 * entry index based on the NIX block assigned to the PF/VF. 137 */ 138 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 139 while (blkaddr) { 140 if (pfvf->nix_blkaddr == blkaddr) 141 break; 142 block = &rvu->hw->block[blkaddr]; 143 max += block->lf.max; 144 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 145 } 146 147 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 148 } 149 150 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 151 u16 pcifunc, int nixlf, int type) 152 { 153 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 154 struct rvu *rvu = hw->rvu; 155 u16 bcast, mcast, promisc, ucast; 156 int index; 157 int rc; 158 int pf; 159 160 if (is_cn20k(rvu->pdev)) { 161 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 162 &promisc, &ucast); 163 if (rc) 164 return -EFAULT; 165 166 if (is_lbk_vf(rvu, pcifunc)) { 167 if (promisc == USHRT_MAX) 168 return -EINVAL; 169 return promisc; 170 } 171 172 if (is_cgx_vf(rvu, pcifunc)) { 173 if (ucast == USHRT_MAX) 174 return -EINVAL; 175 176 return ucast; 177 } 178 179 switch (type) { 180 case NIXLF_BCAST_ENTRY: 181 if (bcast == USHRT_MAX) 182 return -EINVAL; 183 return bcast; 184 case NIXLF_ALLMULTI_ENTRY: 185 if (mcast == USHRT_MAX) 186 return -EINVAL; 187 return mcast; 188 case NIXLF_PROMISC_ENTRY: 189 if (promisc == USHRT_MAX) 190 return -EINVAL; 191 return promisc; 192 case NIXLF_UCAST_ENTRY: 193 if (ucast == USHRT_MAX) 194 return -EINVAL; 195 return ucast; 196 default: 197 return -EINVAL; 198 } 199 } 200 201 /* Check if this is for a PF */ 202 pf = rvu_get_pf(rvu->pdev, pcifunc); 203 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 204 /* Reserved entries exclude PF0 */ 205 pf--; 206 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 207 /* Broadcast address matching entry should be first so 208 * that the packet can be replicated to all VFs. 209 */ 210 if (type == NIXLF_BCAST_ENTRY) 211 return index; 212 else if (type == NIXLF_ALLMULTI_ENTRY) 213 return index + 1; 214 else if (type == NIXLF_PROMISC_ENTRY) 215 return index + 2; 216 } 217 218 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 219 } 220 221 int npc_get_bank(struct npc_mcam *mcam, int index) 222 { 223 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 224 int bank = index / mcam->banksize; 225 struct rvu *rvu = hw->rvu; 226 227 if (is_cn20k(rvu->pdev)) 228 return bank; 229 230 /* 0,1 & 2,3 banks are combined for this keysize */ 231 if (mcam->keysize == NPC_MCAM_KEY_X2) 232 return bank ? 2 : 0; 233 234 return bank; 235 } 236 237 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 238 int blkaddr, int index) 239 { 240 int bank = npc_get_bank(mcam, index); 241 u64 cfg; 242 243 index &= (mcam->banksize - 1); 244 if (is_cn20k(rvu->pdev)) 245 cfg = rvu_read64(rvu, blkaddr, 246 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, 247 bank)); 248 else 249 cfg = rvu_read64(rvu, blkaddr, 250 NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 251 252 return (cfg & 1); 253 } 254 255 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 256 int blkaddr, int index, bool enable) 257 { 258 int bank = npc_get_bank(mcam, index); 259 int actbank = bank; 260 261 if (is_cn20k(rvu->pdev)) { 262 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 263 dev_err(rvu->dev, "Error to %s mcam %u entry\n", 264 enable ? "enable" : "disable", index); 265 return; 266 } 267 268 index &= (mcam->banksize - 1); 269 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 270 rvu_write64(rvu, blkaddr, 271 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 272 enable ? 1 : 0); 273 } 274 } 275 276 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 277 int blkaddr, int index) 278 { 279 int bank = npc_get_bank(mcam, index); 280 int actbank = bank; 281 282 if (is_cn20k(rvu->pdev)) { 283 if (npc_cn20k_clear_mcam_entry(rvu, blkaddr, index)) 284 dev_err(rvu->dev, "%s Failed to clear mcam %u\n", 285 __func__, index); 286 return; 287 } 288 289 index &= (mcam->banksize - 1); 290 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 291 rvu_write64(rvu, blkaddr, 292 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 293 rvu_write64(rvu, blkaddr, 294 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 295 296 rvu_write64(rvu, blkaddr, 297 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 298 rvu_write64(rvu, blkaddr, 299 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 300 301 rvu_write64(rvu, blkaddr, 302 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 303 rvu_write64(rvu, blkaddr, 304 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 305 } 306 } 307 308 static void npc_get_keyword(struct mcam_entry *entry, int idx, 309 u64 *cam0, u64 *cam1) 310 { 311 u64 kw_mask = 0x00; 312 313 #define CAM_MASK(n) (BIT_ULL(n) - 1) 314 315 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 316 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 317 * 318 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 319 */ 320 switch (idx) { 321 case 0: 322 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 323 *cam1 = entry->kw[0]; 324 kw_mask = entry->kw_mask[0]; 325 break; 326 case 1: 327 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 328 *cam1 = entry->kw[1] & CAM_MASK(48); 329 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 330 break; 331 case 2: 332 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 333 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 334 */ 335 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 336 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 337 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 338 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 339 break; 340 case 3: 341 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 342 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 343 */ 344 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 345 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 346 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 347 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 348 break; 349 case 4: 350 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 351 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 352 */ 353 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 354 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 355 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 356 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 357 break; 358 case 5: 359 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 360 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 361 */ 362 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 363 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 364 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 365 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 366 break; 367 case 6: 368 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 369 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 370 */ 371 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 372 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 373 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 374 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 375 break; 376 case 7: 377 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 378 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 379 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 380 break; 381 } 382 383 *cam1 &= kw_mask; 384 *cam0 = ~*cam1 & kw_mask; 385 } 386 387 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 388 u64 cam0, u64 cam1) 389 { 390 /* Similar to npc_get_keyword, but fills mcam_entry structure from 391 * CAM registers. 392 */ 393 switch (idx) { 394 case 0: 395 entry->kw[0] = cam1; 396 entry->kw_mask[0] = cam1 ^ cam0; 397 break; 398 case 1: 399 entry->kw[1] = cam1; 400 entry->kw_mask[1] = cam1 ^ cam0; 401 break; 402 case 2: 403 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 404 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 405 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 406 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 407 break; 408 case 3: 409 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 410 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 411 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 412 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 413 break; 414 case 4: 415 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 416 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 417 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 418 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 419 break; 420 case 5: 421 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 422 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 423 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 424 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 425 break; 426 case 6: 427 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 428 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 429 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 430 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 431 break; 432 case 7: 433 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 434 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 435 break; 436 } 437 } 438 439 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 440 int blkaddr, u16 pf_func) 441 { 442 int bank, nixlf, index; 443 u64 reg; 444 445 /* get ucast entry rule entry index */ 446 if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { 447 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n", 448 __func__, pf_func); 449 /* Action 0 is drop */ 450 return 0; 451 } 452 453 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 454 NIXLF_UCAST_ENTRY); 455 456 if (index < 0) { 457 dev_err(rvu->dev, 458 "%s: failed to get ucast entry pcifunc:0x%x\n", 459 __func__, pf_func); 460 /* Action 0 is drop */ 461 return 0; 462 } 463 464 bank = npc_get_bank(mcam, index); 465 index &= (mcam->banksize - 1); 466 467 if (is_cn20k(rvu->pdev)) 468 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 469 else 470 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 471 472 return rvu_read64(rvu, blkaddr, reg); 473 } 474 475 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 476 int blkaddr, int index, struct mcam_entry *entry, 477 bool *enable) 478 { 479 struct rvu_npc_mcam_rule *rule; 480 u16 owner, target_func; 481 struct rvu_pfvf *pfvf; 482 u64 rx_action; 483 484 owner = mcam->entry2pfvf_map[index]; 485 target_func = (entry->action >> 4) & 0xffff; 486 /* do nothing when target is LBK/PF or owner is not PF */ 487 if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) || 488 (owner & RVU_PFVF_FUNC_MASK) || 489 !(target_func & RVU_PFVF_FUNC_MASK)) 490 return; 491 492 /* save entry2target_pffunc */ 493 pfvf = rvu_get_pfvf(rvu, target_func); 494 mcam->entry2target_pffunc[index] = target_func; 495 496 /* don't enable rule when nixlf not attached or initialized */ 497 if (!(is_nixlf_attached(rvu, target_func) && 498 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 499 *enable = false; 500 501 /* fix up not needed for the rules added by user(ntuple filters) */ 502 list_for_each_entry(rule, &mcam->mcam_rules, list) { 503 if (rule->entry == index) 504 return; 505 } 506 507 /* AF modifies given action iff PF/VF has requested for it */ 508 if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT) 509 return; 510 511 /* copy VF default entry action to the VF mcam entry */ 512 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 513 target_func); 514 if (rx_action) 515 entry->action = rx_action; 516 } 517 518 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 519 int blkaddr, int index, u8 intf, 520 struct mcam_entry *entry, bool enable) 521 { 522 int bank = npc_get_bank(mcam, index); 523 int kw = 0, actbank, actindex; 524 u8 tx_intf_mask = ~intf & 0x3; 525 u8 tx_intf = intf; 526 u64 cam0, cam1; 527 528 actbank = bank; /* Save bank id, to set action later on */ 529 actindex = index; 530 index &= (mcam->banksize - 1); 531 532 /* Disable before mcam entry update */ 533 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 534 535 /* Clear mcam entry to avoid writes being suppressed by NPC */ 536 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 537 538 /* CAM1 takes the comparison value and 539 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 540 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 541 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 542 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 543 */ 544 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 545 /* Interface should be set in all banks */ 546 if (is_npc_intf_tx(intf)) { 547 /* Last bit must be set and rest don't care 548 * for TX interfaces 549 */ 550 tx_intf_mask = 0x1; 551 tx_intf = intf & tx_intf_mask; 552 tx_intf_mask = ~tx_intf & tx_intf_mask; 553 } 554 555 rvu_write64(rvu, blkaddr, 556 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 557 tx_intf); 558 rvu_write64(rvu, blkaddr, 559 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 560 tx_intf_mask); 561 562 /* Set the match key */ 563 npc_get_keyword(entry, kw, &cam0, &cam1); 564 rvu_write64(rvu, blkaddr, 565 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 566 rvu_write64(rvu, blkaddr, 567 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 568 569 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 570 rvu_write64(rvu, blkaddr, 571 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 572 rvu_write64(rvu, blkaddr, 573 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 574 } 575 576 /* PF installing VF rule */ 577 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 578 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 579 580 /* Set 'action' */ 581 rvu_write64(rvu, blkaddr, 582 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 583 584 /* Set TAG 'action' */ 585 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 586 entry->vtag_action); 587 588 /* Enable the entry */ 589 if (enable) 590 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 591 } 592 593 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 594 int blkaddr, u16 src, 595 struct mcam_entry *entry, u8 *intf, u8 *ena) 596 { 597 int sbank = npc_get_bank(mcam, src); 598 int bank, kw = 0; 599 u64 cam0, cam1; 600 601 src &= (mcam->banksize - 1); 602 bank = sbank; 603 604 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 605 cam1 = rvu_read64(rvu, blkaddr, 606 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 607 cam0 = rvu_read64(rvu, blkaddr, 608 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 609 npc_fill_entryword(entry, kw, cam0, cam1); 610 611 cam1 = rvu_read64(rvu, blkaddr, 612 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 613 cam0 = rvu_read64(rvu, blkaddr, 614 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 615 npc_fill_entryword(entry, kw + 1, cam0, cam1); 616 } 617 618 entry->action = rvu_read64(rvu, blkaddr, 619 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 620 entry->vtag_action = 621 rvu_read64(rvu, blkaddr, 622 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 623 *intf = rvu_read64(rvu, blkaddr, 624 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 625 *ena = rvu_read64(rvu, blkaddr, 626 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 627 } 628 629 static int npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 630 int blkaddr, u16 src, u16 dest) 631 { 632 int dbank = npc_get_bank(mcam, dest); 633 int sbank = npc_get_bank(mcam, src); 634 u64 cfg, sreg, dreg; 635 int bank, i; 636 637 if (is_cn20k(rvu->pdev)) 638 return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest); 639 640 src &= (mcam->banksize - 1); 641 dest &= (mcam->banksize - 1); 642 643 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 644 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 645 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 646 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 647 for (i = 0; i < 6; i++) { 648 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 649 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 650 } 651 } 652 653 /* Copy action */ 654 cfg = rvu_read64(rvu, blkaddr, 655 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 656 rvu_write64(rvu, blkaddr, 657 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 658 659 /* Copy TAG action */ 660 cfg = rvu_read64(rvu, blkaddr, 661 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 662 rvu_write64(rvu, blkaddr, 663 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 664 665 /* Enable or disable */ 666 cfg = rvu_read64(rvu, blkaddr, 667 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 668 rvu_write64(rvu, blkaddr, 669 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 670 return 0; 671 } 672 673 u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 674 int blkaddr, int index) 675 { 676 int bank = npc_get_bank(mcam, index); 677 u64 reg; 678 679 index &= (mcam->banksize - 1); 680 681 if (is_cn20k(rvu->pdev)) 682 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 683 else 684 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 685 return rvu_read64(rvu, blkaddr, reg); 686 } 687 688 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 689 int blkaddr, int index, u64 cfg) 690 { 691 int bank = npc_get_bank(mcam, index); 692 u64 reg; 693 694 index &= (mcam->banksize - 1); 695 696 if (is_cn20k(rvu->pdev)) 697 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 698 else 699 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 700 701 return rvu_write64(rvu, blkaddr, reg, cfg); 702 } 703 704 u32 rvu_get_cpt_chan_mask(struct rvu *rvu) 705 { 706 /* For cn10k the upper two bits of the channel number are 707 * cpt channel number. with masking out these bits in the 708 * mcam entry, same entry used for NIX will allow packets 709 * received from cpt for parsing. 710 */ 711 if (!is_rvu_otx2(rvu)) 712 return NIX_CHAN_CPT_X2P_MASK; 713 else 714 return 0xFFFu; 715 } 716 717 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 718 int nixlf, u64 chan, u8 *mac_addr) 719 { 720 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 721 struct npc_install_flow_req req = { 0 }; 722 struct npc_install_flow_rsp rsp = { 0 }; 723 struct npc_mcam *mcam = &rvu->hw->mcam; 724 struct nix_rx_action action = { 0 }; 725 int blkaddr, index; 726 727 /* AF's and SDP VFs work in promiscuous mode */ 728 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 729 return; 730 731 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 732 if (blkaddr < 0) 733 return; 734 735 /* Ucast rule should not be installed if DMAC 736 * extraction is not supported by the profile. 737 */ 738 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf)) 739 return; 740 741 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 742 nixlf, NIXLF_UCAST_ENTRY); 743 if (index < 0) { 744 dev_err(rvu->dev, 745 "%s: Error to get ucast entry for pcifunc=%#x\n", 746 __func__, pcifunc); 747 return; 748 } 749 750 /* Don't change the action if entry is already enabled 751 * Otherwise RSS action may get overwritten. 752 */ 753 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 754 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 755 blkaddr, index); 756 } else { 757 action.op = NIX_RX_ACTIONOP_UCAST; 758 action.pf_func = pcifunc; 759 } 760 761 req.default_rule = 1; 762 ether_addr_copy(req.packet.dmac, mac_addr); 763 eth_broadcast_addr((u8 *)&req.mask.dmac); 764 req.features = BIT_ULL(NPC_DMAC); 765 req.channel = chan; 766 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 767 req.intf = pfvf->nix_rx_intf; 768 req.op = action.op; 769 req.hdr.pcifunc = 0; /* AF is requester */ 770 req.vf = action.pf_func; 771 req.index = action.index; 772 req.match_id = action.match_id; 773 req.flow_key_alg = action.flow_key_alg; 774 775 if (is_cn20k(rvu->pdev)) 776 req.hw_prio = pfvf->hw_prio; 777 778 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 779 } 780 781 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 782 int nixlf, u64 chan, u8 chan_cnt) 783 { 784 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 785 struct npc_install_flow_req req = { 0 }; 786 struct npc_install_flow_rsp rsp = { 0 }; 787 struct npc_mcam *mcam = &rvu->hw->mcam; 788 struct rvu_hwinfo *hw = rvu->hw; 789 int blkaddr, ucast_idx, index; 790 struct nix_rx_action action = { 0 }; 791 u64 relaxed_mask; 792 u8 flow_key_alg; 793 794 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 795 return; 796 797 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 798 if (blkaddr < 0) 799 return; 800 801 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 802 nixlf, NIXLF_PROMISC_ENTRY); 803 804 /* In cn20k, default indexes are installed only for CGX mapped 805 * and lbk interfaces 806 */ 807 if (is_cgx_vf(rvu, pcifunc)) 808 index = npc_get_nixlf_mcam_index(mcam, 809 pcifunc & ~RVU_PFVF_FUNC_MASK, 810 nixlf, NIXLF_PROMISC_ENTRY); 811 812 if (index < 0) { 813 dev_err(rvu->dev, 814 "%s: Error to get promisc entry for pcifunc=%#x\n", 815 __func__, pcifunc); 816 return; 817 } 818 819 /* If the corresponding PF's ucast action is RSS, 820 * use the same action for promisc also 821 * Please note that for lbk(s) "index" and "ucast_idx" 822 * will be same. 823 */ 824 if (is_lbk_vf(rvu, pcifunc)) 825 ucast_idx = index; 826 else 827 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 828 nixlf, NIXLF_UCAST_ENTRY); 829 if (ucast_idx < 0) { 830 dev_err(rvu->dev, 831 "%s: Error to get ucast/promisc entry for pcifunc=%#x\n", 832 __func__, pcifunc); 833 return; 834 } 835 836 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 837 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 838 blkaddr, ucast_idx); 839 840 if (action.op != NIX_RX_ACTIONOP_RSS) { 841 *(u64 *)&action = 0; 842 action.op = NIX_RX_ACTIONOP_UCAST; 843 } 844 845 flow_key_alg = action.flow_key_alg; 846 847 /* RX_ACTION set to MCAST for CGX PF's */ 848 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 849 is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc))) { 850 *(u64 *)&action = 0; 851 action.op = NIX_RX_ACTIONOP_MCAST; 852 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 853 action.index = pfvf->promisc_mce_idx; 854 } 855 856 /* For cn10k the upper two bits of the channel number are 857 * cpt channel number. with masking out these bits in the 858 * mcam entry, same entry used for NIX will allow packets 859 * received from cpt for parsing. 860 */ 861 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 862 863 if (chan_cnt > 1) { 864 if (!is_power_of_2(chan_cnt)) { 865 dev_err(rvu->dev, 866 "%s: channel count more than 1, must be power of 2\n", __func__); 867 return; 868 } 869 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 870 ilog2(chan_cnt)); 871 req.chan_mask &= relaxed_mask; 872 } 873 874 req.channel = chan; 875 req.intf = pfvf->nix_rx_intf; 876 req.entry = index; 877 req.op = action.op; 878 req.hdr.pcifunc = 0; /* AF is requester */ 879 req.vf = pcifunc; 880 req.index = action.index; 881 req.match_id = action.match_id; 882 req.flow_key_alg = flow_key_alg; 883 884 if (is_cn20k(rvu->pdev)) 885 req.hw_prio = pfvf->hw_prio; 886 887 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 888 } 889 890 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 891 int nixlf, bool enable) 892 { 893 struct npc_mcam *mcam = &rvu->hw->mcam; 894 int blkaddr, index; 895 896 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 897 if (blkaddr < 0) 898 return; 899 900 /* Get 'pcifunc' of PF device */ 901 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 902 903 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 904 nixlf, NIXLF_PROMISC_ENTRY); 905 906 if (index < 0) { 907 dev_err(rvu->dev, 908 "%s: Error to get promisc entry for pcifunc=%#x\n", 909 __func__, pcifunc); 910 return; 911 } 912 913 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 914 } 915 916 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 917 int nixlf, u64 chan) 918 { 919 struct rvu_pfvf *pfvf; 920 struct npc_install_flow_req req = { 0 }; 921 struct npc_install_flow_rsp rsp = { 0 }; 922 struct npc_mcam *mcam = &rvu->hw->mcam; 923 struct rvu_hwinfo *hw = rvu->hw; 924 int blkaddr, index; 925 926 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 927 if (blkaddr < 0) 928 return; 929 930 /* Skip LBK VFs */ 931 if (is_lbk_vf(rvu, pcifunc)) 932 return; 933 934 /* If pkt replication is not supported, 935 * then only PF is allowed to add a bcast match entry. 936 */ 937 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 938 return; 939 940 /* Get 'pcifunc' of PF device */ 941 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 942 pfvf = rvu_get_pfvf(rvu, pcifunc); 943 944 /* Bcast rule should not be installed if both DMAC 945 * and LXMB extraction is not supported by the profile. 946 */ 947 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 948 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 949 return; 950 951 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 952 nixlf, NIXLF_BCAST_ENTRY); 953 if (index < 0) { 954 dev_err(rvu->dev, 955 "%s: Error to get bcast entry for pcifunc=%#x\n", 956 __func__, pcifunc); 957 return; 958 } 959 960 if (!hw->cap.nix_rx_multicast) { 961 /* Early silicon doesn't support pkt replication, 962 * so install entry with UCAST action, so that PF 963 * receives all broadcast packets. 964 */ 965 req.op = NIX_RX_ACTIONOP_UCAST; 966 } else { 967 req.op = NIX_RX_ACTIONOP_MCAST; 968 req.index = pfvf->bcast_mce_idx; 969 } 970 971 eth_broadcast_addr((u8 *)&req.packet.dmac); 972 eth_broadcast_addr((u8 *)&req.mask.dmac); 973 req.features = BIT_ULL(NPC_DMAC); 974 req.channel = chan; 975 req.chan_mask = 0xFFFU; 976 req.intf = pfvf->nix_rx_intf; 977 req.entry = index; 978 req.hdr.pcifunc = 0; /* AF is requester */ 979 req.vf = pcifunc; 980 981 if (is_cn20k(rvu->pdev)) 982 req.hw_prio = pfvf->hw_prio; 983 984 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 985 } 986 987 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 988 u64 chan) 989 { 990 struct npc_install_flow_req req = { 0 }; 991 struct npc_install_flow_rsp rsp = { 0 }; 992 struct npc_mcam *mcam = &rvu->hw->mcam; 993 struct rvu_hwinfo *hw = rvu->hw; 994 int blkaddr, ucast_idx, index; 995 u8 mac_addr[ETH_ALEN] = { 0 }; 996 struct nix_rx_action action = { 0 }; 997 struct rvu_pfvf *pfvf; 998 u8 flow_key_alg; 999 u16 vf_func; 1000 1001 /* Only CGX PF/VF can add allmulticast entry */ 1002 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 1003 return; 1004 1005 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1006 if (blkaddr < 0) 1007 return; 1008 1009 /* Get 'pcifunc' of PF device */ 1010 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 1011 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 1012 pfvf = rvu_get_pfvf(rvu, pcifunc); 1013 1014 /* Mcast rule should not be installed if both DMAC 1015 * and LXMB extraction is not supported by the profile. 1016 */ 1017 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 1018 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 1019 return; 1020 1021 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1022 nixlf, NIXLF_ALLMULTI_ENTRY); 1023 if (index < 0) { 1024 dev_err(rvu->dev, 1025 "%s: Error to get mcast entry for pcifunc=%#x\n", 1026 __func__, pcifunc); 1027 return; 1028 } 1029 1030 /* If the corresponding PF's ucast action is RSS, 1031 * use the same action for multicast entry also 1032 */ 1033 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 1034 nixlf, NIXLF_UCAST_ENTRY); 1035 if (ucast_idx < 0) { 1036 dev_err(rvu->dev, 1037 "%s: Error to get ucast entry for pcifunc=%#x\n", 1038 __func__, pcifunc); 1039 return; 1040 } 1041 1042 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 1043 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 1044 blkaddr, ucast_idx); 1045 1046 flow_key_alg = action.flow_key_alg; 1047 if (action.op != NIX_RX_ACTIONOP_RSS) { 1048 *(u64 *)&action = 0; 1049 action.op = NIX_RX_ACTIONOP_UCAST; 1050 action.pf_func = pcifunc; 1051 } 1052 1053 /* RX_ACTION set to MCAST for CGX PF's */ 1054 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 1055 *(u64 *)&action = 0; 1056 action.op = NIX_RX_ACTIONOP_MCAST; 1057 action.index = pfvf->mcast_mce_idx; 1058 } 1059 1060 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 1061 ether_addr_copy(req.packet.dmac, mac_addr); 1062 ether_addr_copy(req.mask.dmac, mac_addr); 1063 req.features = BIT_ULL(NPC_DMAC); 1064 1065 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 1066 req.channel = chan; 1067 req.intf = pfvf->nix_rx_intf; 1068 req.entry = index; 1069 req.op = action.op; 1070 req.hdr.pcifunc = 0; /* AF is requester */ 1071 req.vf = pcifunc | vf_func; 1072 req.index = action.index; 1073 req.match_id = action.match_id; 1074 req.flow_key_alg = flow_key_alg; 1075 1076 if (is_cn20k(rvu->pdev)) 1077 req.hw_prio = pfvf->hw_prio; 1078 1079 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 1080 } 1081 1082 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 1083 bool enable) 1084 { 1085 struct npc_mcam *mcam = &rvu->hw->mcam; 1086 int blkaddr, index; 1087 1088 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1089 if (blkaddr < 0) 1090 return; 1091 1092 /* Get 'pcifunc' of PF device */ 1093 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 1094 1095 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 1096 NIXLF_ALLMULTI_ENTRY); 1097 if (index < 0) { 1098 dev_err(rvu->dev, 1099 "%s: Error to get mcast entry for pcifunc=%#x\n", 1100 __func__, pcifunc); 1101 return; 1102 } 1103 1104 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1105 } 1106 1107 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 1108 int blkaddr, u16 pcifunc, u64 rx_action) 1109 { 1110 int actindex, index, bank, entry; 1111 struct rvu_npc_mcam_rule *rule; 1112 bool enable, update; 1113 1114 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 1115 return; 1116 1117 mutex_lock(&mcam->lock); 1118 for (index = 0; index < mcam->bmap_entries; index++) { 1119 if (mcam->entry2target_pffunc[index] != pcifunc) 1120 continue; 1121 update = true; 1122 /* update not needed for the rules added via ntuple filters */ 1123 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1124 if (rule->entry == index) 1125 update = false; 1126 } 1127 if (!update) 1128 continue; 1129 bank = npc_get_bank(mcam, index); 1130 actindex = index; 1131 entry = index & (mcam->banksize - 1); 1132 1133 /* read vf flow entry enable status */ 1134 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 1135 actindex); 1136 /* disable before mcam entry update */ 1137 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 1138 false); 1139 1140 /* update 'action' */ 1141 if (is_cn20k(rvu->pdev)) 1142 rvu_write64(rvu, blkaddr, 1143 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry, 1144 bank, 1145 0), 1146 rx_action); 1147 else 1148 rvu_write64(rvu, blkaddr, 1149 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 1150 rx_action); 1151 1152 if (enable) 1153 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1154 actindex, true); 1155 } 1156 mutex_unlock(&mcam->lock); 1157 } 1158 1159 static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action, 1160 struct rvu_pfvf *pfvf, int mcam_index, int blkaddr, 1161 int alg_idx) 1162 1163 { 1164 struct npc_mcam *mcam = &rvu->hw->mcam; 1165 struct rvu_hwinfo *hw = rvu->hw; 1166 int bank, op_rss; 1167 u64 reg; 1168 1169 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) 1170 return; 1171 1172 op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list); 1173 1174 bank = npc_get_bank(mcam, mcam_index); 1175 mcam_index &= (mcam->banksize - 1); 1176 1177 if (is_cn20k(rvu->pdev)) 1178 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index, 1179 bank, 0); 1180 else 1181 reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank); 1182 1183 /* If Rx action is MCAST update only RSS algorithm index */ 1184 if (!op_rss) { 1185 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1186 1187 action.flow_key_alg = alg_idx; 1188 } 1189 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1190 } 1191 1192 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 1193 int group, int alg_idx, int mcam_index) 1194 { 1195 struct npc_mcam *mcam = &rvu->hw->mcam; 1196 struct nix_rx_action action; 1197 int blkaddr, index, bank; 1198 struct rvu_pfvf *pfvf; 1199 u64 reg; 1200 1201 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1202 if (blkaddr < 0) 1203 return; 1204 1205 /* Check if this is for reserved default entry */ 1206 if (mcam_index < 0) { 1207 if (group != DEFAULT_RSS_CONTEXT_GROUP) 1208 return; 1209 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1210 nixlf, NIXLF_UCAST_ENTRY); 1211 } else { 1212 /* TODO: validate this mcam index */ 1213 index = mcam_index; 1214 } 1215 1216 if (index < 0 || index >= mcam->total_entries) { 1217 dev_err(rvu->dev, 1218 "%s: Invalid mcam index, pcifunc=%#x\n", 1219 __func__, pcifunc); 1220 return; 1221 } 1222 1223 bank = npc_get_bank(mcam, index); 1224 index &= (mcam->banksize - 1); 1225 1226 if (is_cn20k(rvu->pdev)) 1227 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 1228 else 1229 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 1230 1231 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1232 /* Ignore if no action was set earlier */ 1233 if (!*(u64 *)&action) 1234 return; 1235 1236 action.op = NIX_RX_ACTIONOP_RSS; 1237 action.pf_func = pcifunc; 1238 action.index = group; 1239 action.flow_key_alg = alg_idx; 1240 1241 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1242 /* update the VF flow rule action with the VF default entry action */ 1243 if (mcam_index < 0) 1244 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1245 *(u64 *)&action); 1246 1247 /* update the action change in default rule */ 1248 pfvf = rvu_get_pfvf(rvu, pcifunc); 1249 if (pfvf->def_ucast_rule) 1250 pfvf->def_ucast_rule->rx_action = action; 1251 1252 if (mcam_index < 0) { 1253 /* update the VF flow rule action with the VF default 1254 * entry action 1255 */ 1256 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1257 *(u64 *)&action); 1258 1259 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1260 nixlf, NIXLF_PROMISC_ENTRY); 1261 1262 /* If PF's promiscuous entry is enabled, 1263 * Set RSS action for that entry as well 1264 */ 1265 if (index >= 0) 1266 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1267 blkaddr, alg_idx); 1268 1269 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1270 nixlf, NIXLF_ALLMULTI_ENTRY); 1271 /* If PF's allmulti entry is enabled, 1272 * Set RSS action for that entry as well 1273 */ 1274 if (index >= 0) 1275 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1276 blkaddr, alg_idx); 1277 } 1278 } 1279 1280 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1281 int nixlf, int type, bool enable) 1282 { 1283 struct npc_mcam *mcam = &rvu->hw->mcam; 1284 struct rvu_hwinfo *hw = rvu->hw; 1285 struct nix_mce_list *mce_list; 1286 int index, blkaddr, mce_idx; 1287 struct rvu_pfvf *pfvf; 1288 u16 ptr[4]; 1289 1290 /* multicast pkt replication is not enabled for AF's VFs & SDP links */ 1291 if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(rvu, pcifunc)) 1292 return; 1293 1294 /* In cn20k, only CGX mapped devices have default MCAST entry */ 1295 if (is_cn20k(rvu->pdev) && 1296 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 1297 &ptr[2], &ptr[3])) 1298 return; 1299 1300 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1301 if (blkaddr < 0) 1302 return; 1303 1304 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1305 nixlf, type); 1306 if (index < 0) { 1307 dev_err(rvu->dev, 1308 "%s: Error to get entry for pcifunc=%#x, type=%u\n", 1309 __func__, pcifunc, type); 1310 return; 1311 } 1312 1313 /* disable MCAM entry when packet replication is not supported by hw */ 1314 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1315 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1316 return; 1317 } 1318 1319 /* return incase mce list is not enabled */ 1320 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1321 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1322 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1323 return; 1324 1325 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1326 1327 nix_update_mce_list(rvu, pcifunc, mce_list, 1328 mce_idx, index, enable); 1329 if (enable) 1330 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1331 } 1332 1333 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1334 int nixlf, bool enable) 1335 { 1336 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1337 struct npc_mcam *mcam = &rvu->hw->mcam; 1338 int index, blkaddr; 1339 u16 ptr[4]; 1340 1341 /* only CGX or LBK interfaces have default entries */ 1342 if (is_cn20k(rvu->pdev) && 1343 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 1344 &ptr[2], &ptr[3])) 1345 return; 1346 1347 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1348 if (blkaddr < 0) 1349 return; 1350 1351 /* Ucast MCAM match entry of this PF/VF */ 1352 if (npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), 1353 pfvf->nix_rx_intf)) { 1354 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1355 nixlf, NIXLF_UCAST_ENTRY); 1356 if (index < 0) { 1357 dev_err(rvu->dev, 1358 "%s: Error to get ucast entry for pcifunc=%#x\n", 1359 __func__, pcifunc); 1360 return; 1361 } 1362 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1363 } 1364 1365 /* Nothing to do for VFs, on platforms where pkt replication 1366 * is not supported 1367 */ 1368 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1369 return; 1370 1371 /* add/delete pf_func to broadcast MCE list */ 1372 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1373 NIXLF_BCAST_ENTRY, enable); 1374 } 1375 1376 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1377 { 1378 if (nixlf < 0) 1379 return; 1380 1381 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1382 1383 /* Delete multicast and promisc MCAM entries */ 1384 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1385 NIXLF_ALLMULTI_ENTRY, false); 1386 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1387 NIXLF_PROMISC_ENTRY, false); 1388 } 1389 1390 bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable) 1391 { 1392 int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1393 struct npc_mcam *mcam = &rvu->hw->mcam; 1394 struct rvu_npc_mcam_rule *rule, *tmp; 1395 1396 mutex_lock(&mcam->lock); 1397 1398 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1399 if (rule->intf != intf) 1400 continue; 1401 1402 if (rule->entry != entry) 1403 continue; 1404 1405 rule->enable = enable; 1406 mutex_unlock(&mcam->lock); 1407 1408 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1409 entry, enable); 1410 1411 return true; 1412 } 1413 1414 mutex_unlock(&mcam->lock); 1415 return false; 1416 } 1417 1418 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1419 { 1420 if (nixlf < 0) 1421 return; 1422 1423 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1424 * in set_rx_mode mbox handler. 1425 */ 1426 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1427 } 1428 1429 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1430 { 1431 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1432 struct npc_mcam *mcam = &rvu->hw->mcam; 1433 struct rvu_npc_mcam_rule *rule, *tmp; 1434 int blkaddr; 1435 1436 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1437 if (blkaddr < 0) 1438 return; 1439 1440 mutex_lock(&mcam->lock); 1441 1442 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1443 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1444 if (is_npc_intf_rx(rule->intf) && 1445 rule->rx_action.pf_func == pcifunc && 1446 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1447 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1448 rule->entry, false); 1449 rule->enable = false; 1450 /* Indicate that default rule is disabled */ 1451 if (rule->default_rule) { 1452 pfvf->def_ucast_rule = NULL; 1453 list_del(&rule->list); 1454 kfree(rule); 1455 } 1456 } 1457 } 1458 1459 mutex_unlock(&mcam->lock); 1460 1461 npc_mcam_disable_flows(rvu, pcifunc); 1462 1463 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1464 } 1465 1466 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1467 { 1468 struct npc_mcam *mcam = &rvu->hw->mcam; 1469 struct rvu_npc_mcam_rule *rule, *tmp; 1470 int blkaddr; 1471 1472 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1473 if (blkaddr < 0) 1474 return; 1475 1476 mutex_lock(&mcam->lock); 1477 1478 /* Free all MCAM entries owned by this 'pcifunc' */ 1479 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1480 1481 /* Free all MCAM counters owned by this 'pcifunc' */ 1482 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1483 1484 /* Delete MCAM entries owned by this 'pcifunc' */ 1485 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1486 if (rule->owner == pcifunc && !rule->default_rule) { 1487 list_del(&rule->list); 1488 kfree(rule); 1489 } 1490 } 1491 1492 mutex_unlock(&mcam->lock); 1493 1494 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1495 } 1496 1497 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1498 const struct npc_mcam_kex *mkex, 1499 u8 intf) 1500 { 1501 int lid, lt, ld, fl; 1502 1503 if (is_npc_intf_tx(intf)) 1504 return; 1505 1506 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1507 mkex->keyx_cfg[NIX_INTF_RX]); 1508 1509 /* Program LDATA */ 1510 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1511 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1512 for (ld = 0; ld < NPC_MAX_LD; ld++) 1513 SET_KEX_LD(intf, lid, lt, ld, 1514 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1515 [lid][lt][ld]); 1516 } 1517 } 1518 /* Program LFLAGS */ 1519 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1520 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1521 SET_KEX_LDFLAGS(intf, ld, fl, 1522 mkex->intf_ld_flags[NIX_INTF_RX] 1523 [ld][fl]); 1524 } 1525 } 1526 1527 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1528 const struct npc_mcam_kex *mkex, 1529 u8 intf) 1530 { 1531 int lid, lt, ld, fl; 1532 1533 if (is_npc_intf_rx(intf)) 1534 return; 1535 1536 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1537 mkex->keyx_cfg[NIX_INTF_TX]); 1538 1539 /* Program LDATA */ 1540 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1541 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1542 for (ld = 0; ld < NPC_MAX_LD; ld++) 1543 SET_KEX_LD(intf, lid, lt, ld, 1544 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1545 [lid][lt][ld]); 1546 } 1547 } 1548 /* Program LFLAGS */ 1549 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1550 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1551 SET_KEX_LDFLAGS(intf, ld, fl, 1552 mkex->intf_ld_flags[NIX_INTF_TX] 1553 [ld][fl]); 1554 } 1555 } 1556 1557 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1558 const struct npc_mcam_kex *mkex) 1559 { 1560 struct rvu_hwinfo *hw = rvu->hw; 1561 u8 intf; 1562 int ld; 1563 1564 for (ld = 0; ld < NPC_MAX_LD; ld++) 1565 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1566 mkex->kex_ld_flags[ld]); 1567 1568 for (intf = 0; intf < hw->npc_intfs; intf++) { 1569 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1570 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1571 } 1572 1573 /* Programme mkex hash profile */ 1574 npc_program_mkex_hash(rvu, blkaddr); 1575 } 1576 1577 int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1578 u64 *size) 1579 { 1580 u64 prfl_addr, prfl_sz; 1581 1582 if (!rvu->fwdata) 1583 return -EINVAL; 1584 1585 prfl_addr = rvu->fwdata->mcam_addr; 1586 prfl_sz = rvu->fwdata->mcam_sz; 1587 1588 if (!prfl_addr || !prfl_sz) 1589 return -EINVAL; 1590 1591 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1592 if (!(*prfl_img_addr)) 1593 return -ENOMEM; 1594 1595 *size = prfl_sz; 1596 1597 return 0; 1598 } 1599 1600 /* strtoull of "mkexprof" with base:36 */ 1601 #define MKEX_END_SIGN 0xdeadbeef 1602 1603 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1604 const char *mkex_profile) 1605 { 1606 struct device *dev = &rvu->pdev->dev; 1607 struct npc_mcam_kex *mcam_kex; 1608 void __iomem *mkex_prfl_addr = NULL; 1609 u64 prfl_sz; 1610 int ret; 1611 1612 /* If user not selected mkex profile */ 1613 if (rvu->kpu_fwdata_sz || 1614 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1615 goto program_mkex; 1616 1617 /* Setting up the mapping for mkex profile image */ 1618 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1619 if (ret < 0) 1620 goto program_mkex; 1621 1622 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1623 1624 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1625 /* Compare with mkex mod_param name string */ 1626 if (mcam_kex->mkex_sign == MKEX_SIGN && 1627 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1628 /* Due to an errata (35786) in A0/B0 pass silicon, 1629 * parse nibble enable configuration has to be 1630 * identical for both Rx and Tx interfaces. 1631 */ 1632 if (!is_rvu_96xx_B0(rvu) || 1633 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1634 rvu->kpu.mcam_kex_prfl.mkex = mcam_kex; 1635 goto program_mkex; 1636 } 1637 1638 mcam_kex++; 1639 prfl_sz -= sizeof(struct npc_mcam_kex); 1640 } 1641 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1642 1643 program_mkex: 1644 dev_info(rvu->dev, "Using %s mkex profile\n", 1645 rvu->kpu.mcam_kex_prfl.mkex->name); 1646 /* Program selected mkex profile */ 1647 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex); 1648 if (mkex_prfl_addr) 1649 iounmap(mkex_prfl_addr); 1650 } 1651 1652 void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1653 const struct npc_kpu_profile_action *kpuaction, 1654 int kpu, int entry, bool pkind) 1655 { 1656 struct npc_kpu_action0 action0 = {0}; 1657 struct npc_kpu_action1 action1 = {0}; 1658 u64 reg; 1659 1660 action1.errlev = kpuaction->errlev; 1661 action1.errcode = kpuaction->errcode; 1662 action1.dp0_offset = kpuaction->dp0_offset; 1663 action1.dp1_offset = kpuaction->dp1_offset; 1664 action1.dp2_offset = kpuaction->dp2_offset; 1665 1666 if (pkind) 1667 reg = NPC_AF_PKINDX_ACTION1(entry); 1668 else 1669 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1670 1671 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1672 1673 action0.byp_count = kpuaction->bypass_count; 1674 action0.capture_ena = kpuaction->cap_ena; 1675 action0.parse_done = kpuaction->parse_done; 1676 action0.next_state = kpuaction->next_state; 1677 action0.capture_lid = kpuaction->lid; 1678 action0.capture_ltype = kpuaction->ltype; 1679 action0.capture_flags = kpuaction->flags; 1680 action0.ptr_advance = kpuaction->ptr_advance; 1681 action0.var_len_offset = kpuaction->offset; 1682 action0.var_len_mask = kpuaction->mask; 1683 action0.var_len_right = kpuaction->right; 1684 action0.var_len_shift = kpuaction->shift; 1685 1686 if (pkind) 1687 reg = NPC_AF_PKINDX_ACTION0(entry); 1688 else 1689 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1690 1691 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1692 } 1693 1694 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1695 const struct npc_kpu_profile_cam *kpucam, 1696 int kpu, int entry) 1697 { 1698 const struct npc_kpu_profile_cam2 *kpucam2 = (void *)kpucam; 1699 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1700 struct npc_kpu_cam cam0 = {0}; 1701 struct npc_kpu_cam cam1 = {0}; 1702 u64 *val = (u64 *)&cam1; 1703 u64 *mask = (u64 *)&cam0; 1704 1705 cam1.state = kpucam->state & kpucam->state_mask; 1706 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1707 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1708 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1709 1710 cam0.state = ~kpucam->state & kpucam->state_mask; 1711 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1712 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1713 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1714 1715 if (profile->from_fs) { 1716 u8 ptype = kpucam2->ptype; 1717 u8 pmask = kpucam2->ptype_mask; 1718 1719 *val |= FIELD_PREP(GENMASK_ULL(57, 56), ptype & pmask); 1720 *mask |= FIELD_PREP(GENMASK_ULL(57, 56), ~ptype & pmask); 1721 } 1722 1723 rvu_write64(rvu, blkaddr, 1724 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1725 rvu_write64(rvu, blkaddr, 1726 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1727 } 1728 1729 u64 npc_enable_mask(int count) 1730 { 1731 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1732 } 1733 1734 struct npc_kpu_profile_action * 1735 npc_get_ikpu_nth_entry(struct rvu *rvu, int n) 1736 { 1737 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1738 1739 if (profile->from_fs) 1740 return &profile->ikpu2[n]; 1741 1742 return &profile->ikpu[n]; 1743 } 1744 1745 int 1746 npc_get_num_kpu_cam_entries(struct rvu *rvu, 1747 const struct npc_kpu_profile *kpu_pfl) 1748 { 1749 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1750 1751 if (profile->from_fs) 1752 return kpu_pfl->cam_entries2; 1753 1754 return kpu_pfl->cam_entries; 1755 } 1756 1757 struct npc_kpu_profile_cam * 1758 npc_get_kpu_cam_nth_entry(struct rvu *rvu, 1759 const struct npc_kpu_profile *kpu_pfl, int n) 1760 { 1761 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1762 1763 if (profile->from_fs) 1764 return (void *)&kpu_pfl->cam2[n]; 1765 1766 return (void *)&kpu_pfl->cam[n]; 1767 } 1768 1769 int 1770 npc_get_num_kpu_action_entries(struct rvu *rvu, 1771 const struct npc_kpu_profile *kpu_pfl) 1772 { 1773 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1774 1775 if (profile->from_fs) 1776 return kpu_pfl->action_entries2; 1777 1778 return kpu_pfl->action_entries; 1779 } 1780 1781 struct npc_kpu_profile_action * 1782 npc_get_kpu_action_nth_entry(struct rvu *rvu, 1783 const struct npc_kpu_profile *kpu_pfl, 1784 int n) 1785 { 1786 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1787 1788 if (profile->from_fs) 1789 return (void *)&kpu_pfl->action2[n]; 1790 1791 return (void *)&kpu_pfl->action[n]; 1792 } 1793 1794 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1795 const struct npc_kpu_profile *profile) 1796 { 1797 int num_cam_entries, num_action_entries; 1798 int entry, num_entries, max_entries; 1799 u64 entry_mask; 1800 1801 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile); 1802 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile); 1803 1804 if (num_cam_entries != num_action_entries) { 1805 dev_err(rvu->dev, 1806 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1807 kpu, num_cam_entries, num_action_entries); 1808 } 1809 1810 max_entries = rvu->hw->npc_kpu_entries; 1811 1812 WARN(num_cam_entries > max_entries, 1813 "KPU%u: err: hw max entries=%u, input entries=%u\n", 1814 kpu, rvu->hw->npc_kpu_entries, num_cam_entries); 1815 1816 /* Program CAM match entries for previous KPU extracted data */ 1817 num_entries = min_t(int, num_cam_entries, max_entries); 1818 for (entry = 0; entry < num_entries; entry++) 1819 npc_config_kpucam(rvu, blkaddr, 1820 (void *)npc_get_kpu_cam_nth_entry(rvu, profile, entry), 1821 kpu, entry); 1822 1823 /* Program this KPU's actions */ 1824 num_entries = min_t(int, num_action_entries, max_entries); 1825 for (entry = 0; entry < num_entries; entry++) 1826 npc_config_kpuaction(rvu, blkaddr, 1827 (void *)npc_get_kpu_action_nth_entry(rvu, profile, entry), 1828 kpu, entry, false); 1829 1830 /* Enable all programmed entries */ 1831 num_entries = min_t(int, num_action_entries, num_cam_entries); 1832 entry_mask = npc_enable_mask(num_entries); 1833 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1834 if (!rvu->kpu.custom) 1835 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1836 rvu_write64(rvu, blkaddr, 1837 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1838 if (num_entries > 64) { 1839 rvu_write64(rvu, blkaddr, 1840 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1841 npc_enable_mask(num_entries - 64)); 1842 } 1843 1844 /* Enable this KPU */ 1845 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1846 } 1847 1848 static void npc_prepare_default_kpu(struct rvu *rvu, 1849 struct npc_kpu_profile_adapter *profile) 1850 { 1851 profile->custom = 0; 1852 profile->name = def_pfl_name; 1853 profile->version = NPC_KPU_PROFILE_VER; 1854 profile->ikpu = ikpu_action_entries; 1855 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1856 profile->kpu = npc_kpu_profiles; 1857 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1858 profile->lt_def = &npc_lt_defaults; 1859 profile->mkex_hash = &npc_mkex_hash_default; 1860 1861 if (!is_cn20k(rvu->pdev)) { 1862 profile->mcam_kex_prfl.mkex = &npc_mkex_default; 1863 return; 1864 } 1865 1866 profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 1867 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6; 1868 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0; 1869 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5; 1870 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1; 1871 1872 npc_cn20k_update_action_entries_n_flags(rvu, profile); 1873 } 1874 1875 static int npc_alloc_kpu_cam2_n_action2(struct rvu *rvu, int kpu_num, 1876 int num_entries) 1877 { 1878 struct npc_kpu_profile_adapter *adapter = &rvu->kpu; 1879 struct npc_kpu_profile *kpu; 1880 1881 kpu = &adapter->kpu[kpu_num]; 1882 1883 kpu->cam2 = devm_kcalloc(rvu->dev, num_entries, 1884 sizeof(*kpu->cam2), GFP_KERNEL); 1885 if (!kpu->cam2) 1886 return -ENOMEM; 1887 1888 kpu->action2 = devm_kcalloc(rvu->dev, num_entries, 1889 sizeof(*kpu->action2), GFP_KERNEL); 1890 if (!kpu->action2) 1891 return -ENOMEM; 1892 1893 return 0; 1894 } 1895 1896 static int npc_apply_custom_kpu_from_fw(struct rvu *rvu, 1897 struct npc_kpu_profile_adapter *profile) 1898 { 1899 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1900 const struct npc_kpu_profile_fwdata *fw; 1901 struct npc_kpu_profile_action *action; 1902 struct npc_kpu_profile_cam *cam; 1903 struct npc_kpu_fwdata *fw_kpu; 1904 int entries, entry, kpu; 1905 1906 fw = rvu->kpu_fwdata; 1907 1908 for (kpu = 0; kpu < fw->kpus; kpu++) { 1909 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1910 dev_warn(rvu->dev, 1911 "Profile size mismatch on KPU%i parsing\n", 1912 kpu + 1); 1913 return -EINVAL; 1914 } 1915 1916 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1917 if (fw_kpu->entries < 0) { 1918 dev_warn(rvu->dev, 1919 "Profile entries is negative on KPU%i parsing\n", 1920 kpu + 1); 1921 return -EINVAL; 1922 } 1923 1924 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1925 dev_warn(rvu->dev, 1926 "Too many custom entries on KPU%d: %d > %d\n", 1927 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1928 entries = min_t(int, fw_kpu->entries, KPU_MAX_CST_ENT); 1929 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1930 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1931 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1932 offset += fw_kpu->entries * sizeof(*action); 1933 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1934 dev_warn(rvu->dev, 1935 "Profile size mismatch on KPU%i parsing.\n", 1936 kpu + 1); 1937 return -EINVAL; 1938 } 1939 for (entry = 0; entry < entries; entry++) { 1940 profile->kpu[kpu].cam[entry] = cam[entry]; 1941 profile->kpu[kpu].action[entry] = action[entry]; 1942 } 1943 } 1944 1945 return 0; 1946 } 1947 1948 static int npc_apply_custom_kpu_from_fs(struct rvu *rvu, 1949 struct npc_kpu_profile_adapter *profile) 1950 { 1951 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1952 const struct npc_kpu_profile_fwdata *fw; 1953 struct npc_kpu_profile_action *action; 1954 struct npc_kpu_profile_cam2 *cam2; 1955 struct npc_kpu_fwdata *fw_kpu; 1956 int entries, ret, entry, kpu; 1957 1958 fw = rvu->kpu_fwdata; 1959 1960 /* Binary blob contains ikpu actions entries at start of data[0] */ 1961 profile->ikpu2 = devm_kcalloc(rvu->dev, 1, 1962 sizeof(ikpu_action_entries), 1963 GFP_KERNEL); 1964 if (!profile->ikpu2) 1965 return -ENOMEM; 1966 1967 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1968 1969 if (rvu->kpu_fwdata_sz < hdr_sz + sizeof(ikpu_action_entries)) 1970 return -EINVAL; 1971 1972 /* The firmware layout does dependent on the internal size of 1973 * ikpu_action_entries. 1974 */ 1975 memcpy((void *)profile->ikpu2, action, sizeof(ikpu_action_entries)); 1976 offset += sizeof(ikpu_action_entries); 1977 1978 for (kpu = 0; kpu < fw->kpus; kpu++) { 1979 if (rvu->kpu_fwdata_sz < hdr_sz + offset + sizeof(*fw_kpu)) { 1980 dev_warn(rvu->dev, 1981 "profile size mismatch on kpu%i parsing\n", 1982 kpu + 1); 1983 return -EINVAL; 1984 } 1985 1986 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1987 if (fw_kpu->entries <= 0) { 1988 dev_warn(rvu->dev, 1989 "Invalid kpu entries on KPU%d\n", kpu); 1990 return -EINVAL; 1991 } 1992 1993 entries = min_t(int, fw_kpu->entries, rvu->hw->npc_kpu_entries); 1994 dev_info(rvu->dev, 1995 "Loading %u entries on KPU%d\n", entries, kpu); 1996 1997 cam2 = (struct npc_kpu_profile_cam2 *)fw_kpu->data; 1998 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam2); 1999 action = (struct npc_kpu_profile_action *)(fw->data + offset); 2000 offset += fw_kpu->entries * sizeof(*action); 2001 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 2002 dev_warn(rvu->dev, 2003 "profile size mismatch on kpu%i parsing.\n", 2004 kpu + 1); 2005 return -EINVAL; 2006 } 2007 2008 profile->kpu[kpu].cam_entries2 = entries; 2009 profile->kpu[kpu].action_entries2 = entries; 2010 ret = npc_alloc_kpu_cam2_n_action2(rvu, kpu, entries); 2011 if (ret) { 2012 dev_warn(rvu->dev, 2013 "profile entry allocation failed for kpu=%d for %d entries\n", 2014 kpu, entries); 2015 return -EINVAL; 2016 } 2017 2018 for (entry = 0; entry < entries; entry++) { 2019 profile->kpu[kpu].cam2[entry] = cam2[entry]; 2020 profile->kpu[kpu].action2[entry] = action[entry]; 2021 } 2022 } 2023 2024 return 0; 2025 } 2026 2027 static int npc_apply_custom_kpu(struct rvu *rvu, 2028 struct npc_kpu_profile_adapter *profile, 2029 bool from_fs, int *fw_kpus) 2030 { 2031 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata); 2032 const struct npc_kpu_profile_fwdata *fw; 2033 struct npc_kpu_profile_fwdata *sfw; 2034 2035 if (is_cn20k(rvu->pdev)) 2036 return npc_cn20k_apply_custom_kpu(rvu, profile); 2037 2038 if (rvu->kpu_fwdata_sz < hdr_sz) { 2039 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 2040 return -EINVAL; 2041 } 2042 2043 fw = rvu->kpu_fwdata; 2044 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 2045 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 2046 fw->signature); 2047 return -EINVAL; 2048 } 2049 /* Verify if the using known profile structure */ 2050 if (NPC_KPU_VER_MAJ(profile->version) > 2051 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 2052 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 2053 NPC_KPU_VER_MAJ(profile->version), 2054 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 2055 return -EINVAL; 2056 } 2057 /* Verify if profile is aligned with the required kernel changes */ 2058 if (NPC_KPU_VER_MIN(profile->version) < 2059 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 2060 dev_warn(rvu->dev, 2061 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 2062 NPC_KPU_VER_MAJ(profile->version), 2063 NPC_KPU_VER_MIN(profile->version), 2064 NPC_KPU_VER_PATCH(profile->version), 2065 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 2066 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 2067 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 2068 return -EINVAL; 2069 } 2070 /* Verify if profile fits the HW */ 2071 if (fw->kpus > rvu->hw->npc_kpus) { 2072 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus, 2073 rvu->hw->npc_kpus); 2074 return -EINVAL; 2075 } 2076 2077 /* Check if there is enough memory for fw loading. 2078 * Check if there is enough entries for profile->kpu[] to 2079 * set cam_entries2 and action_entries2 2080 */ 2081 if (fw->kpus > profile->kpus) { 2082 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus, 2083 profile->kpus); 2084 return -EINVAL; 2085 } 2086 2087 *fw_kpus = fw->kpus; 2088 2089 sfw = devm_kcalloc(rvu->dev, 1, sizeof(*sfw), GFP_KERNEL); 2090 if (!sfw) 2091 return -ENOMEM; 2092 2093 memcpy(sfw, fw, sizeof(*sfw)); 2094 2095 profile->custom = 1; 2096 profile->name = sfw->name; 2097 profile->version = le64_to_cpu(fw->version); 2098 profile->mcam_kex_prfl.mkex = &sfw->mkex; 2099 profile->lt_def = &sfw->lt_def; 2100 2101 return from_fs ? npc_apply_custom_kpu_from_fs(rvu, profile) : 2102 npc_apply_custom_kpu_from_fw(rvu, profile); 2103 } 2104 2105 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 2106 u64 prfl_sz, const char *kpu_profile) 2107 { 2108 struct npc_kpu_profile_fwdata *kpu_data = NULL; 2109 int rc = -EINVAL; 2110 2111 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 2112 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 2113 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 2114 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 2115 kpu_profile); 2116 rvu->kpu_fwdata = kpu_data; 2117 rvu->kpu_fwdata_sz = prfl_sz; 2118 rvu->kpu_prfl_addr = prfl_addr; 2119 rc = 0; 2120 } 2121 2122 return rc; 2123 } 2124 2125 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 2126 const char *kpu_profile) 2127 { 2128 struct npc_coalesced_kpu_prfl *img_data = NULL; 2129 int i = 0, rc = -EINVAL; 2130 void __iomem *kpu_prfl_addr; 2131 u32 offset; 2132 2133 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 2134 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 2135 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 2136 /* Loaded profile is a single KPU profile. */ 2137 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 2138 prfl_sz, kpu_profile); 2139 goto done; 2140 } 2141 2142 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 2143 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 2144 (img_data->num_prfl * sizeof(uint16_t)); 2145 /* Check if mapped image is coalesced image. */ 2146 while (i < img_data->num_prfl) { 2147 /* Profile image offsets are rounded up to next 8 multiple.*/ 2148 offset = ALIGN_8B_CEIL(offset); 2149 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 2150 offset); 2151 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 2152 img_data->prfl_sz[i], kpu_profile); 2153 if (!rc) 2154 break; 2155 /* Calculating offset of profile image based on profile size.*/ 2156 offset += img_data->prfl_sz[i]; 2157 i++; 2158 } 2159 done: 2160 return rc; 2161 } 2162 2163 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 2164 { 2165 int ret = -EINVAL; 2166 u64 prfl_sz; 2167 2168 /* Setting up the mapping for NPC profile image */ 2169 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 2170 if (ret < 0) 2171 goto done; 2172 2173 /* Detect if profile is coalesced or single KPU profile and load */ 2174 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 2175 if (ret == 0) 2176 goto done; 2177 2178 /* Cleaning up if KPU profile image from fwdata is not valid. */ 2179 if (rvu->kpu_prfl_addr) { 2180 iounmap(rvu->kpu_prfl_addr); 2181 rvu->kpu_prfl_addr = NULL; 2182 rvu->kpu_fwdata_sz = 0; 2183 rvu->kpu_fwdata = NULL; 2184 } 2185 2186 done: 2187 return ret; 2188 } 2189 2190 static int npc_load_kpu_profile_from_fw(struct rvu *rvu) 2191 { 2192 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2193 const char *kpu_profile = rvu->kpu_pfl_name; 2194 int fw_kpus = 0; 2195 2196 /* Loading the KPU profile using firmware database */ 2197 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 2198 return -EFAULT; 2199 2200 /* Apply profile customization if firmware was loaded. */ 2201 if (!rvu->kpu_fwdata_sz || 2202 npc_apply_custom_kpu(rvu, profile, false, &fw_kpus)) { 2203 /* If image from firmware filesystem fails to load or invalid 2204 * retry with firmware database method. 2205 */ 2206 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 2207 /* Loading image from firmware database failed. */ 2208 if (rvu->kpu_prfl_addr) { 2209 iounmap(rvu->kpu_prfl_addr); 2210 rvu->kpu_prfl_addr = NULL; 2211 } else { 2212 kfree(rvu->kpu_fwdata); 2213 } 2214 rvu->kpu_fwdata = NULL; 2215 rvu->kpu_fwdata_sz = 0; 2216 } 2217 2218 dev_warn(rvu->dev, 2219 "Can't load KPU profile %s. Using default.\n", 2220 kpu_profile); 2221 kfree(rvu->kpu_fwdata); 2222 rvu->kpu_fwdata = NULL; 2223 return -EFAULT; 2224 } 2225 2226 dev_info(rvu->dev, "Using custom profile '%.32s', version %d.%d.%d\n", 2227 profile->name, NPC_KPU_VER_MAJ(profile->version), 2228 NPC_KPU_VER_MIN(profile->version), 2229 NPC_KPU_VER_PATCH(profile->version)); 2230 2231 return 0; 2232 } 2233 2234 static int npc_load_kpu_profile_from_fs(struct rvu *rvu) 2235 { 2236 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2237 const char *kpu_profile = rvu->kpu_pfl_name; 2238 const struct firmware *fw = NULL; 2239 int ret, fw_kpus = 0; 2240 char path[512] = "kpu/"; 2241 2242 if (strlen(kpu_profile) > sizeof(path) - strlen("kpu/") - 1) { 2243 dev_err(rvu->dev, "kpu profile name is too big\n"); 2244 return -ENOSPC; 2245 } 2246 2247 strcat(path, kpu_profile); 2248 2249 if (request_firmware_direct(&fw, path, rvu->dev)) 2250 return -ENOENT; 2251 2252 dev_info(rvu->dev, "Loading KPU profile from filesystem: %s\n", 2253 path); 2254 2255 rvu->kpu_fwdata = fw->data; 2256 rvu->kpu_fwdata_sz = fw->size; 2257 2258 ret = npc_apply_custom_kpu(rvu, profile, true, &fw_kpus); 2259 release_firmware(fw); 2260 rvu->kpu_fwdata = NULL; 2261 2262 if (ret) { 2263 rvu->kpu_fwdata_sz = 0; 2264 dev_err(rvu->dev, 2265 "Loading KPU profile from filesystem failed\n"); 2266 return ret; 2267 } 2268 2269 /* In firmware loading from filesystem method, all entries are from 2270 * same binary blob. 2271 */ 2272 rvu->kpu.kpus = fw_kpus; 2273 profile->kpus = fw_kpus; 2274 profile->from_fs = true; 2275 return 0; 2276 } 2277 2278 void npc_load_kpu_profile(struct rvu *rvu) 2279 { 2280 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2281 const char *kpu_profile = rvu->kpu_pfl_name; 2282 2283 profile->from_fs = false; 2284 2285 npc_prepare_default_kpu(rvu, profile); 2286 2287 /* If user not specified profile customization */ 2288 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 2289 return; 2290 2291 /* Order of preceedence for load loading NPC profile (high to low) 2292 * Firmware binary in filesystem. 2293 * Firmware database method. 2294 * Default KPU profile. 2295 */ 2296 2297 /* Filesystem-based KPU loading is not supported on cn20k. 2298 * npc_prepare_default_kpu() was invoked earlier, but control 2299 * reached this point because the default profile was not selected. 2300 * No need to call it again. 2301 */ 2302 if (!is_cn20k(rvu->pdev)) { 2303 if (!npc_load_kpu_profile_from_fs(rvu)) 2304 return; 2305 } 2306 2307 /* First prepare default KPU, then we'll customize top entries. */ 2308 npc_prepare_default_kpu(rvu, profile); 2309 if (!npc_load_kpu_profile_from_fw(rvu)) 2310 return; 2311 2312 npc_prepare_default_kpu(rvu, profile); 2313 } 2314 2315 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 2316 { 2317 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2318 struct rvu_hwinfo *hw = rvu->hw; 2319 int num_pkinds, num_kpus, idx; 2320 2321 /* Disable all KPUs and their entries */ 2322 for (idx = 0; idx < hw->npc_kpus; idx++) { 2323 rvu_write64(rvu, blkaddr, 2324 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 2325 rvu_write64(rvu, blkaddr, 2326 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 2327 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 2328 } 2329 2330 /* Load and customize KPU profile. */ 2331 npc_load_kpu_profile(rvu); 2332 2333 /* First program IKPU profile i.e PKIND configs. 2334 * Check HW max count to avoid configuring junk or 2335 * writing to unsupported CSR addresses. 2336 */ 2337 num_pkinds = rvu->kpu.pkinds; 2338 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 2339 2340 for (idx = 0; idx < num_pkinds; idx++) 2341 npc_config_kpuaction(rvu, blkaddr, 2342 npc_get_ikpu_nth_entry(rvu, idx), 2343 0, idx, true); 2344 2345 /* Program KPU CAM and Action profiles */ 2346 num_kpus = rvu->kpu.kpus; 2347 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 2348 2349 for (idx = 0; idx < num_kpus; idx++) 2350 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 2351 2352 if (profile->from_fs) { 2353 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_TYPE(54), 0x03); 2354 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_TYPE(58), 0x03); 2355 } 2356 } 2357 2358 void npc_mcam_rsrcs_deinit(struct rvu *rvu) 2359 { 2360 struct npc_mcam *mcam = &rvu->hw->mcam; 2361 2362 bitmap_free(mcam->bmap); 2363 bitmap_free(mcam->bmap_reverse); 2364 kfree(mcam->entry2pfvf_map); 2365 kfree(mcam->cntr2pfvf_map); 2366 kfree(mcam->entry2cntr_map); 2367 kfree(mcam->cntr_refcnt); 2368 kfree(mcam->entry2target_pffunc); 2369 kfree(mcam->counters.bmap); 2370 } 2371 2372 int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 2373 { 2374 int nixlf_count = rvu_get_nixlf_count(rvu); 2375 struct npc_mcam *mcam = &rvu->hw->mcam; 2376 int rsvd, err; 2377 u16 index; 2378 int cntr; 2379 u64 cfg; 2380 2381 cfg = (rvu_read64(rvu, blkaddr, 2382 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 2383 mcam->keysize = cfg; 2384 2385 /* Number of banks combined per MCAM entry */ 2386 if (is_cn20k(rvu->pdev)) { 2387 /* In cn20k, x2 entries is allowed for x4 profile. 2388 * set total_entries as 8192 * 2 and key size as x2. 2389 */ 2390 mcam->total_entries = mcam->banks * mcam->banksize; 2391 if (cfg == NPC_MCAM_KEY_X2) 2392 mcam->banks_per_entry = 1; 2393 else 2394 mcam->banks_per_entry = 2; 2395 2396 rsvd = 0; 2397 } else { 2398 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * 2399 mcam->banksize; 2400 2401 if (cfg == NPC_MCAM_KEY_X4) 2402 mcam->banks_per_entry = 4; 2403 else if (cfg == NPC_MCAM_KEY_X2) 2404 mcam->banks_per_entry = 2; 2405 else 2406 mcam->banks_per_entry = 1; 2407 2408 /* Reserve one MCAM entry for each of the NIX LF to 2409 * guarantee space to install default matching DMAC rule. 2410 * Also reserve 2 MCAM entries for each PF for default 2411 * channel based matching or 'bcast & promisc' matching to 2412 * support BCAST and PROMISC modes of operation for PFs. 2413 * PF0 is excluded. 2414 */ 2415 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 2416 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 2417 if (mcam->total_entries <= rsvd) { 2418 dev_warn(rvu->dev, 2419 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 2420 mcam->total_entries); 2421 return -ENOMEM; 2422 } 2423 } 2424 2425 mcam->bmap_entries = mcam->total_entries - rsvd; 2426 /* cn20k does not need offsets to alloc mcam entries */ 2427 if (!is_cn20k(rvu->pdev)) { 2428 mcam->nixlf_offset = mcam->bmap_entries; 2429 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 2430 } 2431 2432 /* Allocate bitmaps for managing MCAM entries */ 2433 mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2434 if (!mcam->bmap) 2435 return -ENOMEM; 2436 2437 mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2438 if (!mcam->bmap_reverse) 2439 goto free_bmap; 2440 2441 mcam->bmap_fcnt = mcam->bmap_entries; 2442 2443 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 2444 mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16), 2445 GFP_KERNEL); 2446 2447 if (!mcam->entry2pfvf_map) 2448 goto free_bmap_reverse; 2449 2450 /* Reserve 1/8th of MCAM entries at the bottom for low priority 2451 * allocations and another 1/8th at the top for high priority 2452 * allocations. 2453 */ 2454 if (!is_cn20k(rvu->pdev)) { 2455 mcam->lprio_count = mcam->bmap_entries / 8; 2456 if (mcam->lprio_count > BITS_PER_LONG) 2457 mcam->lprio_count = round_down(mcam->lprio_count, 2458 BITS_PER_LONG); 2459 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 2460 mcam->hprio_count = mcam->lprio_count; 2461 mcam->hprio_end = mcam->hprio_count; 2462 } 2463 2464 /* Allocate bitmap for managing MCAM counters and memory 2465 * for saving counter to RVU PFFUNC allocation mapping. 2466 */ 2467 err = rvu_alloc_bitmap(&mcam->counters); 2468 if (err) 2469 goto free_entry_map; 2470 2471 mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16), 2472 GFP_KERNEL); 2473 if (!mcam->cntr2pfvf_map) 2474 goto free_cntr_bmap; 2475 2476 /* Alloc memory for MCAM entry to counter mapping and for tracking 2477 * counter's reference count. 2478 */ 2479 mcam->entry2cntr_map = kcalloc(mcam->total_entries, sizeof(u16), 2480 GFP_KERNEL); 2481 if (!mcam->entry2cntr_map) 2482 goto free_cntr_map; 2483 2484 mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16), 2485 GFP_KERNEL); 2486 if (!mcam->cntr_refcnt) 2487 goto free_entry_cntr_map; 2488 2489 /* Alloc memory for saving target device of mcam rule */ 2490 mcam->entry2target_pffunc = kcalloc(mcam->total_entries, 2491 sizeof(u16), GFP_KERNEL); 2492 if (!mcam->entry2target_pffunc) 2493 goto free_cntr_refcnt; 2494 2495 for (index = 0; index < mcam->bmap_entries; index++) 2496 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2497 2498 for (index = 0; index < mcam->total_entries; index++) 2499 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2500 2501 for (cntr = 0; cntr < mcam->counters.max; cntr++) 2502 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2503 2504 mutex_init(&mcam->lock); 2505 2506 return 0; 2507 2508 free_cntr_refcnt: 2509 kfree(mcam->cntr_refcnt); 2510 free_entry_cntr_map: 2511 kfree(mcam->entry2cntr_map); 2512 free_cntr_map: 2513 kfree(mcam->cntr2pfvf_map); 2514 free_cntr_bmap: 2515 kfree(mcam->counters.bmap); 2516 free_entry_map: 2517 kfree(mcam->entry2pfvf_map); 2518 free_bmap_reverse: 2519 bitmap_free(mcam->bmap_reverse); 2520 free_bmap: 2521 bitmap_free(mcam->bmap); 2522 2523 return -ENOMEM; 2524 } 2525 2526 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 2527 { 2528 struct npc_pkind *pkind = &rvu->hw->pkind; 2529 struct npc_mcam *mcam = &rvu->hw->mcam; 2530 struct rvu_hwinfo *hw = rvu->hw; 2531 u64 npc_const, npc_const1; 2532 u64 npc_const2 = 0; 2533 2534 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 2535 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 2536 if (npc_const1 & BIT_ULL(63)) 2537 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 2538 2539 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 2540 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 2541 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 2542 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 2543 /* For Cn20k silicon, check if enhanced parser 2544 * is present, then set the NUM_KPMS = NUM_KPUS / 2 and 2545 * number of LDATA extractors per KEX. 2546 */ 2547 if (is_cn20k(rvu->pdev)) { 2548 if (!(npc_const1 & BIT_ULL(62))) { 2549 WARN(1, "Enhanced parser is not supported\n"); 2550 return; 2551 } 2552 hw->npc_kpms = hw->npc_kpus / 2; 2553 hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL; 2554 } 2555 2556 hw->npc_intfs = npc_const & 0xFULL; 2557 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 2558 2559 mcam->banks = (npc_const >> 44) & 0xFULL; 2560 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 2561 hw->npc_stat_ena = BIT_ULL(9); 2562 /* Extended set */ 2563 if (npc_const2) { 2564 hw->npc_ext_set = true; 2565 /* 96xx supports only match_stats and npc_counters 2566 * reflected in NPC_AF_CONST reg. 2567 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 2568 * 98xx has both match_stat and ext and npc_counter 2569 * reflected in NPC_AF_CONST2 2570 * STAT_SEL_EXT added at [12:14] bit position. 2571 * cn10k supports only ext and hence npc_counters in 2572 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 2573 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 2574 */ 2575 if (!hw->npc_counters) 2576 hw->npc_stat_ena = BIT_ULL(63); 2577 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 2578 mcam->banksize = npc_const2 & 0xFFFFULL; 2579 } 2580 2581 mcam->counters.max = hw->npc_counters; 2582 } 2583 2584 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 2585 { 2586 const struct npc_mcam_kex_extr *mkex_extr; 2587 struct npc_mcam *mcam = &rvu->hw->mcam; 2588 struct rvu_hwinfo *hw = rvu->hw; 2589 const struct npc_mcam_kex *mkex; 2590 u64 nibble_ena, rx_kex, tx_kex; 2591 u64 *keyx_cfg, reg; 2592 u8 intf; 2593 2594 mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr; 2595 mkex = rvu->kpu.mcam_kex_prfl.mkex; 2596 2597 if (is_cn20k(rvu->pdev)) { 2598 keyx_cfg = (u64 *)mkex_extr->keyx_cfg; 2599 } else { 2600 keyx_cfg = (u64 *)mkex->keyx_cfg; 2601 /* Reserve last counter for MCAM RX miss action which is set to 2602 * drop packet. This way we will know how many pkts didn't 2603 * match any MCAM entry. 2604 */ 2605 mcam->counters.max--; 2606 mcam->rx_miss_act_cntr = mcam->counters.max; 2607 } 2608 2609 rx_kex = keyx_cfg[NIX_INTF_RX]; 2610 tx_kex = keyx_cfg[NIX_INTF_TX]; 2611 2612 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 2613 2614 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 2615 if (nibble_ena) { 2616 tx_kex &= ~NPC_PARSE_NIBBLE; 2617 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 2618 keyx_cfg[NIX_INTF_TX] = tx_kex; 2619 } 2620 2621 /* Configure RX interfaces */ 2622 for (intf = 0; intf < hw->npc_intfs; intf++) { 2623 if (is_npc_intf_tx(intf)) 2624 continue; 2625 2626 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 2627 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2628 rx_kex); 2629 2630 if (is_cn20k(rvu->pdev)) 2631 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2632 else 2633 reg = NPC_AF_INTFX_MISS_ACT(intf); 2634 2635 /* If MCAM lookup doesn't result in a match, drop the received 2636 * packet. And map this action to a counter to count dropped 2637 * packets. 2638 */ 2639 rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP); 2640 2641 if (is_cn20k(rvu->pdev)) 2642 continue; 2643 2644 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 2645 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 2646 */ 2647 rvu_write64(rvu, blkaddr, 2648 NPC_AF_INTFX_MISS_STAT_ACT(intf), 2649 ((mcam->rx_miss_act_cntr >> 9) << 12) | 2650 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 2651 } 2652 2653 /* Configure TX interfaces */ 2654 for (intf = 0; intf < hw->npc_intfs; intf++) { 2655 if (is_npc_intf_rx(intf)) 2656 continue; 2657 2658 /* Extract Ltypes LID_LA to LID_LE */ 2659 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2660 tx_kex); 2661 2662 if (is_cn20k(rvu->pdev)) 2663 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2664 else 2665 reg = NPC_AF_INTFX_MISS_ACT(intf); 2666 2667 /* Set TX miss action to UCAST_DEFAULT i.e 2668 * transmit the packet on NIX LF SQ's default channel. 2669 */ 2670 rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT); 2671 } 2672 } 2673 2674 int rvu_npc_init(struct rvu *rvu) 2675 { 2676 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 2677 struct npc_pkind *pkind = &rvu->hw->pkind; 2678 struct npc_mcam *mcam = &rvu->hw->mcam; 2679 int blkaddr, entry, bank, err; 2680 2681 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2682 if (blkaddr < 0) { 2683 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 2684 return -ENODEV; 2685 } 2686 2687 rvu_npc_hw_init(rvu, blkaddr); 2688 2689 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2690 for (bank = 0; bank < mcam->banks; bank++) { 2691 for (entry = 0; entry < mcam->banksize; entry++) 2692 rvu_write64(rvu, blkaddr, 2693 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2694 } 2695 2696 err = rvu_alloc_bitmap(&pkind->rsrc); 2697 if (err) 2698 return err; 2699 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2700 * no need to configure PKIND for all LBKs separately. 2701 */ 2702 rvu_alloc_rsrc(&pkind->rsrc); 2703 2704 /* Allocate mem for pkind to PF and channel mapping info */ 2705 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2706 sizeof(u32), GFP_KERNEL); 2707 if (!pkind->pfchan_map) 2708 return -ENOMEM; 2709 2710 /* Configure KPU profile */ 2711 if (is_cn20k(rvu->pdev)) 2712 npc_cn20k_parser_profile_init(rvu, blkaddr); 2713 else 2714 npc_parser_profile_init(rvu, blkaddr); 2715 2716 /* Config Outer L2, IPv4's NPC layer info */ 2717 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2718 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2719 kpu->lt_def->pck_ol2.ltype_mask); 2720 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2721 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2722 kpu->lt_def->pck_oip4.ltype_mask); 2723 2724 /* Config Inner IPV4 NPC layer info */ 2725 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2726 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2727 kpu->lt_def->pck_iip4.ltype_mask); 2728 2729 /* Enable below for Rx pkts. 2730 * - Outer IPv4 header checksum validation. 2731 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2732 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2733 * - Inner IPv4 header checksum validation. 2734 * - Set non zero checksum error code value 2735 */ 2736 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2737 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2738 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2739 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2740 2741 rvu_npc_setup_interfaces(rvu, blkaddr); 2742 2743 npc_config_secret_key(rvu, blkaddr); 2744 /* Configure MKEX profile */ 2745 if (is_cn20k(rvu->pdev)) 2746 npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2747 else 2748 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2749 2750 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2751 if (err) 2752 return err; 2753 2754 err = npc_flow_steering_init(rvu, blkaddr); 2755 if (err) { 2756 dev_err(rvu->dev, 2757 "Incorrect mkex profile loaded using default mkex\n"); 2758 if (is_cn20k(rvu->pdev)) 2759 npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2760 else 2761 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2762 } 2763 2764 if (is_cn20k(rvu->pdev)) 2765 return npc_cn20k_init(rvu); 2766 2767 return 0; 2768 } 2769 2770 void rvu_npc_freemem(struct rvu *rvu) 2771 { 2772 struct npc_pkind *pkind = &rvu->hw->pkind; 2773 struct npc_mcam *mcam = &rvu->hw->mcam; 2774 2775 kfree(pkind->rsrc.bmap); 2776 npc_mcam_rsrcs_deinit(rvu); 2777 if (rvu->kpu_prfl_addr) 2778 iounmap(rvu->kpu_prfl_addr); 2779 else 2780 kfree(rvu->kpu_fwdata); 2781 mutex_destroy(&mcam->lock); 2782 2783 if (is_cn20k(rvu->pdev)) 2784 npc_cn20k_deinit(rvu); 2785 } 2786 2787 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2788 int blkaddr, int *alloc_cnt, 2789 int *enable_cnt) 2790 { 2791 struct npc_mcam *mcam = &rvu->hw->mcam; 2792 int entry; 2793 2794 *alloc_cnt = 0; 2795 *enable_cnt = 0; 2796 2797 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2798 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2799 (*alloc_cnt)++; 2800 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2801 (*enable_cnt)++; 2802 } 2803 } 2804 } 2805 2806 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2807 int blkaddr, int *alloc_cnt, 2808 int *enable_cnt) 2809 { 2810 struct npc_mcam *mcam = &rvu->hw->mcam; 2811 int cntr; 2812 2813 *alloc_cnt = 0; 2814 *enable_cnt = 0; 2815 2816 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2817 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2818 (*alloc_cnt)++; 2819 if (mcam->cntr_refcnt[cntr]) 2820 (*enable_cnt)++; 2821 } 2822 } 2823 } 2824 2825 int npc_mcam_verify_entry(struct npc_mcam *mcam, 2826 u16 pcifunc, int entry) 2827 { 2828 /* verify AF installed entries */ 2829 if (is_pffunc_af(pcifunc)) 2830 return 0; 2831 /* Verify if entry is valid and if it is indeed 2832 * allocated to the requesting PFFUNC. 2833 */ 2834 if (entry >= mcam->bmap_entries) 2835 return NPC_MCAM_INVALID_REQ; 2836 2837 if (pcifunc != mcam->entry2pfvf_map[entry]) 2838 return NPC_MCAM_PERM_DENIED; 2839 2840 return 0; 2841 } 2842 2843 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2844 u16 pcifunc, int cntr) 2845 { 2846 /* Verify if counter is valid and if it is indeed 2847 * allocated to the requesting PFFUNC. 2848 */ 2849 if (cntr >= mcam->counters.max) 2850 return NPC_MCAM_INVALID_REQ; 2851 2852 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2853 return NPC_MCAM_PERM_DENIED; 2854 2855 return 0; 2856 } 2857 2858 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2859 int blkaddr, u16 entry, u16 cntr) 2860 { 2861 u16 index = entry & (mcam->banksize - 1); 2862 u32 bank = npc_get_bank(mcam, entry); 2863 struct rvu_hwinfo *hw = rvu->hw; 2864 2865 /* Set mapping and increment counter's refcnt */ 2866 mcam->entry2cntr_map[entry] = cntr; 2867 mcam->cntr_refcnt[cntr]++; 2868 2869 if (is_cn20k(rvu->pdev)) 2870 return; 2871 2872 /* Enable stats */ 2873 rvu_write64(rvu, blkaddr, 2874 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2875 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2876 } 2877 2878 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2879 struct npc_mcam *mcam, 2880 int blkaddr, u16 entry, u16 cntr) 2881 { 2882 u16 index = entry & (mcam->banksize - 1); 2883 u32 bank = npc_get_bank(mcam, entry); 2884 2885 /* Remove mapping and reduce counter's refcnt */ 2886 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2887 mcam->cntr_refcnt[cntr]--; 2888 2889 if (is_cn20k(rvu->pdev)) 2890 return; 2891 2892 /* Disable stats */ 2893 rvu_write64(rvu, blkaddr, 2894 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2895 } 2896 2897 /* Sets MCAM entry in bitmap as used. Update 2898 * reverse bitmap too. Should be called with 2899 * 'mcam->lock' held. 2900 */ 2901 void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2902 { 2903 u16 entry, rentry; 2904 2905 entry = index; 2906 rentry = mcam->bmap_entries - index - 1; 2907 2908 __set_bit(entry, mcam->bmap); 2909 __set_bit(rentry, mcam->bmap_reverse); 2910 mcam->bmap_fcnt--; 2911 } 2912 2913 /* Sets MCAM entry in bitmap as free. Update 2914 * reverse bitmap too. Should be called with 2915 * 'mcam->lock' held. 2916 */ 2917 void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2918 { 2919 u16 entry, rentry; 2920 2921 entry = index; 2922 rentry = mcam->bmap_entries - index - 1; 2923 2924 __clear_bit(entry, mcam->bmap); 2925 __clear_bit(rentry, mcam->bmap_reverse); 2926 mcam->bmap_fcnt++; 2927 } 2928 2929 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2930 int blkaddr, u16 pcifunc) 2931 { 2932 u16 dft_idxs[NPC_DFT_RULE_MAX_ID] = {[0 ... NPC_DFT_RULE_MAX_ID - 1] = USHRT_MAX}; 2933 bool cn20k_dft_rl; 2934 u16 index, cntr; 2935 int rc; 2936 2937 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, 2938 &dft_idxs[NPC_DFT_RULE_BCAST_ID], 2939 &dft_idxs[NPC_DFT_RULE_MCAST_ID], 2940 &dft_idxs[NPC_DFT_RULE_PROMISC_ID], 2941 &dft_idxs[NPC_DFT_RULE_UCAST_ID]); 2942 2943 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2944 for (index = 0; index < mcam->bmap_entries; index++) { 2945 if (mcam->entry2pfvf_map[index] != pcifunc) 2946 continue; 2947 2948 cn20k_dft_rl = false; 2949 2950 if (is_cn20k(rvu->pdev)) { 2951 if (dft_idxs[NPC_DFT_RULE_BCAST_ID] == index || 2952 dft_idxs[NPC_DFT_RULE_MCAST_ID] == index || 2953 dft_idxs[NPC_DFT_RULE_PROMISC_ID] == index || 2954 dft_idxs[NPC_DFT_RULE_UCAST_ID] == index) { 2955 cn20k_dft_rl = true; 2956 } 2957 } 2958 2959 /* Disable the entry */ 2960 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2961 2962 if (!cn20k_dft_rl) { 2963 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2964 /* Free the entry in bitmap */ 2965 npc_mcam_clear_bit(mcam, index); 2966 mcam->entry2target_pffunc[index] = 0x0; 2967 } 2968 2969 /* Update entry2counter mapping */ 2970 cntr = mcam->entry2cntr_map[index]; 2971 if (cntr != NPC_MCAM_INVALID_MAP) 2972 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2973 blkaddr, index, 2974 cntr); 2975 2976 if (!is_cn20k(rvu->pdev) || cn20k_dft_rl) 2977 continue; 2978 2979 rc = npc_cn20k_idx_free(rvu, &index, 1); 2980 if (rc) 2981 dev_err(rvu->dev, 2982 "Failed to free mcam idx=%u pcifunc=%#x\n", 2983 index, pcifunc); 2984 } 2985 } 2986 2987 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2988 u16 pcifunc) 2989 { 2990 u16 cntr; 2991 2992 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2993 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2994 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2995 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2996 mcam->cntr_refcnt[cntr] = 0; 2997 rvu_free_rsrc(&mcam->counters, cntr); 2998 /* This API is expected to be called after freeing 2999 * MCAM entries, which inturn will remove 3000 * 'entry to counter' mapping. 3001 * No need to do it again. 3002 */ 3003 } 3004 } 3005 } 3006 3007 /* Find area of contiguous free entries of size 'nr'. 3008 * If not found return max contiguous free entries available. 3009 */ 3010 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 3011 u16 nr, u16 *max_area) 3012 { 3013 u16 max_area_start = 0; 3014 u16 index, next, end; 3015 3016 *max_area = 0; 3017 3018 again: 3019 index = find_next_zero_bit(map, size, start); 3020 if (index >= size) 3021 return max_area_start; 3022 3023 end = ((index + nr) >= size) ? size : index + nr; 3024 next = find_next_bit(map, end, index); 3025 if (*max_area < (next - index)) { 3026 *max_area = next - index; 3027 max_area_start = index; 3028 } 3029 3030 if (next < end) { 3031 start = next + 1; 3032 goto again; 3033 } 3034 3035 return max_area_start; 3036 } 3037 3038 /* Find number of free MCAM entries available 3039 * within range i.e in between 'start' and 'end'. 3040 */ 3041 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 3042 { 3043 u16 index, next; 3044 u16 fcnt = 0; 3045 3046 again: 3047 if (start >= end) 3048 return fcnt; 3049 3050 index = find_next_zero_bit(map, end, start); 3051 if (index >= end) 3052 return fcnt; 3053 3054 next = find_next_bit(map, end, index); 3055 if (next <= end) { 3056 fcnt += next - index; 3057 start = next + 1; 3058 goto again; 3059 } 3060 3061 fcnt += end - index; 3062 return fcnt; 3063 } 3064 3065 static void 3066 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 3067 struct npc_mcam_alloc_entry_req *req, 3068 u16 *start, u16 *end, bool *reverse) 3069 { 3070 u16 fcnt; 3071 3072 if (req->ref_prio == NPC_MCAM_HIGHER_PRIO) 3073 goto hprio; 3074 3075 /* For a low priority entry allocation 3076 * - If reference entry is not in hprio zone then 3077 * search range: ref_entry to end. 3078 * - If reference entry is in hprio zone and if 3079 * request can be accomodated in non-hprio zone then 3080 * search range: 'start of middle zone' to 'end' 3081 * - else search in reverse, so that less number of hprio 3082 * zone entries are allocated. 3083 */ 3084 3085 *reverse = false; 3086 *start = req->ref_entry + 1; 3087 *end = mcam->bmap_entries; 3088 3089 if (req->ref_entry >= mcam->hprio_end) 3090 return; 3091 3092 fcnt = npc_mcam_get_free_count(mcam->bmap, 3093 mcam->hprio_end, mcam->bmap_entries); 3094 if (fcnt > req->count) 3095 *start = mcam->hprio_end; 3096 else 3097 *reverse = true; 3098 return; 3099 3100 hprio: 3101 /* For a high priority entry allocation, search is always 3102 * in reverse to preserve hprio zone entries. 3103 * - If reference entry is not in lprio zone then 3104 * search range: 0 to ref_entry. 3105 * - If reference entry is in lprio zone and if 3106 * request can be accomodated in middle zone then 3107 * search range: 'hprio_end' to 'lprio_start' 3108 */ 3109 3110 *reverse = true; 3111 *start = 0; 3112 *end = req->ref_entry; 3113 3114 if (req->ref_entry <= mcam->lprio_start) 3115 return; 3116 3117 fcnt = npc_mcam_get_free_count(mcam->bmap, 3118 mcam->hprio_end, mcam->lprio_start); 3119 if (fcnt < req->count) 3120 return; 3121 *start = mcam->hprio_end; 3122 *end = mcam->lprio_start; 3123 } 3124 3125 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 3126 struct npc_mcam_alloc_entry_req *req, 3127 struct npc_mcam_alloc_entry_rsp *rsp) 3128 { 3129 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 3130 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 3131 u16 fcnt, hp_fcnt, lp_fcnt; 3132 struct rvu *rvu = hw->rvu; 3133 u16 start, end, index; 3134 int entry, next_start; 3135 bool reverse = false; 3136 unsigned long *bmap; 3137 int ret, limit = 0; 3138 u16 max_contig; 3139 3140 if (!is_cn20k(rvu->pdev)) 3141 goto not_cn20k; 3142 3143 /* Only x2 or x4 key types are accepted */ 3144 if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4) 3145 return NPC_MCAM_INVALID_REQ; 3146 3147 /* The below table is being followed during allocation, 3148 * 3149 * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to 3150 * allocate 0th index 3151 * 2. ref_entry == 0 && prio == HIGH && count > 1 : Invalid request 3152 * 3. ref_entry == 0 && prio == LOW && count >= 1 : limit = 0 3153 * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0 3154 * 5. ref_entry != 0 && prio == LOW && count >=1 : limit = Max 3155 * (X2 2*8192, X4 8192) 3156 */ 3157 if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) { 3158 if (req->kw_type == NPC_MCAM_KEY_X2) 3159 limit = 2 * mcam->bmap_entries; 3160 else 3161 limit = mcam->bmap_entries; 3162 } 3163 3164 ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type, 3165 req->ref_prio, rsp->entry_list, 3166 req->ref_entry, limit, 3167 req->contig, req->count, !!req->virt); 3168 3169 if (ret) { 3170 rsp->count = 0; 3171 return NPC_MCAM_ALLOC_FAILED; 3172 } 3173 3174 rsp->count = req->count; 3175 if (req->contig) 3176 rsp->entry = rsp->entry_list[0]; 3177 3178 /* cn20k, entries allocation algorithm is different. 3179 * This common API updates some bitmap on usage etc, which 3180 * will be used by other functions. So update those for 3181 * cn20k as well. 3182 */ 3183 3184 mutex_lock(&mcam->lock); 3185 /* Mark the allocated entries as used and set nixlf mapping */ 3186 for (entry = 0; entry < rsp->count; entry++) { 3187 index = npc_cn20k_vidx2idx(rsp->entry_list[entry]); 3188 npc_mcam_set_bit(mcam, index); 3189 mcam->entry2pfvf_map[index] = pcifunc; 3190 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 3191 } 3192 3193 /* cn20k, free count is provided thru different mbox message. 3194 * one counter to indicate free x2 slots and free x4 slots 3195 * does not provide any useful information to the user. 3196 */ 3197 rsp->free_count = -1; 3198 mutex_unlock(&mcam->lock); 3199 3200 return 0; 3201 3202 not_cn20k: 3203 mutex_lock(&mcam->lock); 3204 3205 /* Check if there are any free entries */ 3206 if (!mcam->bmap_fcnt) { 3207 mutex_unlock(&mcam->lock); 3208 return NPC_MCAM_ALLOC_FAILED; 3209 } 3210 3211 /* MCAM entries are divided into high priority, middle and 3212 * low priority zones. Idea is to not allocate top and lower 3213 * most entries as much as possible, this is to increase 3214 * probability of honouring priority allocation requests. 3215 * 3216 * Two bitmaps are used for mcam entry management, 3217 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 3218 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 3219 * 3220 * Reverse bitmap is used to allocate entries 3221 * - when a higher priority entry is requested 3222 * - when available free entries are less. 3223 * Lower priority ones out of avaialble free entries are always 3224 * chosen when 'high vs low' question arises. 3225 * 3226 * For a VF base MCAM match rule is set by its PF. And all the 3227 * further MCAM rules installed by VF on its own are 3228 * concatenated with the base rule set by its PF. Hence PF entries 3229 * should be at lower priority compared to VF entries. Otherwise 3230 * base rule is hit always and rules installed by VF will be of 3231 * no use. Hence if the request is from PF then allocate low 3232 * priority entries. 3233 */ 3234 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 3235 goto lprio_alloc; 3236 3237 /* Get the search range for priority allocation request */ 3238 if (req->ref_prio) { 3239 npc_get_mcam_search_range_priority(mcam, req, 3240 &start, &end, &reverse); 3241 goto alloc; 3242 } 3243 3244 /* Find out the search range for non-priority allocation request 3245 * 3246 * Get MCAM free entry count in middle zone. 3247 */ 3248 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 3249 mcam->lprio_start, 3250 mcam->bmap_entries); 3251 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 3252 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 3253 3254 /* Check if request can be accomodated in the middle zone */ 3255 if (fcnt > req->count) { 3256 start = mcam->hprio_end; 3257 end = mcam->lprio_start; 3258 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 3259 /* Expand search zone from half of hprio zone to 3260 * half of lprio zone. 3261 */ 3262 start = mcam->hprio_end / 2; 3263 end = mcam->bmap_entries - (mcam->lprio_count / 2); 3264 reverse = true; 3265 } else { 3266 /* Not enough free entries, search all entries in reverse, 3267 * so that low priority ones will get used up. 3268 */ 3269 lprio_alloc: 3270 reverse = true; 3271 start = 0; 3272 end = mcam->bmap_entries; 3273 /* Ensure PF requests are always at bottom and if PF requests 3274 * for higher/lower priority entry wrt reference entry then 3275 * honour that criteria and start search for entries from bottom 3276 * and not in mid zone. 3277 */ 3278 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 3279 req->ref_prio == NPC_MCAM_HIGHER_PRIO) 3280 end = req->ref_entry; 3281 3282 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 3283 req->ref_prio == NPC_MCAM_LOWER_PRIO) 3284 start = req->ref_entry; 3285 } 3286 3287 alloc: 3288 if (reverse) { 3289 bmap = mcam->bmap_reverse; 3290 start = mcam->bmap_entries - start; 3291 end = mcam->bmap_entries - end; 3292 swap(start, end); 3293 } else { 3294 bmap = mcam->bmap; 3295 } 3296 3297 if (req->contig) { 3298 /* Allocate requested number of contiguous entries, if 3299 * unsuccessful find max contiguous entries available. 3300 */ 3301 index = npc_mcam_find_zero_area(bmap, end, start, 3302 req->count, &max_contig); 3303 rsp->count = max_contig; 3304 if (reverse) 3305 rsp->entry = mcam->bmap_entries - index - max_contig; 3306 else 3307 rsp->entry = index; 3308 } else { 3309 /* Allocate requested number of non-contiguous entries, 3310 * if unsuccessful allocate as many as possible. 3311 */ 3312 rsp->count = 0; 3313 next_start = start; 3314 for (entry = 0; entry < req->count; entry++) { 3315 index = find_next_zero_bit(bmap, end, next_start); 3316 if (index >= end) 3317 break; 3318 3319 next_start = start + (index - start) + 1; 3320 3321 /* Save the entry's index */ 3322 if (reverse) 3323 index = mcam->bmap_entries - index - 1; 3324 entry_list[entry] = index; 3325 rsp->count++; 3326 } 3327 } 3328 3329 /* If allocating requested no of entries is unsucessful, 3330 * expand the search range to full bitmap length and retry. 3331 */ 3332 if (!req->ref_prio && rsp->count < req->count && 3333 ((end - start) != mcam->bmap_entries)) { 3334 reverse = true; 3335 start = 0; 3336 end = mcam->bmap_entries; 3337 goto alloc; 3338 } 3339 3340 /* For priority entry allocation requests, if allocation is 3341 * failed then expand search to max possible range and retry. 3342 */ 3343 if (req->ref_prio && rsp->count < req->count) { 3344 if (req->ref_prio == NPC_MCAM_LOWER_PRIO && 3345 (start != (req->ref_entry + 1))) { 3346 start = req->ref_entry + 1; 3347 end = mcam->bmap_entries; 3348 reverse = false; 3349 goto alloc; 3350 } else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) && 3351 ((end - start) != req->ref_entry)) { 3352 start = 0; 3353 end = req->ref_entry; 3354 reverse = true; 3355 goto alloc; 3356 } 3357 } 3358 3359 /* Copy MCAM entry indices into mbox response entry_list. 3360 * Requester always expects indices in ascending order, so 3361 * reverse the list if reverse bitmap is used for allocation. 3362 */ 3363 if (!req->contig && rsp->count) { 3364 index = 0; 3365 for (entry = rsp->count - 1; entry >= 0; entry--) { 3366 if (reverse) 3367 rsp->entry_list[index++] = entry_list[entry]; 3368 else 3369 rsp->entry_list[entry] = entry_list[entry]; 3370 } 3371 } 3372 3373 /* Mark the allocated entries as used and set nixlf mapping */ 3374 for (entry = 0; entry < rsp->count; entry++) { 3375 index = req->contig ? 3376 (rsp->entry + entry) : rsp->entry_list[entry]; 3377 npc_mcam_set_bit(mcam, index); 3378 mcam->entry2pfvf_map[index] = pcifunc; 3379 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 3380 } 3381 3382 /* Update available free count in mbox response */ 3383 rsp->free_count = mcam->bmap_fcnt; 3384 3385 mutex_unlock(&mcam->lock); 3386 return 0; 3387 } 3388 3389 /* Marks bitmaps to reserved the mcam slot */ 3390 void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx) 3391 { 3392 struct npc_mcam *mcam = &rvu->hw->mcam; 3393 3394 npc_mcam_set_bit(mcam, entry_idx); 3395 } 3396 3397 int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) 3398 { 3399 struct npc_mcam *mcam = &rvu->hw->mcam; 3400 struct npc_install_flow_rsp rsp; 3401 struct rvu_npc_mcam_rule *rule; 3402 int blkaddr; 3403 3404 /* Counter is set for each rule by default */ 3405 if (is_cn20k(rvu->pdev)) 3406 return -EINVAL; 3407 3408 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3409 if (blkaddr < 0) 3410 return -EINVAL; 3411 3412 mutex_lock(&mcam->lock); 3413 list_for_each_entry(rule, &mcam->mcam_rules, list) { 3414 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, rule->entry)) 3415 continue; 3416 if (!rule->default_rule) 3417 continue; 3418 if (enable && !rule->has_cntr) { /* Alloc and map new counter */ 3419 __rvu_mcam_add_counter_to_rule(rvu, rule->owner, 3420 rule, &rsp); 3421 if (rsp.counter < 0) { 3422 dev_err(rvu->dev, 3423 "%s: Failed to allocate cntr for default rule (err=%d)\n", 3424 __func__, rsp.counter); 3425 break; 3426 } 3427 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3428 rule->entry, rsp.counter); 3429 /* Reset counter before use */ 3430 rvu_write64(rvu, blkaddr, 3431 NPC_AF_MATCH_STATX(rule->cntr), 0x0); 3432 } 3433 3434 /* Free and unmap counter */ 3435 if (!enable && rule->has_cntr) 3436 __rvu_mcam_remove_counter_from_rule(rvu, rule->owner, 3437 rule); 3438 } 3439 mutex_unlock(&mcam->lock); 3440 3441 return 0; 3442 } 3443 3444 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 3445 struct npc_mcam_alloc_entry_req *req, 3446 struct npc_mcam_alloc_entry_rsp *rsp) 3447 { 3448 struct npc_mcam *mcam = &rvu->hw->mcam; 3449 u16 pcifunc = req->hdr.pcifunc; 3450 int blkaddr; 3451 3452 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3453 if (blkaddr < 0) 3454 return NPC_MCAM_INVALID_REQ; 3455 3456 rsp->entry = NPC_MCAM_ENTRY_INVALID; 3457 rsp->free_count = 0; 3458 3459 /* Check if ref_entry is greater that the range 3460 * then set it to max value. 3461 */ 3462 if (req->ref_entry > mcam->bmap_entries) 3463 req->ref_entry = mcam->bmap_entries; 3464 3465 /* ref_entry can't be '0' if requested priority is high. 3466 * Can't be last entry if requested priority is low. 3467 */ 3468 if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) || 3469 (req->ref_entry == mcam->bmap_entries && 3470 req->ref_prio == NPC_MCAM_LOWER_PRIO)) 3471 return NPC_MCAM_INVALID_REQ; 3472 3473 /* Since list of allocated indices needs to be sent to requester, 3474 * max number of non-contiguous entries per mbox msg is limited. 3475 */ 3476 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 3477 dev_err(rvu->dev, 3478 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 3479 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 3480 return NPC_MCAM_INVALID_REQ; 3481 } 3482 3483 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 3484 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3485 return NPC_MCAM_ALLOC_DENIED; 3486 3487 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 3488 } 3489 3490 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 3491 struct npc_mcam_free_entry_req *req, 3492 struct msg_rsp *rsp) 3493 { 3494 struct npc_mcam *mcam = &rvu->hw->mcam; 3495 u16 pcifunc = req->hdr.pcifunc; 3496 int blkaddr, rc = 0; 3497 u16 cntr; 3498 3499 req->entry = npc_cn20k_vidx2idx(req->entry); 3500 3501 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3502 if (blkaddr < 0) 3503 return NPC_MCAM_INVALID_REQ; 3504 3505 /* Free request from PFFUNC with no NIXLF attached, ignore */ 3506 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3507 return NPC_MCAM_INVALID_REQ; 3508 3509 mutex_lock(&mcam->lock); 3510 3511 if (req->all) 3512 goto free_all; 3513 3514 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3515 if (rc) 3516 goto exit; 3517 3518 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 3519 mcam->entry2target_pffunc[req->entry] = 0x0; 3520 npc_mcam_clear_bit(mcam, req->entry); 3521 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3522 3523 /* Update entry2counter mapping */ 3524 cntr = mcam->entry2cntr_map[req->entry]; 3525 if (cntr != NPC_MCAM_INVALID_MAP) 3526 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3527 req->entry, cntr); 3528 3529 if (is_cn20k(rvu->pdev)) { 3530 rc = npc_cn20k_idx_free(rvu, &req->entry, 1); 3531 if (rc) 3532 dev_err(rvu->dev, 3533 "Failed to free index=%u\n", 3534 req->entry); 3535 } 3536 3537 goto exit; 3538 3539 free_all: 3540 /* Free up all entries allocated to requesting PFFUNC */ 3541 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 3542 exit: 3543 mutex_unlock(&mcam->lock); 3544 return rc; 3545 } 3546 3547 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 3548 struct npc_mcam_read_entry_req *req, 3549 struct npc_mcam_read_entry_rsp *rsp) 3550 { 3551 struct npc_mcam *mcam = &rvu->hw->mcam; 3552 u16 pcifunc = req->hdr.pcifunc; 3553 int blkaddr, rc; 3554 3555 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3556 if (blkaddr < 0) 3557 return NPC_MCAM_INVALID_REQ; 3558 3559 mutex_lock(&mcam->lock); 3560 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3561 if (!rc) { 3562 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 3563 &rsp->entry_data, 3564 &rsp->intf, &rsp->enable); 3565 } 3566 3567 mutex_unlock(&mcam->lock); 3568 return rc; 3569 } 3570 3571 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 3572 struct npc_mcam_write_entry_req *req, 3573 struct msg_rsp *rsp) 3574 { 3575 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3576 struct npc_mcam *mcam = &rvu->hw->mcam; 3577 u16 pcifunc = req->hdr.pcifunc; 3578 int blkaddr, rc; 3579 u8 nix_intf; 3580 3581 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3582 if (blkaddr < 0) 3583 return NPC_MCAM_INVALID_REQ; 3584 3585 mutex_lock(&mcam->lock); 3586 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3587 if (rc) 3588 goto exit; 3589 3590 if (!is_cn20k(rvu->pdev)) { 3591 /* Verify counter in SoCs other than cn20k */ 3592 if (req->set_cntr && 3593 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 3594 rc = NPC_MCAM_INVALID_REQ; 3595 goto exit; 3596 } 3597 } 3598 3599 if (!is_npc_interface_valid(rvu, req->intf)) { 3600 rc = NPC_MCAM_INVALID_REQ; 3601 goto exit; 3602 } 3603 3604 if (is_npc_intf_tx(req->intf)) 3605 nix_intf = pfvf->nix_tx_intf; 3606 else 3607 nix_intf = pfvf->nix_rx_intf; 3608 3609 /* For AF installed rules, the nix_intf should be set to target NIX */ 3610 if (is_pffunc_af(req->hdr.pcifunc)) 3611 nix_intf = req->intf; 3612 3613 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 3614 &req->entry_data, req->enable_entry); 3615 3616 if (req->set_cntr) 3617 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3618 req->entry, req->cntr); 3619 3620 rc = 0; 3621 exit: 3622 mutex_unlock(&mcam->lock); 3623 return rc; 3624 } 3625 3626 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 3627 struct npc_mcam_ena_dis_entry_req *req, 3628 struct msg_rsp *rsp) 3629 { 3630 struct npc_mcam *mcam = &rvu->hw->mcam; 3631 u16 pcifunc = req->hdr.pcifunc; 3632 int blkaddr, rc; 3633 3634 req->entry = npc_cn20k_vidx2idx(req->entry); 3635 3636 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3637 if (blkaddr < 0) 3638 return NPC_MCAM_INVALID_REQ; 3639 3640 mutex_lock(&mcam->lock); 3641 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3642 mutex_unlock(&mcam->lock); 3643 if (rc) 3644 return rc; 3645 3646 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 3647 3648 return 0; 3649 } 3650 3651 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 3652 struct npc_mcam_ena_dis_entry_req *req, 3653 struct msg_rsp *rsp) 3654 { 3655 struct npc_mcam *mcam = &rvu->hw->mcam; 3656 u16 pcifunc = req->hdr.pcifunc; 3657 int blkaddr, rc; 3658 3659 req->entry = npc_cn20k_vidx2idx(req->entry); 3660 3661 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3662 if (blkaddr < 0) 3663 return NPC_MCAM_INVALID_REQ; 3664 3665 mutex_lock(&mcam->lock); 3666 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3667 mutex_unlock(&mcam->lock); 3668 if (rc) 3669 return rc; 3670 3671 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3672 3673 return 0; 3674 } 3675 3676 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 3677 struct npc_mcam_shift_entry_req *req, 3678 struct npc_mcam_shift_entry_rsp *rsp) 3679 { 3680 struct npc_mcam *mcam = &rvu->hw->mcam; 3681 u16 pcifunc = req->hdr.pcifunc; 3682 u16 old_entry, new_entry; 3683 int blkaddr, rc = 0; 3684 u16 index, cntr; 3685 3686 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3687 if (blkaddr < 0) 3688 return NPC_MCAM_INVALID_REQ; 3689 3690 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 3691 return NPC_MCAM_INVALID_REQ; 3692 3693 mutex_lock(&mcam->lock); 3694 for (index = 0; index < req->shift_count; index++) { 3695 old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]); 3696 new_entry = npc_cn20k_vidx2idx(req->new_entry[index]); 3697 3698 /* Check if both old and new entries are valid and 3699 * does belong to this PFFUNC or not. 3700 */ 3701 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 3702 if (rc) 3703 break; 3704 3705 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 3706 if (rc) 3707 break; 3708 3709 /* new_entry should not have a counter mapped */ 3710 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 3711 rc = NPC_MCAM_PERM_DENIED; 3712 break; 3713 } 3714 3715 /* Disable the new_entry */ 3716 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 3717 3718 /* Copy rule from old entry to new entry */ 3719 if (npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry)) { 3720 rc = NPC_MCAM_INVALID_REQ; 3721 break; 3722 } 3723 3724 /* Copy counter mapping, if any */ 3725 cntr = mcam->entry2cntr_map[old_entry]; 3726 if (cntr != NPC_MCAM_INVALID_MAP) { 3727 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3728 old_entry, cntr); 3729 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3730 new_entry, cntr); 3731 } 3732 3733 /* Enable new_entry and disable old_entry */ 3734 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 3735 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 3736 } 3737 3738 /* If shift has failed then report the failed index */ 3739 if (index != req->shift_count) { 3740 if (!rc) 3741 rc = NPC_MCAM_PERM_DENIED; 3742 rsp->failed_entry_idx = index; 3743 } 3744 3745 mutex_unlock(&mcam->lock); 3746 return rc; 3747 } 3748 3749 static int __npc_mcam_alloc_counter(struct rvu *rvu, 3750 struct npc_mcam_alloc_counter_req *req, 3751 struct npc_mcam_alloc_counter_rsp *rsp) 3752 { 3753 struct npc_mcam *mcam = &rvu->hw->mcam; 3754 u16 pcifunc = req->hdr.pcifunc; 3755 u16 max_contig, cntr; 3756 int blkaddr, index; 3757 3758 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3759 if (blkaddr < 0) 3760 return NPC_MCAM_INVALID_REQ; 3761 3762 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 3763 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3764 return NPC_MCAM_INVALID_REQ; 3765 3766 /* Since list of allocated counter IDs needs to be sent to requester, 3767 * max number of non-contiguous counters per mbox msg is limited. 3768 */ 3769 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 3770 return NPC_MCAM_INVALID_REQ; 3771 3772 /* Check if unused counters are available or not */ 3773 if (!rvu_rsrc_free_count(&mcam->counters)) { 3774 return NPC_MCAM_ALLOC_FAILED; 3775 } 3776 3777 rsp->count = 0; 3778 3779 if (req->contig) { 3780 /* Allocate requested number of contiguous counters, if 3781 * unsuccessful find max contiguous entries available. 3782 */ 3783 index = npc_mcam_find_zero_area(mcam->counters.bmap, 3784 mcam->counters.max, 0, 3785 req->count, &max_contig); 3786 rsp->count = max_contig; 3787 rsp->cntr = index; 3788 for (cntr = index; cntr < (index + max_contig); cntr++) { 3789 __set_bit(cntr, mcam->counters.bmap); 3790 mcam->cntr2pfvf_map[cntr] = pcifunc; 3791 } 3792 } else { 3793 /* Allocate requested number of non-contiguous counters, 3794 * if unsuccessful allocate as many as possible. 3795 */ 3796 for (cntr = 0; cntr < req->count; cntr++) { 3797 index = rvu_alloc_rsrc(&mcam->counters); 3798 if (index < 0) 3799 break; 3800 rsp->cntr_list[cntr] = index; 3801 rsp->count++; 3802 mcam->cntr2pfvf_map[index] = pcifunc; 3803 } 3804 } 3805 3806 return 0; 3807 } 3808 3809 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 3810 struct npc_mcam_alloc_counter_req *req, 3811 struct npc_mcam_alloc_counter_rsp *rsp) 3812 { 3813 struct npc_mcam *mcam = &rvu->hw->mcam; 3814 int err; 3815 3816 /* Counter is not supported for CN20K */ 3817 if (is_cn20k(rvu->pdev)) 3818 return NPC_MCAM_INVALID_REQ; 3819 3820 mutex_lock(&mcam->lock); 3821 3822 err = __npc_mcam_alloc_counter(rvu, req, rsp); 3823 3824 mutex_unlock(&mcam->lock); 3825 return err; 3826 } 3827 3828 static int __npc_mcam_free_counter(struct rvu *rvu, 3829 struct npc_mcam_oper_counter_req *req, 3830 struct msg_rsp *rsp) 3831 { 3832 struct npc_mcam *mcam = &rvu->hw->mcam; 3833 u16 index; 3834 int blkaddr, err; 3835 3836 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3837 if (blkaddr < 0) 3838 return NPC_MCAM_INVALID_REQ; 3839 3840 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3841 if (err) { 3842 return err; 3843 } 3844 3845 /* Mark counter as free/unused */ 3846 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 3847 rvu_free_rsrc(&mcam->counters, req->cntr); 3848 3849 /* Disable all MCAM entry's stats which are using this counter. 3850 * Scan the full MCAM index range: AF-reserved rules (e.g. CPT pass-2) 3851 */ 3852 for (index = 0; index < mcam->total_entries; index++) { 3853 if (!mcam->cntr_refcnt[req->cntr]) 3854 break; 3855 if (mcam->entry2cntr_map[index] != req->cntr) 3856 continue; 3857 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, 3858 req->cntr); 3859 } 3860 3861 return 0; 3862 } 3863 3864 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 3865 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3866 { 3867 struct npc_mcam *mcam = &rvu->hw->mcam; 3868 int err; 3869 3870 /* Counter is not supported for CN20K */ 3871 if (is_cn20k(rvu->pdev)) 3872 return NPC_MCAM_INVALID_REQ; 3873 3874 mutex_lock(&mcam->lock); 3875 3876 err = __npc_mcam_free_counter(rvu, req, rsp); 3877 3878 mutex_unlock(&mcam->lock); 3879 3880 return err; 3881 } 3882 3883 void __rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 3884 struct rvu_npc_mcam_rule *rule) 3885 { 3886 struct npc_mcam_oper_counter_req free_req = { 0 }; 3887 struct msg_rsp free_rsp; 3888 3889 if (!rule->has_cntr) 3890 return; 3891 3892 free_req.hdr.pcifunc = pcifunc; 3893 free_req.cntr = rule->cntr; 3894 3895 __npc_mcam_free_counter(rvu, &free_req, &free_rsp); 3896 rule->has_cntr = false; 3897 } 3898 3899 void __rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 3900 struct rvu_npc_mcam_rule *rule, 3901 struct npc_install_flow_rsp *rsp) 3902 { 3903 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 3904 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 3905 int err; 3906 3907 cntr_req.hdr.pcifunc = pcifunc; 3908 cntr_req.contig = true; 3909 cntr_req.count = 1; 3910 3911 /* we try to allocate a counter to track the stats of this 3912 * rule. If counter could not be allocated then proceed 3913 * without counter because counters are limited than entries. 3914 */ 3915 err = __npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3916 if (!err && cntr_rsp.count) { 3917 rule->cntr = cntr_rsp.cntr; 3918 rule->has_cntr = true; 3919 rsp->counter = rule->cntr; 3920 } else { 3921 rsp->counter = err; 3922 } 3923 } 3924 3925 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 3926 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 3927 { 3928 struct npc_mcam *mcam = &rvu->hw->mcam; 3929 u16 index; 3930 int blkaddr, rc; 3931 3932 /* Counter is not supported for CN20K */ 3933 if (is_cn20k(rvu->pdev)) 3934 return NPC_MCAM_INVALID_REQ; 3935 3936 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3937 if (blkaddr < 0) 3938 return NPC_MCAM_INVALID_REQ; 3939 3940 mutex_lock(&mcam->lock); 3941 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3942 if (rc) 3943 goto exit; 3944 3945 /* Unmap the MCAM entry and counter */ 3946 if (!req->all) { 3947 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 3948 if (rc) 3949 goto exit; 3950 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3951 req->entry, req->cntr); 3952 goto exit; 3953 } 3954 3955 /* Disable all MCAM entry's stats which are using this counter */ 3956 for (index = 0; index < mcam->total_entries; index++) { 3957 if (!mcam->cntr_refcnt[req->cntr]) 3958 break; 3959 if (mcam->entry2cntr_map[index] != req->cntr) 3960 continue; 3961 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, 3962 req->cntr); 3963 } 3964 exit: 3965 mutex_unlock(&mcam->lock); 3966 return rc; 3967 } 3968 3969 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3970 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3971 { 3972 struct npc_mcam *mcam = &rvu->hw->mcam; 3973 int blkaddr, err, index, bank; 3974 3975 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3976 if (blkaddr < 0) 3977 return NPC_MCAM_INVALID_REQ; 3978 3979 /* For cn20k, npc mcam index is passed as cntr, as each 3980 * mcam entry has corresponding counter. 3981 */ 3982 if (is_cn20k(rvu->pdev)) { 3983 index = req->cntr & (mcam->banksize - 1); 3984 bank = npc_get_bank(mcam, req->cntr); 3985 rvu_write64(rvu, blkaddr, 3986 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 3987 return 0; 3988 } 3989 3990 mutex_lock(&mcam->lock); 3991 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3992 mutex_unlock(&mcam->lock); 3993 if (err) 3994 return err; 3995 3996 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3997 3998 return 0; 3999 } 4000 4001 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 4002 struct npc_mcam_oper_counter_req *req, 4003 struct npc_mcam_oper_counter_rsp *rsp) 4004 { 4005 struct npc_mcam *mcam = &rvu->hw->mcam; 4006 int blkaddr, err, index, bank; 4007 u64 regval; 4008 4009 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4010 if (blkaddr < 0) 4011 return NPC_MCAM_INVALID_REQ; 4012 4013 /* In CN20K, mcam index is passed cntr. Each cn20k mcam entry 4014 * has its own counter. No need to verify the counter index. 4015 */ 4016 if (is_cn20k(rvu->pdev)) { 4017 index = req->cntr & (mcam->banksize - 1); 4018 bank = npc_get_bank(mcam, req->cntr); 4019 regval = rvu_read64(rvu, blkaddr, 4020 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 4021 bank)); 4022 rsp->stat = regval; 4023 return 0; 4024 } 4025 4026 mutex_lock(&mcam->lock); 4027 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 4028 mutex_unlock(&mcam->lock); 4029 if (err) 4030 return err; 4031 4032 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 4033 rsp->stat &= BIT_ULL(48) - 1; 4034 4035 return 0; 4036 } 4037 4038 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 4039 struct npc_mcam_alloc_and_write_entry_req *req, 4040 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 4041 { 4042 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 4043 struct npc_mcam_alloc_counter_req cntr_req; 4044 struct npc_mcam_alloc_counter_rsp cntr_rsp; 4045 struct npc_mcam_alloc_entry_req entry_req; 4046 struct npc_mcam_alloc_entry_rsp entry_rsp; 4047 struct npc_mcam *mcam = &rvu->hw->mcam; 4048 u16 entry = NPC_MCAM_ENTRY_INVALID; 4049 u16 cntr = NPC_MCAM_ENTRY_INVALID; 4050 int blkaddr, rc; 4051 u8 nix_intf; 4052 4053 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4054 if (blkaddr < 0) 4055 return NPC_MCAM_INVALID_REQ; 4056 4057 if (!is_npc_interface_valid(rvu, req->intf)) 4058 return NPC_MCAM_INVALID_REQ; 4059 4060 /* Try to allocate a MCAM entry */ 4061 entry_req.hdr.pcifunc = req->hdr.pcifunc; 4062 entry_req.contig = true; 4063 entry_req.ref_prio = req->ref_prio; 4064 entry_req.ref_entry = req->ref_entry; 4065 entry_req.count = 1; 4066 4067 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 4068 &entry_req, &entry_rsp); 4069 if (rc) 4070 return rc; 4071 4072 if (!entry_rsp.count) 4073 return NPC_MCAM_ALLOC_FAILED; 4074 4075 entry = entry_rsp.entry; 4076 4077 if (!req->alloc_cntr) 4078 goto write_entry; 4079 4080 /* Now allocate counter */ 4081 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 4082 cntr_req.contig = true; 4083 cntr_req.count = 1; 4084 4085 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 4086 if (rc) { 4087 /* Free allocated MCAM entry */ 4088 mutex_lock(&mcam->lock); 4089 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 4090 npc_mcam_clear_bit(mcam, entry); 4091 mutex_unlock(&mcam->lock); 4092 return rc; 4093 } 4094 4095 cntr = cntr_rsp.cntr; 4096 4097 write_entry: 4098 mutex_lock(&mcam->lock); 4099 4100 if (is_npc_intf_tx(req->intf)) 4101 nix_intf = pfvf->nix_tx_intf; 4102 else 4103 nix_intf = pfvf->nix_rx_intf; 4104 4105 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 4106 &req->entry_data, req->enable_entry); 4107 4108 if (req->alloc_cntr) 4109 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 4110 mutex_unlock(&mcam->lock); 4111 4112 rsp->entry = entry; 4113 rsp->cntr = cntr; 4114 4115 return 0; 4116 } 4117 4118 #define GET_KEX_CFG(intf) \ 4119 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 4120 4121 #define GET_KEX_FLAGS(ld) \ 4122 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 4123 4124 #define GET_KEX_LD(intf, lid, lt, ld) \ 4125 rvu_read64(rvu, BLKADDR_NPC, \ 4126 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 4127 4128 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 4129 rvu_read64(rvu, BLKADDR_NPC, \ 4130 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 4131 4132 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 4133 struct npc_get_kex_cfg_rsp *rsp) 4134 { 4135 int lid, lt, ld, fl; 4136 4137 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 4138 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 4139 for (lid = 0; lid < NPC_MAX_LID; lid++) { 4140 for (lt = 0; lt < NPC_MAX_LT; lt++) { 4141 for (ld = 0; ld < NPC_MAX_LD; ld++) { 4142 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 4143 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 4144 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 4145 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 4146 } 4147 } 4148 } 4149 for (ld = 0; ld < NPC_MAX_LD; ld++) 4150 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 4151 4152 for (ld = 0; ld < NPC_MAX_LD; ld++) { 4153 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 4154 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 4155 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 4156 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 4157 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 4158 } 4159 } 4160 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 4161 return 0; 4162 } 4163 4164 static int 4165 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 4166 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 4167 { 4168 struct npc_kpu_action0 *act0; 4169 u8 shift_count = 0; 4170 int blkaddr; 4171 u64 val; 4172 4173 if (!var_len_off_mask) 4174 return -EINVAL; 4175 4176 if (var_len_off_mask != 0xff) { 4177 if (shift_dir) 4178 shift_count = __ffs(var_len_off_mask); 4179 else 4180 shift_count = (8 - __fls(var_len_off_mask)); 4181 } 4182 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 4183 if (blkaddr < 0) { 4184 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4185 return -EINVAL; 4186 } 4187 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 4188 act0 = (struct npc_kpu_action0 *)&val; 4189 act0->var_len_shift = shift_count; 4190 act0->var_len_right = shift_dir; 4191 act0->var_len_mask = var_len_off_mask; 4192 act0->var_len_offset = var_len_off; 4193 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 4194 return 0; 4195 } 4196 4197 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 4198 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 4199 u8 shift_dir) 4200 4201 { 4202 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 4203 int blkaddr, nixlf, rc, intf_mode; 4204 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4205 u64 rxpkind, txpkind; 4206 u8 cgx_id, lmac_id; 4207 4208 /* use default pkind to disable edsa/higig */ 4209 rxpkind = rvu_npc_get_pkind(rvu, pf); 4210 txpkind = NPC_TX_DEF_PKIND; 4211 intf_mode = NPC_INTF_MODE_DEF; 4212 4213 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 4214 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 4215 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 4216 var_len_off, 4217 var_len_off_mask, 4218 shift_dir); 4219 if (rc) 4220 return rc; 4221 } 4222 rxpkind = pkind; 4223 txpkind = pkind; 4224 } 4225 4226 if (dir & PKIND_RX) { 4227 /* rx pkind set req valid only for cgx mapped PFs */ 4228 if (!is_cgx_config_permitted(rvu, pcifunc)) 4229 return 0; 4230 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 4231 4232 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 4233 rxpkind); 4234 if (rc) 4235 return rc; 4236 } 4237 4238 if (dir & PKIND_TX) { 4239 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 4240 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4241 if (rc) 4242 return rc; 4243 4244 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 4245 txpkind); 4246 } 4247 4248 pfvf->intf_mode = intf_mode; 4249 return 0; 4250 } 4251 4252 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 4253 struct msg_rsp *rsp) 4254 { 4255 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 4256 req->dir, req->pkind, req->var_len_off, 4257 req->var_len_off_mask, req->shift_dir); 4258 } 4259 4260 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 4261 struct msg_req *req, 4262 struct npc_mcam_read_base_rule_rsp *rsp) 4263 { 4264 struct npc_mcam *mcam = &rvu->hw->mcam; 4265 int index, blkaddr, nixlf, rc = 0; 4266 u16 pcifunc = req->hdr.pcifunc; 4267 struct rvu_pfvf *pfvf; 4268 u8 intf, enable; 4269 4270 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4271 if (blkaddr < 0) 4272 return NPC_MCAM_INVALID_REQ; 4273 4274 /* Return the channel number in case of PF */ 4275 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 4276 pfvf = rvu_get_pfvf(rvu, pcifunc); 4277 rsp->entry.kw[0] = pfvf->rx_chan_base; 4278 rsp->entry.kw_mask[0] = 0xFFFULL; 4279 goto out; 4280 } 4281 4282 /* Find the pkt steering rule installed by PF to this VF */ 4283 mutex_lock(&mcam->lock); 4284 for (index = 0; index < mcam->bmap_entries; index++) { 4285 if (mcam->entry2target_pffunc[index] == pcifunc) 4286 goto read_entry; 4287 } 4288 4289 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4290 if (rc < 0) { 4291 mutex_unlock(&mcam->lock); 4292 goto out; 4293 } 4294 /* Read the default ucast entry if there is no pkt steering rule */ 4295 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 4296 NIXLF_UCAST_ENTRY); 4297 if (index < 0) { 4298 mutex_unlock(&mcam->lock); 4299 rc = NIX_AF_ERR_AF_LF_INVALID; 4300 goto out; 4301 } 4302 4303 read_entry: 4304 /* Read the mcam entry */ 4305 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 4306 &enable); 4307 mutex_unlock(&mcam->lock); 4308 out: 4309 return rc; 4310 } 4311 4312 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 4313 struct npc_mcam_get_stats_req *req, 4314 struct npc_mcam_get_stats_rsp *rsp) 4315 { 4316 struct npc_mcam *mcam = &rvu->hw->mcam; 4317 u16 index, cntr; 4318 int blkaddr; 4319 u64 regval; 4320 u32 bank; 4321 4322 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4323 if (blkaddr < 0) 4324 return NPC_MCAM_INVALID_REQ; 4325 4326 req->entry = npc_cn20k_vidx2idx(req->entry); 4327 4328 index = req->entry & (mcam->banksize - 1); 4329 bank = npc_get_bank(mcam, req->entry); 4330 4331 mutex_lock(&mcam->lock); 4332 4333 if (is_cn20k(rvu->pdev)) { 4334 regval = rvu_read64(rvu, blkaddr, 4335 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 4336 bank)); 4337 rsp->stat_ena = 1; 4338 rsp->stat = regval; 4339 mutex_unlock(&mcam->lock); 4340 return 0; 4341 } 4342 4343 /* read MCAM entry STAT_ACT register */ 4344 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, 4345 bank)); 4346 4347 if (!(regval & rvu->hw->npc_stat_ena)) { 4348 rsp->stat_ena = 0; 4349 mutex_unlock(&mcam->lock); 4350 return 0; 4351 } 4352 4353 cntr = regval & 0x1FF; 4354 4355 rsp->stat_ena = 1; 4356 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 4357 rsp->stat &= BIT_ULL(48) - 1; 4358 4359 mutex_unlock(&mcam->lock); 4360 4361 return 0; 4362 } 4363 4364 void rvu_npc_clear_ucast_entry(struct rvu *rvu, int pcifunc, int nixlf) 4365 { 4366 struct npc_mcam *mcam = &rvu->hw->mcam; 4367 struct rvu_npc_mcam_rule *rule; 4368 int ucast_idx, blkaddr; 4369 4370 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4371 if (blkaddr < 0) 4372 return; 4373 4374 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 4375 nixlf, NIXLF_UCAST_ENTRY); 4376 4377 /* In cn20k, default rules are freed before detach rsrc */ 4378 if (ucast_idx < 0) 4379 return; 4380 4381 npc_enable_mcam_entry(rvu, mcam, blkaddr, ucast_idx, false); 4382 4383 npc_set_mcam_action(rvu, mcam, blkaddr, ucast_idx, 0); 4384 4385 npc_clear_mcam_entry(rvu, mcam, blkaddr, ucast_idx); 4386 4387 mutex_lock(&mcam->lock); 4388 list_for_each_entry(rule, &mcam->mcam_rules, list) { 4389 if (rule->entry == ucast_idx) { 4390 list_del(&rule->list); 4391 kfree(rule); 4392 break; 4393 } 4394 } 4395 mutex_unlock(&mcam->lock); 4396 } 4397