1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 #include "rvu_npc_hash.h" 19 #include "cn20k/npc.h" 20 #include "rvu_npc.h" 21 #include "cn20k/reg.h" 22 #include "lmac_common.h" 23 24 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 25 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 26 27 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 28 #define NPC_HW_TSTAMP_OFFSET 8ULL 29 #define NPC_KEX_CHAN_MASK 0xFFFULL 30 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 31 32 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 33 34 static const char def_pfl_name[] = "default"; 35 36 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 37 int blkaddr, u16 pcifunc); 38 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 39 u16 pcifunc); 40 41 bool is_npc_intf_tx(u8 intf) 42 { 43 return !!(intf & 0x1); 44 } 45 46 bool is_npc_intf_rx(u8 intf) 47 { 48 return !(intf & 0x1); 49 } 50 51 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 52 { 53 struct rvu_hwinfo *hw = rvu->hw; 54 55 return intf < hw->npc_intfs; 56 } 57 58 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 59 { 60 /* Due to a HW issue in these silicon versions, parse nibble enable 61 * configuration has to be identical for both Rx and Tx interfaces. 62 */ 63 if (is_rvu_96xx_B0(rvu)) 64 return nibble_ena; 65 return 0; 66 } 67 68 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 69 { 70 int blkaddr; 71 u64 val = 0; 72 73 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 74 if (blkaddr < 0) 75 return; 76 77 /* Config CPI base for the PKIND */ 78 val = pkind | 1ULL << 62; 79 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 80 } 81 82 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 83 { 84 struct npc_pkind *pkind = &rvu->hw->pkind; 85 u32 map; 86 int i; 87 88 for (i = 0; i < pkind->rsrc.max; i++) { 89 map = pkind->pfchan_map[i]; 90 if (((map >> 16) & 0x3F) == pf) 91 return i; 92 } 93 return -1; 94 } 95 96 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 97 98 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 99 { 100 int pkind, blkaddr; 101 u64 val; 102 103 pkind = rvu_npc_get_pkind(rvu, pf); 104 if (pkind < 0) { 105 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 106 return -EINVAL; 107 } 108 109 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 110 if (blkaddr < 0) { 111 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 112 return -EINVAL; 113 } 114 115 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 116 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 117 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 118 if (enable) 119 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 120 NPC_HW_TSTAMP_OFFSET); 121 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 122 123 return 0; 124 } 125 126 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 127 int nixlf) 128 { 129 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 130 struct rvu *rvu = hw->rvu; 131 int blkaddr = 0, max = 0; 132 struct rvu_block *block; 133 struct rvu_pfvf *pfvf; 134 135 pfvf = rvu_get_pfvf(rvu, pcifunc); 136 /* Given a PF/VF and NIX LF number calculate the unicast mcam 137 * entry index based on the NIX block assigned to the PF/VF. 138 */ 139 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 140 while (blkaddr) { 141 if (pfvf->nix_blkaddr == blkaddr) 142 break; 143 block = &rvu->hw->block[blkaddr]; 144 max += block->lf.max; 145 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 146 } 147 148 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 149 } 150 151 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 152 u16 pcifunc, int nixlf, int type) 153 { 154 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 155 struct rvu *rvu = hw->rvu; 156 u16 bcast, mcast, promisc, ucast; 157 int index; 158 int rc; 159 int pf; 160 161 if (is_cn20k(rvu->pdev)) { 162 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 163 &promisc, &ucast); 164 if (rc) 165 return -EFAULT; 166 167 if (is_lbk_vf(rvu, pcifunc)) { 168 if (promisc == USHRT_MAX) 169 return -EINVAL; 170 return promisc; 171 } 172 173 if (is_cgx_vf(rvu, pcifunc)) { 174 if (ucast == USHRT_MAX) 175 return -EINVAL; 176 177 return ucast; 178 } 179 180 switch (type) { 181 case NIXLF_BCAST_ENTRY: 182 if (bcast == USHRT_MAX) 183 return -EINVAL; 184 return bcast; 185 case NIXLF_ALLMULTI_ENTRY: 186 if (mcast == USHRT_MAX) 187 return -EINVAL; 188 return mcast; 189 case NIXLF_PROMISC_ENTRY: 190 if (promisc == USHRT_MAX) 191 return -EINVAL; 192 return promisc; 193 case NIXLF_UCAST_ENTRY: 194 if (ucast == USHRT_MAX) 195 return -EINVAL; 196 return ucast; 197 default: 198 return -EINVAL; 199 } 200 } 201 202 /* Check if this is for a PF */ 203 pf = rvu_get_pf(rvu->pdev, pcifunc); 204 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 205 /* Reserved entries exclude PF0 */ 206 pf--; 207 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 208 /* Broadcast address matching entry should be first so 209 * that the packet can be replicated to all VFs. 210 */ 211 if (type == NIXLF_BCAST_ENTRY) 212 return index; 213 else if (type == NIXLF_ALLMULTI_ENTRY) 214 return index + 1; 215 else if (type == NIXLF_PROMISC_ENTRY) 216 return index + 2; 217 } 218 219 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 220 } 221 222 int npc_get_bank(struct npc_mcam *mcam, int index) 223 { 224 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 225 int bank = index / mcam->banksize; 226 struct rvu *rvu = hw->rvu; 227 228 if (is_cn20k(rvu->pdev)) 229 return bank; 230 231 /* 0,1 & 2,3 banks are combined for this keysize */ 232 if (mcam->keysize == NPC_MCAM_KEY_X2) 233 return bank ? 2 : 0; 234 235 return bank; 236 } 237 238 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 239 int blkaddr, int index) 240 { 241 int bank = npc_get_bank(mcam, index); 242 u64 cfg; 243 244 index &= (mcam->banksize - 1); 245 if (is_cn20k(rvu->pdev)) 246 cfg = rvu_read64(rvu, blkaddr, 247 NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, 248 bank)); 249 else 250 cfg = rvu_read64(rvu, blkaddr, 251 NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 252 253 return (cfg & 1); 254 } 255 256 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 257 int blkaddr, int index, bool enable) 258 { 259 int bank = npc_get_bank(mcam, index); 260 int actbank = bank; 261 262 if (is_cn20k(rvu->pdev)) { 263 if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable)) 264 dev_err(rvu->dev, "Error to %s mcam %u entry\n", 265 enable ? "enable" : "disable", index); 266 return; 267 } 268 269 index &= (mcam->banksize - 1); 270 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 271 rvu_write64(rvu, blkaddr, 272 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 273 enable ? 1 : 0); 274 } 275 } 276 277 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 278 int blkaddr, int index) 279 { 280 int bank = npc_get_bank(mcam, index); 281 int actbank = bank; 282 283 if (is_cn20k(rvu->pdev)) { 284 if (npc_cn20k_clear_mcam_entry(rvu, blkaddr, index)) 285 dev_err(rvu->dev, "%s Failed to clear mcam %u\n", 286 __func__, index); 287 return; 288 } 289 290 index &= (mcam->banksize - 1); 291 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 292 rvu_write64(rvu, blkaddr, 293 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 294 rvu_write64(rvu, blkaddr, 295 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 296 297 rvu_write64(rvu, blkaddr, 298 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 299 rvu_write64(rvu, blkaddr, 300 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 301 302 rvu_write64(rvu, blkaddr, 303 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 304 rvu_write64(rvu, blkaddr, 305 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 306 } 307 } 308 309 static void npc_get_keyword(struct mcam_entry *entry, int idx, 310 u64 *cam0, u64 *cam1) 311 { 312 u64 kw_mask = 0x00; 313 314 #define CAM_MASK(n) (BIT_ULL(n) - 1) 315 316 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 317 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 318 * 319 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 320 */ 321 switch (idx) { 322 case 0: 323 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 324 *cam1 = entry->kw[0]; 325 kw_mask = entry->kw_mask[0]; 326 break; 327 case 1: 328 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 329 *cam1 = entry->kw[1] & CAM_MASK(48); 330 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 331 break; 332 case 2: 333 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 334 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 335 */ 336 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 337 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 338 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 339 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 340 break; 341 case 3: 342 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 343 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 344 */ 345 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 346 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 347 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 348 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 349 break; 350 case 4: 351 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 352 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 353 */ 354 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 355 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 356 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 357 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 358 break; 359 case 5: 360 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 361 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 362 */ 363 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 364 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 365 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 366 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 367 break; 368 case 6: 369 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 370 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 371 */ 372 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 373 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 374 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 375 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 376 break; 377 case 7: 378 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 379 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 380 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 381 break; 382 } 383 384 *cam1 &= kw_mask; 385 *cam0 = ~*cam1 & kw_mask; 386 } 387 388 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 389 u64 cam0, u64 cam1) 390 { 391 /* Similar to npc_get_keyword, but fills mcam_entry structure from 392 * CAM registers. 393 */ 394 switch (idx) { 395 case 0: 396 entry->kw[0] = cam1; 397 entry->kw_mask[0] = cam1 ^ cam0; 398 break; 399 case 1: 400 entry->kw[1] = cam1; 401 entry->kw_mask[1] = cam1 ^ cam0; 402 break; 403 case 2: 404 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 405 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 406 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 407 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 408 break; 409 case 3: 410 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 411 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 412 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 413 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 414 break; 415 case 4: 416 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 417 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 418 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 419 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 420 break; 421 case 5: 422 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 423 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 424 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 425 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 426 break; 427 case 6: 428 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 429 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 430 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 431 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 432 break; 433 case 7: 434 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 435 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 436 break; 437 } 438 } 439 440 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 441 int blkaddr, u16 pf_func) 442 { 443 int bank, nixlf, index; 444 u64 reg; 445 446 /* get ucast entry rule entry index */ 447 if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { 448 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n", 449 __func__, pf_func); 450 /* Action 0 is drop */ 451 return 0; 452 } 453 454 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 455 NIXLF_UCAST_ENTRY); 456 457 if (index < 0) { 458 dev_err(rvu->dev, 459 "%s: failed to get ucast entry pcifunc:0x%x\n", 460 __func__, pf_func); 461 /* Action 0 is drop */ 462 return 0; 463 } 464 465 bank = npc_get_bank(mcam, index); 466 index &= (mcam->banksize - 1); 467 468 if (is_cn20k(rvu->pdev)) 469 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 470 else 471 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 472 473 return rvu_read64(rvu, blkaddr, reg); 474 } 475 476 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 477 int blkaddr, int index, struct mcam_entry *entry, 478 bool *enable) 479 { 480 struct rvu_npc_mcam_rule *rule; 481 u16 owner, target_func; 482 struct rvu_pfvf *pfvf; 483 u64 rx_action; 484 485 owner = mcam->entry2pfvf_map[index]; 486 target_func = (entry->action >> 4) & 0xffff; 487 /* do nothing when target is LBK/PF or owner is not PF */ 488 if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) || 489 (owner & RVU_PFVF_FUNC_MASK) || 490 !(target_func & RVU_PFVF_FUNC_MASK)) 491 return; 492 493 /* save entry2target_pffunc */ 494 pfvf = rvu_get_pfvf(rvu, target_func); 495 mcam->entry2target_pffunc[index] = target_func; 496 497 /* don't enable rule when nixlf not attached or initialized */ 498 if (!(is_nixlf_attached(rvu, target_func) && 499 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 500 *enable = false; 501 502 /* fix up not needed for the rules added by user(ntuple filters) */ 503 list_for_each_entry(rule, &mcam->mcam_rules, list) { 504 if (rule->entry == index) 505 return; 506 } 507 508 /* AF modifies given action iff PF/VF has requested for it */ 509 if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT) 510 return; 511 512 /* copy VF default entry action to the VF mcam entry */ 513 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 514 target_func); 515 if (rx_action) 516 entry->action = rx_action; 517 } 518 519 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 520 int blkaddr, int index, u8 intf, 521 struct mcam_entry *entry, bool enable) 522 { 523 int bank = npc_get_bank(mcam, index); 524 int kw = 0, actbank, actindex; 525 u8 tx_intf_mask = ~intf & 0x3; 526 u8 tx_intf = intf; 527 u64 cam0, cam1; 528 529 actbank = bank; /* Save bank id, to set action later on */ 530 actindex = index; 531 index &= (mcam->banksize - 1); 532 533 /* Disable before mcam entry update */ 534 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 535 536 /* Clear mcam entry to avoid writes being suppressed by NPC */ 537 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 538 539 /* CAM1 takes the comparison value and 540 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 541 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 542 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 543 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 544 */ 545 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 546 /* Interface should be set in all banks */ 547 if (is_npc_intf_tx(intf)) { 548 /* Last bit must be set and rest don't care 549 * for TX interfaces 550 */ 551 tx_intf_mask = 0x1; 552 tx_intf = intf & tx_intf_mask; 553 tx_intf_mask = ~tx_intf & tx_intf_mask; 554 } 555 556 rvu_write64(rvu, blkaddr, 557 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 558 tx_intf); 559 rvu_write64(rvu, blkaddr, 560 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 561 tx_intf_mask); 562 563 /* Set the match key */ 564 npc_get_keyword(entry, kw, &cam0, &cam1); 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 567 rvu_write64(rvu, blkaddr, 568 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 569 570 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 571 rvu_write64(rvu, blkaddr, 572 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 573 rvu_write64(rvu, blkaddr, 574 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 575 } 576 577 /* PF installing VF rule */ 578 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 579 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 580 581 /* Set 'action' */ 582 rvu_write64(rvu, blkaddr, 583 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 584 585 /* Set TAG 'action' */ 586 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 587 entry->vtag_action); 588 589 /* Enable the entry */ 590 if (enable) 591 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 592 } 593 594 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 595 int blkaddr, u16 src, 596 struct mcam_entry *entry, u8 *intf, u8 *ena) 597 { 598 int sbank = npc_get_bank(mcam, src); 599 int bank, kw = 0; 600 u64 cam0, cam1; 601 602 src &= (mcam->banksize - 1); 603 bank = sbank; 604 605 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 606 cam1 = rvu_read64(rvu, blkaddr, 607 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 608 cam0 = rvu_read64(rvu, blkaddr, 609 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 610 npc_fill_entryword(entry, kw, cam0, cam1); 611 612 cam1 = rvu_read64(rvu, blkaddr, 613 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 614 cam0 = rvu_read64(rvu, blkaddr, 615 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 616 npc_fill_entryword(entry, kw + 1, cam0, cam1); 617 } 618 619 entry->action = rvu_read64(rvu, blkaddr, 620 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 621 entry->vtag_action = 622 rvu_read64(rvu, blkaddr, 623 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 624 *intf = rvu_read64(rvu, blkaddr, 625 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 626 *ena = rvu_read64(rvu, blkaddr, 627 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 628 } 629 630 static int npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 631 int blkaddr, u16 src, u16 dest) 632 { 633 int dbank = npc_get_bank(mcam, dest); 634 int sbank = npc_get_bank(mcam, src); 635 u64 cfg, sreg, dreg; 636 int bank, i; 637 638 if (is_cn20k(rvu->pdev)) 639 return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest); 640 641 src &= (mcam->banksize - 1); 642 dest &= (mcam->banksize - 1); 643 644 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 645 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 646 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 647 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 648 for (i = 0; i < 6; i++) { 649 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 650 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 651 } 652 } 653 654 /* Copy action */ 655 cfg = rvu_read64(rvu, blkaddr, 656 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 657 rvu_write64(rvu, blkaddr, 658 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 659 660 /* Copy TAG action */ 661 cfg = rvu_read64(rvu, blkaddr, 662 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 663 rvu_write64(rvu, blkaddr, 664 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 665 666 /* Enable or disable */ 667 cfg = rvu_read64(rvu, blkaddr, 668 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 669 rvu_write64(rvu, blkaddr, 670 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 671 return 0; 672 } 673 674 u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 675 int blkaddr, int index) 676 { 677 int bank = npc_get_bank(mcam, index); 678 u64 reg; 679 680 index &= (mcam->banksize - 1); 681 682 if (is_cn20k(rvu->pdev)) 683 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 684 else 685 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 686 return rvu_read64(rvu, blkaddr, reg); 687 } 688 689 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 690 int blkaddr, int index, u64 cfg) 691 { 692 int bank = npc_get_bank(mcam, index); 693 u64 reg; 694 695 index &= (mcam->banksize - 1); 696 697 if (is_cn20k(rvu->pdev)) 698 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 699 else 700 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 701 702 return rvu_write64(rvu, blkaddr, reg, cfg); 703 } 704 705 u32 rvu_get_cpt_chan_mask(struct rvu *rvu) 706 { 707 /* For cn10k the upper two bits of the channel number are 708 * cpt channel number. with masking out these bits in the 709 * mcam entry, same entry used for NIX will allow packets 710 * received from cpt for parsing. 711 */ 712 if (!is_rvu_otx2(rvu)) 713 return NIX_CHAN_CPT_X2P_MASK; 714 else 715 return 0xFFFu; 716 } 717 718 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 719 int nixlf, u64 chan, u8 *mac_addr) 720 { 721 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 722 struct npc_install_flow_req req = { 0 }; 723 struct npc_install_flow_rsp rsp = { 0 }; 724 struct npc_mcam *mcam = &rvu->hw->mcam; 725 struct nix_rx_action action = { 0 }; 726 int blkaddr, index; 727 728 /* AF's and SDP VFs work in promiscuous mode */ 729 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 730 return; 731 732 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 733 if (blkaddr < 0) 734 return; 735 736 /* Ucast rule should not be installed if DMAC 737 * extraction is not supported by the profile. 738 */ 739 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf)) 740 return; 741 742 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 743 nixlf, NIXLF_UCAST_ENTRY); 744 if (index < 0) { 745 dev_err(rvu->dev, 746 "%s: Error to get ucast entry for pcifunc=%#x\n", 747 __func__, pcifunc); 748 return; 749 } 750 751 /* Don't change the action if entry is already enabled 752 * Otherwise RSS action may get overwritten. 753 */ 754 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 755 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 756 blkaddr, index); 757 } else { 758 action.op = NIX_RX_ACTIONOP_UCAST; 759 action.pf_func = pcifunc; 760 } 761 762 req.default_rule = 1; 763 ether_addr_copy(req.packet.dmac, mac_addr); 764 eth_broadcast_addr((u8 *)&req.mask.dmac); 765 req.features = BIT_ULL(NPC_DMAC); 766 req.channel = chan; 767 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 768 req.intf = pfvf->nix_rx_intf; 769 req.op = action.op; 770 req.hdr.pcifunc = 0; /* AF is requester */ 771 req.vf = action.pf_func; 772 req.index = action.index; 773 req.match_id = action.match_id; 774 req.flow_key_alg = action.flow_key_alg; 775 776 if (is_cn20k(rvu->pdev)) 777 req.hw_prio = pfvf->hw_prio; 778 779 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 780 } 781 782 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 783 int nixlf, u64 chan, u8 chan_cnt) 784 { 785 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 786 struct npc_install_flow_req req = { 0 }; 787 struct npc_install_flow_rsp rsp = { 0 }; 788 struct npc_mcam *mcam = &rvu->hw->mcam; 789 struct rvu_hwinfo *hw = rvu->hw; 790 int blkaddr, ucast_idx, index; 791 struct nix_rx_action action = { 0 }; 792 u64 relaxed_mask; 793 u8 flow_key_alg; 794 795 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 796 return; 797 798 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 799 if (blkaddr < 0) 800 return; 801 802 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 803 nixlf, NIXLF_PROMISC_ENTRY); 804 805 /* In cn20k, default indexes are installed only for CGX mapped 806 * and lbk interfaces 807 */ 808 if (is_cgx_vf(rvu, pcifunc)) 809 index = npc_get_nixlf_mcam_index(mcam, 810 pcifunc & ~RVU_PFVF_FUNC_MASK, 811 nixlf, NIXLF_PROMISC_ENTRY); 812 813 if (index < 0) { 814 dev_err(rvu->dev, 815 "%s: Error to get promisc entry for pcifunc=%#x\n", 816 __func__, pcifunc); 817 return; 818 } 819 820 /* If the corresponding PF's ucast action is RSS, 821 * use the same action for promisc also 822 * Please note that for lbk(s) "index" and "ucast_idx" 823 * will be same. 824 */ 825 if (is_lbk_vf(rvu, pcifunc)) 826 ucast_idx = index; 827 else 828 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 829 nixlf, NIXLF_UCAST_ENTRY); 830 if (ucast_idx < 0) { 831 dev_err(rvu->dev, 832 "%s: Error to get ucast/promisc entry for pcifunc=%#x\n", 833 __func__, pcifunc); 834 return; 835 } 836 837 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 838 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 839 blkaddr, ucast_idx); 840 841 if (action.op != NIX_RX_ACTIONOP_RSS) { 842 *(u64 *)&action = 0; 843 action.op = NIX_RX_ACTIONOP_UCAST; 844 } 845 846 flow_key_alg = action.flow_key_alg; 847 848 /* RX_ACTION set to MCAST for CGX PF's */ 849 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 850 is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc))) { 851 *(u64 *)&action = 0; 852 action.op = NIX_RX_ACTIONOP_MCAST; 853 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 854 action.index = pfvf->promisc_mce_idx; 855 } 856 857 /* For cn10k the upper two bits of the channel number are 858 * cpt channel number. with masking out these bits in the 859 * mcam entry, same entry used for NIX will allow packets 860 * received from cpt for parsing. 861 */ 862 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 863 864 if (chan_cnt > 1) { 865 if (!is_power_of_2(chan_cnt)) { 866 dev_err(rvu->dev, 867 "%s: channel count more than 1, must be power of 2\n", __func__); 868 return; 869 } 870 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 871 ilog2(chan_cnt)); 872 req.chan_mask &= relaxed_mask; 873 } 874 875 req.channel = chan; 876 req.intf = pfvf->nix_rx_intf; 877 req.entry = index; 878 req.op = action.op; 879 req.hdr.pcifunc = 0; /* AF is requester */ 880 req.vf = pcifunc; 881 req.index = action.index; 882 req.match_id = action.match_id; 883 req.flow_key_alg = flow_key_alg; 884 885 if (is_cn20k(rvu->pdev)) 886 req.hw_prio = pfvf->hw_prio; 887 888 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 889 } 890 891 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 892 int nixlf, bool enable) 893 { 894 struct npc_mcam *mcam = &rvu->hw->mcam; 895 int blkaddr, index; 896 897 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 898 if (blkaddr < 0) 899 return; 900 901 /* Get 'pcifunc' of PF device */ 902 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 903 904 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 905 nixlf, NIXLF_PROMISC_ENTRY); 906 907 if (index < 0) { 908 dev_err(rvu->dev, 909 "%s: Error to get promisc entry for pcifunc=%#x\n", 910 __func__, pcifunc); 911 return; 912 } 913 914 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 915 } 916 917 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 918 int nixlf, u64 chan) 919 { 920 struct rvu_pfvf *pfvf; 921 struct npc_install_flow_req req = { 0 }; 922 struct npc_install_flow_rsp rsp = { 0 }; 923 struct npc_mcam *mcam = &rvu->hw->mcam; 924 struct rvu_hwinfo *hw = rvu->hw; 925 int blkaddr, index; 926 927 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 928 if (blkaddr < 0) 929 return; 930 931 /* Skip LBK VFs */ 932 if (is_lbk_vf(rvu, pcifunc)) 933 return; 934 935 /* If pkt replication is not supported, 936 * then only PF is allowed to add a bcast match entry. 937 */ 938 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 939 return; 940 941 /* Get 'pcifunc' of PF device */ 942 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 943 pfvf = rvu_get_pfvf(rvu, pcifunc); 944 945 /* Bcast rule should not be installed if both DMAC 946 * and LXMB extraction is not supported by the profile. 947 */ 948 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 949 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 950 return; 951 952 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 953 nixlf, NIXLF_BCAST_ENTRY); 954 if (index < 0) { 955 dev_err(rvu->dev, 956 "%s: Error to get bcast entry for pcifunc=%#x\n", 957 __func__, pcifunc); 958 return; 959 } 960 961 if (!hw->cap.nix_rx_multicast) { 962 /* Early silicon doesn't support pkt replication, 963 * so install entry with UCAST action, so that PF 964 * receives all broadcast packets. 965 */ 966 req.op = NIX_RX_ACTIONOP_UCAST; 967 } else { 968 req.op = NIX_RX_ACTIONOP_MCAST; 969 req.index = pfvf->bcast_mce_idx; 970 } 971 972 eth_broadcast_addr((u8 *)&req.packet.dmac); 973 eth_broadcast_addr((u8 *)&req.mask.dmac); 974 req.features = BIT_ULL(NPC_DMAC); 975 req.channel = chan; 976 req.chan_mask = 0xFFFU; 977 req.intf = pfvf->nix_rx_intf; 978 req.entry = index; 979 req.hdr.pcifunc = 0; /* AF is requester */ 980 req.vf = pcifunc; 981 982 if (is_cn20k(rvu->pdev)) 983 req.hw_prio = pfvf->hw_prio; 984 985 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 986 } 987 988 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 989 u64 chan) 990 { 991 struct npc_install_flow_req req = { 0 }; 992 struct npc_install_flow_rsp rsp = { 0 }; 993 struct npc_mcam *mcam = &rvu->hw->mcam; 994 struct rvu_hwinfo *hw = rvu->hw; 995 int blkaddr, ucast_idx, index; 996 u8 mac_addr[ETH_ALEN] = { 0 }; 997 struct nix_rx_action action = { 0 }; 998 struct rvu_pfvf *pfvf; 999 u8 flow_key_alg; 1000 u16 vf_func; 1001 1002 /* Only CGX PF/VF can add allmulticast entry */ 1003 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 1004 return; 1005 1006 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1007 if (blkaddr < 0) 1008 return; 1009 1010 /* Get 'pcifunc' of PF device */ 1011 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 1012 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 1013 pfvf = rvu_get_pfvf(rvu, pcifunc); 1014 1015 /* Mcast rule should not be installed if both DMAC 1016 * and LXMB extraction is not supported by the profile. 1017 */ 1018 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 1019 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 1020 return; 1021 1022 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1023 nixlf, NIXLF_ALLMULTI_ENTRY); 1024 if (index < 0) { 1025 dev_err(rvu->dev, 1026 "%s: Error to get mcast entry for pcifunc=%#x\n", 1027 __func__, pcifunc); 1028 return; 1029 } 1030 1031 /* If the corresponding PF's ucast action is RSS, 1032 * use the same action for multicast entry also 1033 */ 1034 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 1035 nixlf, NIXLF_UCAST_ENTRY); 1036 if (ucast_idx < 0) { 1037 dev_err(rvu->dev, 1038 "%s: Error to get ucast entry for pcifunc=%#x\n", 1039 __func__, pcifunc); 1040 return; 1041 } 1042 1043 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 1044 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 1045 blkaddr, ucast_idx); 1046 1047 flow_key_alg = action.flow_key_alg; 1048 if (action.op != NIX_RX_ACTIONOP_RSS) { 1049 *(u64 *)&action = 0; 1050 action.op = NIX_RX_ACTIONOP_UCAST; 1051 action.pf_func = pcifunc; 1052 } 1053 1054 /* RX_ACTION set to MCAST for CGX PF's */ 1055 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 1056 *(u64 *)&action = 0; 1057 action.op = NIX_RX_ACTIONOP_MCAST; 1058 action.index = pfvf->mcast_mce_idx; 1059 } 1060 1061 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 1062 ether_addr_copy(req.packet.dmac, mac_addr); 1063 ether_addr_copy(req.mask.dmac, mac_addr); 1064 req.features = BIT_ULL(NPC_DMAC); 1065 1066 req.chan_mask = rvu_get_cpt_chan_mask(rvu); 1067 req.channel = chan; 1068 req.intf = pfvf->nix_rx_intf; 1069 req.entry = index; 1070 req.op = action.op; 1071 req.hdr.pcifunc = 0; /* AF is requester */ 1072 req.vf = pcifunc | vf_func; 1073 req.index = action.index; 1074 req.match_id = action.match_id; 1075 req.flow_key_alg = flow_key_alg; 1076 1077 if (is_cn20k(rvu->pdev)) 1078 req.hw_prio = pfvf->hw_prio; 1079 1080 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 1081 } 1082 1083 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 1084 bool enable) 1085 { 1086 struct npc_mcam *mcam = &rvu->hw->mcam; 1087 int blkaddr, index; 1088 1089 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1090 if (blkaddr < 0) 1091 return; 1092 1093 /* Get 'pcifunc' of PF device */ 1094 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 1095 1096 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 1097 NIXLF_ALLMULTI_ENTRY); 1098 if (index < 0) { 1099 dev_err(rvu->dev, 1100 "%s: Error to get mcast entry for pcifunc=%#x\n", 1101 __func__, pcifunc); 1102 return; 1103 } 1104 1105 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1106 } 1107 1108 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 1109 int blkaddr, u16 pcifunc, u64 rx_action) 1110 { 1111 int actindex, index, bank, entry; 1112 struct rvu_npc_mcam_rule *rule; 1113 bool enable, update; 1114 1115 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 1116 return; 1117 1118 mutex_lock(&mcam->lock); 1119 for (index = 0; index < mcam->bmap_entries; index++) { 1120 if (mcam->entry2target_pffunc[index] != pcifunc) 1121 continue; 1122 update = true; 1123 /* update not needed for the rules added via ntuple filters */ 1124 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1125 if (rule->entry == index) 1126 update = false; 1127 } 1128 if (!update) 1129 continue; 1130 bank = npc_get_bank(mcam, index); 1131 actindex = index; 1132 entry = index & (mcam->banksize - 1); 1133 1134 /* read vf flow entry enable status */ 1135 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 1136 actindex); 1137 /* disable before mcam entry update */ 1138 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 1139 false); 1140 1141 /* update 'action' */ 1142 if (is_cn20k(rvu->pdev)) 1143 rvu_write64(rvu, blkaddr, 1144 NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry, 1145 bank, 1146 0), 1147 rx_action); 1148 else 1149 rvu_write64(rvu, blkaddr, 1150 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 1151 rx_action); 1152 1153 if (enable) 1154 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1155 actindex, true); 1156 } 1157 mutex_unlock(&mcam->lock); 1158 } 1159 1160 static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action, 1161 struct rvu_pfvf *pfvf, int mcam_index, int blkaddr, 1162 int alg_idx) 1163 1164 { 1165 struct npc_mcam *mcam = &rvu->hw->mcam; 1166 struct rvu_hwinfo *hw = rvu->hw; 1167 int bank, op_rss; 1168 u64 reg; 1169 1170 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) 1171 return; 1172 1173 op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list); 1174 1175 bank = npc_get_bank(mcam, mcam_index); 1176 mcam_index &= (mcam->banksize - 1); 1177 1178 if (is_cn20k(rvu->pdev)) 1179 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index, 1180 bank, 0); 1181 else 1182 reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank); 1183 1184 /* If Rx action is MCAST update only RSS algorithm index */ 1185 if (!op_rss) { 1186 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1187 1188 action.flow_key_alg = alg_idx; 1189 } 1190 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1191 } 1192 1193 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 1194 int group, int alg_idx, int mcam_index) 1195 { 1196 struct npc_mcam *mcam = &rvu->hw->mcam; 1197 struct nix_rx_action action; 1198 int blkaddr, index, bank; 1199 struct rvu_pfvf *pfvf; 1200 u64 reg; 1201 1202 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1203 if (blkaddr < 0) 1204 return; 1205 1206 /* Check if this is for reserved default entry */ 1207 if (mcam_index < 0) { 1208 if (group != DEFAULT_RSS_CONTEXT_GROUP) 1209 return; 1210 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1211 nixlf, NIXLF_UCAST_ENTRY); 1212 } else { 1213 /* TODO: validate this mcam index */ 1214 index = mcam_index; 1215 } 1216 1217 if (index < 0 || index >= mcam->total_entries) { 1218 dev_err(rvu->dev, 1219 "%s: Invalid mcam index, pcifunc=%#x\n", 1220 __func__, pcifunc); 1221 return; 1222 } 1223 1224 bank = npc_get_bank(mcam, index); 1225 index &= (mcam->banksize - 1); 1226 1227 if (is_cn20k(rvu->pdev)) 1228 reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); 1229 else 1230 reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); 1231 1232 *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); 1233 /* Ignore if no action was set earlier */ 1234 if (!*(u64 *)&action) 1235 return; 1236 1237 action.op = NIX_RX_ACTIONOP_RSS; 1238 action.pf_func = pcifunc; 1239 action.index = group; 1240 action.flow_key_alg = alg_idx; 1241 1242 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); 1243 /* update the VF flow rule action with the VF default entry action */ 1244 if (mcam_index < 0) 1245 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1246 *(u64 *)&action); 1247 1248 /* update the action change in default rule */ 1249 pfvf = rvu_get_pfvf(rvu, pcifunc); 1250 if (pfvf->def_ucast_rule) 1251 pfvf->def_ucast_rule->rx_action = action; 1252 1253 if (mcam_index < 0) { 1254 /* update the VF flow rule action with the VF default 1255 * entry action 1256 */ 1257 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1258 *(u64 *)&action); 1259 1260 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1261 nixlf, NIXLF_PROMISC_ENTRY); 1262 1263 /* If PF's promiscuous entry is enabled, 1264 * Set RSS action for that entry as well 1265 */ 1266 if (index >= 0) 1267 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1268 blkaddr, alg_idx); 1269 1270 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1271 nixlf, NIXLF_ALLMULTI_ENTRY); 1272 /* If PF's allmulti entry is enabled, 1273 * Set RSS action for that entry as well 1274 */ 1275 if (index >= 0) 1276 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, 1277 blkaddr, alg_idx); 1278 } 1279 } 1280 1281 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1282 int nixlf, int type, bool enable) 1283 { 1284 struct npc_mcam *mcam = &rvu->hw->mcam; 1285 struct rvu_hwinfo *hw = rvu->hw; 1286 struct nix_mce_list *mce_list; 1287 int index, blkaddr, mce_idx; 1288 struct rvu_pfvf *pfvf; 1289 u16 ptr[4]; 1290 1291 /* multicast pkt replication is not enabled for AF's VFs & SDP links */ 1292 if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(rvu, pcifunc)) 1293 return; 1294 1295 /* In cn20k, only CGX mapped devices have default MCAST entry */ 1296 if (is_cn20k(rvu->pdev) && 1297 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 1298 &ptr[2], &ptr[3])) 1299 return; 1300 1301 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1302 if (blkaddr < 0) 1303 return; 1304 1305 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1306 nixlf, type); 1307 if (index < 0) { 1308 dev_err(rvu->dev, 1309 "%s: Error to get entry for pcifunc=%#x, type=%u\n", 1310 __func__, pcifunc, type); 1311 return; 1312 } 1313 1314 /* disable MCAM entry when packet replication is not supported by hw */ 1315 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1316 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1317 return; 1318 } 1319 1320 /* return incase mce list is not enabled */ 1321 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1322 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1323 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1324 return; 1325 1326 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1327 1328 nix_update_mce_list(rvu, pcifunc, mce_list, 1329 mce_idx, index, enable); 1330 if (enable) 1331 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1332 } 1333 1334 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1335 int nixlf, bool enable) 1336 { 1337 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1338 struct npc_mcam *mcam = &rvu->hw->mcam; 1339 int index, blkaddr; 1340 u16 ptr[4]; 1341 1342 /* only CGX or LBK interfaces have default entries */ 1343 if (is_cn20k(rvu->pdev) && 1344 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], 1345 &ptr[2], &ptr[3])) 1346 return; 1347 1348 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1349 if (blkaddr < 0) 1350 return; 1351 1352 /* Ucast MCAM match entry of this PF/VF */ 1353 if (npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), 1354 pfvf->nix_rx_intf)) { 1355 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1356 nixlf, NIXLF_UCAST_ENTRY); 1357 if (index < 0) { 1358 dev_err(rvu->dev, 1359 "%s: Error to get ucast entry for pcifunc=%#x\n", 1360 __func__, pcifunc); 1361 return; 1362 } 1363 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1364 } 1365 1366 /* Nothing to do for VFs, on platforms where pkt replication 1367 * is not supported 1368 */ 1369 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1370 return; 1371 1372 /* add/delete pf_func to broadcast MCE list */ 1373 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1374 NIXLF_BCAST_ENTRY, enable); 1375 } 1376 1377 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1378 { 1379 if (nixlf < 0) 1380 return; 1381 1382 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1383 1384 /* Delete multicast and promisc MCAM entries */ 1385 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1386 NIXLF_ALLMULTI_ENTRY, false); 1387 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1388 NIXLF_PROMISC_ENTRY, false); 1389 } 1390 1391 bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable) 1392 { 1393 int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1394 struct npc_mcam *mcam = &rvu->hw->mcam; 1395 struct rvu_npc_mcam_rule *rule, *tmp; 1396 1397 mutex_lock(&mcam->lock); 1398 1399 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1400 if (rule->intf != intf) 1401 continue; 1402 1403 if (rule->entry != entry) 1404 continue; 1405 1406 rule->enable = enable; 1407 mutex_unlock(&mcam->lock); 1408 1409 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1410 entry, enable); 1411 1412 return true; 1413 } 1414 1415 mutex_unlock(&mcam->lock); 1416 return false; 1417 } 1418 1419 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1420 { 1421 if (nixlf < 0) 1422 return; 1423 1424 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1425 * in set_rx_mode mbox handler. 1426 */ 1427 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1428 } 1429 1430 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1431 { 1432 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1433 struct npc_mcam *mcam = &rvu->hw->mcam; 1434 struct rvu_npc_mcam_rule *rule, *tmp; 1435 int blkaddr; 1436 1437 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1438 if (blkaddr < 0) 1439 return; 1440 1441 mutex_lock(&mcam->lock); 1442 1443 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1444 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1445 if (is_npc_intf_rx(rule->intf) && 1446 rule->rx_action.pf_func == pcifunc && 1447 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1448 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1449 rule->entry, false); 1450 rule->enable = false; 1451 /* Indicate that default rule is disabled */ 1452 if (rule->default_rule) { 1453 pfvf->def_ucast_rule = NULL; 1454 list_del(&rule->list); 1455 kfree(rule); 1456 } 1457 } 1458 } 1459 1460 mutex_unlock(&mcam->lock); 1461 1462 npc_mcam_disable_flows(rvu, pcifunc); 1463 1464 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1465 } 1466 1467 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1468 { 1469 struct npc_mcam *mcam = &rvu->hw->mcam; 1470 struct rvu_npc_mcam_rule *rule, *tmp; 1471 int blkaddr; 1472 1473 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1474 if (blkaddr < 0) 1475 return; 1476 1477 mutex_lock(&mcam->lock); 1478 1479 /* Free all MCAM entries owned by this 'pcifunc' */ 1480 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1481 1482 /* Free all MCAM counters owned by this 'pcifunc' */ 1483 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1484 1485 /* Delete MCAM entries owned by this 'pcifunc' */ 1486 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1487 if (rule->owner == pcifunc && !rule->default_rule) { 1488 list_del(&rule->list); 1489 kfree(rule); 1490 } 1491 } 1492 1493 mutex_unlock(&mcam->lock); 1494 1495 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1496 } 1497 1498 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1499 const struct npc_mcam_kex *mkex, 1500 u8 intf) 1501 { 1502 int lid, lt, ld, fl; 1503 1504 if (is_npc_intf_tx(intf)) 1505 return; 1506 1507 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1508 mkex->keyx_cfg[NIX_INTF_RX]); 1509 1510 /* Program LDATA */ 1511 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1512 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1513 for (ld = 0; ld < NPC_MAX_LD; ld++) 1514 SET_KEX_LD(intf, lid, lt, ld, 1515 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1516 [lid][lt][ld]); 1517 } 1518 } 1519 /* Program LFLAGS */ 1520 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1521 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1522 SET_KEX_LDFLAGS(intf, ld, fl, 1523 mkex->intf_ld_flags[NIX_INTF_RX] 1524 [ld][fl]); 1525 } 1526 } 1527 1528 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1529 const struct npc_mcam_kex *mkex, 1530 u8 intf) 1531 { 1532 int lid, lt, ld, fl; 1533 1534 if (is_npc_intf_rx(intf)) 1535 return; 1536 1537 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1538 mkex->keyx_cfg[NIX_INTF_TX]); 1539 1540 /* Program LDATA */ 1541 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1542 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1543 for (ld = 0; ld < NPC_MAX_LD; ld++) 1544 SET_KEX_LD(intf, lid, lt, ld, 1545 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1546 [lid][lt][ld]); 1547 } 1548 } 1549 /* Program LFLAGS */ 1550 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1551 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1552 SET_KEX_LDFLAGS(intf, ld, fl, 1553 mkex->intf_ld_flags[NIX_INTF_TX] 1554 [ld][fl]); 1555 } 1556 } 1557 1558 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1559 const struct npc_mcam_kex *mkex) 1560 { 1561 struct rvu_hwinfo *hw = rvu->hw; 1562 u8 intf; 1563 int ld; 1564 1565 for (ld = 0; ld < NPC_MAX_LD; ld++) 1566 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1567 mkex->kex_ld_flags[ld]); 1568 1569 for (intf = 0; intf < hw->npc_intfs; intf++) { 1570 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1571 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1572 } 1573 1574 /* Programme mkex hash profile */ 1575 npc_program_mkex_hash(rvu, blkaddr); 1576 } 1577 1578 int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1579 u64 *size) 1580 { 1581 u64 prfl_addr, prfl_sz; 1582 1583 if (!rvu->fwdata) 1584 return -EINVAL; 1585 1586 prfl_addr = rvu->fwdata->mcam_addr; 1587 prfl_sz = rvu->fwdata->mcam_sz; 1588 1589 if (!prfl_addr || !prfl_sz) 1590 return -EINVAL; 1591 1592 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1593 if (!(*prfl_img_addr)) 1594 return -ENOMEM; 1595 1596 *size = prfl_sz; 1597 1598 return 0; 1599 } 1600 1601 /* strtoull of "mkexprof" with base:36 */ 1602 #define MKEX_END_SIGN 0xdeadbeef 1603 1604 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1605 const char *mkex_profile) 1606 { 1607 struct device *dev = &rvu->pdev->dev; 1608 struct npc_mcam_kex *mcam_kex; 1609 void __iomem *mkex_prfl_addr = NULL; 1610 u64 prfl_sz; 1611 int ret; 1612 1613 /* If user not selected mkex profile */ 1614 if (rvu->kpu_fwdata_sz || 1615 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1616 goto program_mkex; 1617 1618 /* Setting up the mapping for mkex profile image */ 1619 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1620 if (ret < 0) 1621 goto program_mkex; 1622 1623 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1624 1625 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1626 /* Compare with mkex mod_param name string */ 1627 if (mcam_kex->mkex_sign == MKEX_SIGN && 1628 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1629 /* Due to an errata (35786) in A0/B0 pass silicon, 1630 * parse nibble enable configuration has to be 1631 * identical for both Rx and Tx interfaces. 1632 */ 1633 if (!is_rvu_96xx_B0(rvu) || 1634 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1635 rvu->kpu.mcam_kex_prfl.mkex = mcam_kex; 1636 goto program_mkex; 1637 } 1638 1639 mcam_kex++; 1640 prfl_sz -= sizeof(struct npc_mcam_kex); 1641 } 1642 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1643 1644 program_mkex: 1645 dev_info(rvu->dev, "Using %s mkex profile\n", 1646 rvu->kpu.mcam_kex_prfl.mkex->name); 1647 /* Program selected mkex profile */ 1648 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex); 1649 if (mkex_prfl_addr) 1650 iounmap(mkex_prfl_addr); 1651 } 1652 1653 void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1654 const struct npc_kpu_profile_action *kpuaction, 1655 int kpu, int entry, bool pkind) 1656 { 1657 struct npc_kpu_action0 action0 = {0}; 1658 struct npc_kpu_action1 action1 = {0}; 1659 u64 reg; 1660 1661 action1.errlev = kpuaction->errlev; 1662 action1.errcode = kpuaction->errcode; 1663 action1.dp0_offset = kpuaction->dp0_offset; 1664 action1.dp1_offset = kpuaction->dp1_offset; 1665 action1.dp2_offset = kpuaction->dp2_offset; 1666 1667 if (pkind) 1668 reg = NPC_AF_PKINDX_ACTION1(entry); 1669 else 1670 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1671 1672 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1673 1674 action0.byp_count = kpuaction->bypass_count; 1675 action0.capture_ena = kpuaction->cap_ena; 1676 action0.parse_done = kpuaction->parse_done; 1677 action0.next_state = kpuaction->next_state; 1678 action0.capture_lid = kpuaction->lid; 1679 action0.capture_ltype = kpuaction->ltype; 1680 action0.capture_flags = kpuaction->flags; 1681 action0.ptr_advance = kpuaction->ptr_advance; 1682 action0.var_len_offset = kpuaction->offset; 1683 action0.var_len_mask = kpuaction->mask; 1684 action0.var_len_right = kpuaction->right; 1685 action0.var_len_shift = kpuaction->shift; 1686 1687 if (pkind) 1688 reg = NPC_AF_PKINDX_ACTION0(entry); 1689 else 1690 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1691 1692 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1693 } 1694 1695 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1696 const struct npc_kpu_profile_cam *kpucam, 1697 int kpu, int entry) 1698 { 1699 const struct npc_kpu_profile_cam2 *kpucam2 = (void *)kpucam; 1700 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1701 struct npc_kpu_cam cam0 = {0}; 1702 struct npc_kpu_cam cam1 = {0}; 1703 u64 *val = (u64 *)&cam1; 1704 u64 *mask = (u64 *)&cam0; 1705 1706 cam1.state = kpucam->state & kpucam->state_mask; 1707 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1708 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1709 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1710 1711 cam0.state = ~kpucam->state & kpucam->state_mask; 1712 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1713 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1714 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1715 1716 if (profile->from_fs) { 1717 u8 ptype = kpucam2->ptype; 1718 u8 pmask = kpucam2->ptype_mask; 1719 1720 *val |= FIELD_PREP(GENMASK_ULL(57, 56), ptype & pmask); 1721 *mask |= FIELD_PREP(GENMASK_ULL(57, 56), ~ptype & pmask); 1722 } 1723 1724 rvu_write64(rvu, blkaddr, 1725 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1726 rvu_write64(rvu, blkaddr, 1727 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1728 } 1729 1730 u64 npc_enable_mask(int count) 1731 { 1732 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1733 } 1734 1735 struct npc_kpu_profile_action * 1736 npc_get_ikpu_nth_entry(struct rvu *rvu, int n) 1737 { 1738 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1739 1740 if (profile->from_fs) 1741 return &profile->ikpu2[n]; 1742 1743 return &profile->ikpu[n]; 1744 } 1745 1746 int 1747 npc_get_num_kpu_cam_entries(struct rvu *rvu, 1748 const struct npc_kpu_profile *kpu_pfl) 1749 { 1750 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1751 1752 if (profile->from_fs) 1753 return kpu_pfl->cam_entries2; 1754 1755 return kpu_pfl->cam_entries; 1756 } 1757 1758 struct npc_kpu_profile_cam * 1759 npc_get_kpu_cam_nth_entry(struct rvu *rvu, 1760 const struct npc_kpu_profile *kpu_pfl, int n) 1761 { 1762 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1763 1764 if (profile->from_fs) 1765 return (void *)&kpu_pfl->cam2[n]; 1766 1767 return (void *)&kpu_pfl->cam[n]; 1768 } 1769 1770 int 1771 npc_get_num_kpu_action_entries(struct rvu *rvu, 1772 const struct npc_kpu_profile *kpu_pfl) 1773 { 1774 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1775 1776 if (profile->from_fs) 1777 return kpu_pfl->action_entries2; 1778 1779 return kpu_pfl->action_entries; 1780 } 1781 1782 struct npc_kpu_profile_action * 1783 npc_get_kpu_action_nth_entry(struct rvu *rvu, 1784 const struct npc_kpu_profile *kpu_pfl, 1785 int n) 1786 { 1787 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1788 1789 if (profile->from_fs) 1790 return (void *)&kpu_pfl->action2[n]; 1791 1792 return (void *)&kpu_pfl->action[n]; 1793 } 1794 1795 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1796 const struct npc_kpu_profile *profile) 1797 { 1798 int num_cam_entries, num_action_entries; 1799 int entry, num_entries, max_entries; 1800 u64 entry_mask; 1801 1802 num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile); 1803 num_action_entries = npc_get_num_kpu_action_entries(rvu, profile); 1804 1805 if (num_cam_entries != num_action_entries) { 1806 dev_err(rvu->dev, 1807 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1808 kpu, num_cam_entries, num_action_entries); 1809 } 1810 1811 max_entries = rvu->hw->npc_kpu_entries; 1812 1813 WARN(num_cam_entries > max_entries, 1814 "KPU%u: err: hw max entries=%u, input entries=%u\n", 1815 kpu, rvu->hw->npc_kpu_entries, num_cam_entries); 1816 1817 /* Program CAM match entries for previous KPU extracted data */ 1818 num_entries = min_t(int, num_cam_entries, max_entries); 1819 for (entry = 0; entry < num_entries; entry++) 1820 npc_config_kpucam(rvu, blkaddr, 1821 (void *)npc_get_kpu_cam_nth_entry(rvu, profile, entry), 1822 kpu, entry); 1823 1824 /* Program this KPU's actions */ 1825 num_entries = min_t(int, num_action_entries, max_entries); 1826 for (entry = 0; entry < num_entries; entry++) 1827 npc_config_kpuaction(rvu, blkaddr, 1828 (void *)npc_get_kpu_action_nth_entry(rvu, profile, entry), 1829 kpu, entry, false); 1830 1831 /* Enable all programmed entries */ 1832 num_entries = min_t(int, num_action_entries, num_cam_entries); 1833 entry_mask = npc_enable_mask(num_entries); 1834 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1835 if (!rvu->kpu.custom) 1836 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1837 rvu_write64(rvu, blkaddr, 1838 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1839 if (num_entries > 64) { 1840 rvu_write64(rvu, blkaddr, 1841 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1842 npc_enable_mask(num_entries - 64)); 1843 } 1844 1845 /* Enable this KPU */ 1846 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1847 } 1848 1849 static void npc_prepare_default_kpu(struct rvu *rvu, 1850 struct npc_kpu_profile_adapter *profile) 1851 { 1852 profile->custom = 0; 1853 profile->name = def_pfl_name; 1854 profile->version = NPC_KPU_PROFILE_VER; 1855 profile->ikpu = ikpu_action_entries; 1856 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1857 profile->kpu = npc_kpu_profiles; 1858 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1859 profile->lt_def = &npc_lt_defaults; 1860 profile->mkex_hash = &npc_mkex_hash_default; 1861 1862 if (!is_cn20k(rvu->pdev)) { 1863 profile->mcam_kex_prfl.mkex = &npc_mkex_default; 1864 return; 1865 } 1866 1867 profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); 1868 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6; 1869 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0; 1870 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5; 1871 ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1; 1872 1873 npc_cn20k_update_action_entries_n_flags(rvu, profile); 1874 } 1875 1876 static int npc_alloc_kpu_cam2_n_action2(struct rvu *rvu, int kpu_num, 1877 int num_entries) 1878 { 1879 struct npc_kpu_profile_adapter *adapter = &rvu->kpu; 1880 struct npc_kpu_profile *kpu; 1881 1882 kpu = &adapter->kpu[kpu_num]; 1883 1884 kpu->cam2 = devm_kcalloc(rvu->dev, num_entries, 1885 sizeof(*kpu->cam2), GFP_KERNEL); 1886 if (!kpu->cam2) 1887 return -ENOMEM; 1888 1889 kpu->action2 = devm_kcalloc(rvu->dev, num_entries, 1890 sizeof(*kpu->action2), GFP_KERNEL); 1891 if (!kpu->action2) 1892 return -ENOMEM; 1893 1894 return 0; 1895 } 1896 1897 static int npc_apply_custom_kpu_from_fw(struct rvu *rvu, 1898 struct npc_kpu_profile_adapter *profile) 1899 { 1900 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1901 const struct npc_kpu_profile_fwdata *fw; 1902 struct npc_kpu_profile_action *action; 1903 struct npc_kpu_profile_cam *cam; 1904 struct npc_kpu_fwdata *fw_kpu; 1905 int entries, entry, kpu; 1906 1907 fw = rvu->kpu_fwdata; 1908 1909 for (kpu = 0; kpu < fw->kpus; kpu++) { 1910 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1911 dev_warn(rvu->dev, 1912 "Profile size mismatch on KPU%i parsing\n", 1913 kpu + 1); 1914 return -EINVAL; 1915 } 1916 1917 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1918 if (fw_kpu->entries < 0) { 1919 dev_warn(rvu->dev, 1920 "Profile entries is negative on KPU%i parsing\n", 1921 kpu + 1); 1922 return -EINVAL; 1923 } 1924 1925 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1926 dev_warn(rvu->dev, 1927 "Too many custom entries on KPU%d: %d > %d\n", 1928 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1929 entries = min_t(int, fw_kpu->entries, KPU_MAX_CST_ENT); 1930 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1931 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1932 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1933 offset += fw_kpu->entries * sizeof(*action); 1934 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1935 dev_warn(rvu->dev, 1936 "Profile size mismatch on KPU%i parsing.\n", 1937 kpu + 1); 1938 return -EINVAL; 1939 } 1940 for (entry = 0; entry < entries; entry++) { 1941 profile->kpu[kpu].cam[entry] = cam[entry]; 1942 profile->kpu[kpu].action[entry] = action[entry]; 1943 } 1944 } 1945 1946 return 0; 1947 } 1948 1949 static int npc_apply_custom_kpu_from_fs(struct rvu *rvu, 1950 struct npc_kpu_profile_adapter *profile) 1951 { 1952 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1953 const struct npc_kpu_profile_fwdata *fw; 1954 struct npc_kpu_profile_action *action; 1955 struct npc_kpu_profile_cam2 *cam2; 1956 struct npc_kpu_fwdata *fw_kpu; 1957 int entries, ret, entry, kpu; 1958 1959 fw = rvu->kpu_fwdata; 1960 1961 /* Binary blob contains ikpu actions entries at start of data[0] */ 1962 profile->ikpu2 = devm_kcalloc(rvu->dev, 1, 1963 sizeof(ikpu_action_entries), 1964 GFP_KERNEL); 1965 if (!profile->ikpu2) 1966 return -ENOMEM; 1967 1968 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1969 1970 if (rvu->kpu_fwdata_sz < hdr_sz + sizeof(ikpu_action_entries)) 1971 return -EINVAL; 1972 1973 /* The firmware layout does dependent on the internal size of 1974 * ikpu_action_entries. 1975 */ 1976 memcpy((void *)profile->ikpu2, action, sizeof(ikpu_action_entries)); 1977 offset += sizeof(ikpu_action_entries); 1978 1979 for (kpu = 0; kpu < fw->kpus; kpu++) { 1980 if (rvu->kpu_fwdata_sz < hdr_sz + offset + sizeof(*fw_kpu)) { 1981 dev_warn(rvu->dev, 1982 "profile size mismatch on kpu%i parsing\n", 1983 kpu + 1); 1984 return -EINVAL; 1985 } 1986 1987 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1988 if (fw_kpu->entries <= 0) { 1989 dev_warn(rvu->dev, 1990 "Invalid kpu entries on KPU%d\n", kpu); 1991 return -EINVAL; 1992 } 1993 1994 entries = min_t(int, fw_kpu->entries, rvu->hw->npc_kpu_entries); 1995 dev_info(rvu->dev, 1996 "Loading %u entries on KPU%d\n", entries, kpu); 1997 1998 cam2 = (struct npc_kpu_profile_cam2 *)fw_kpu->data; 1999 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam2); 2000 action = (struct npc_kpu_profile_action *)(fw->data + offset); 2001 offset += fw_kpu->entries * sizeof(*action); 2002 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 2003 dev_warn(rvu->dev, 2004 "profile size mismatch on kpu%i parsing.\n", 2005 kpu + 1); 2006 return -EINVAL; 2007 } 2008 2009 profile->kpu[kpu].cam_entries2 = entries; 2010 profile->kpu[kpu].action_entries2 = entries; 2011 ret = npc_alloc_kpu_cam2_n_action2(rvu, kpu, entries); 2012 if (ret) { 2013 dev_warn(rvu->dev, 2014 "profile entry allocation failed for kpu=%d for %d entries\n", 2015 kpu, entries); 2016 return -EINVAL; 2017 } 2018 2019 for (entry = 0; entry < entries; entry++) { 2020 profile->kpu[kpu].cam2[entry] = cam2[entry]; 2021 profile->kpu[kpu].action2[entry] = action[entry]; 2022 } 2023 } 2024 2025 return 0; 2026 } 2027 2028 static int npc_apply_custom_kpu(struct rvu *rvu, 2029 struct npc_kpu_profile_adapter *profile, 2030 bool from_fs, int *fw_kpus) 2031 { 2032 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata); 2033 const struct npc_kpu_profile_fwdata *fw; 2034 struct npc_kpu_profile_fwdata *sfw; 2035 2036 if (is_cn20k(rvu->pdev)) 2037 return npc_cn20k_apply_custom_kpu(rvu, profile); 2038 2039 if (rvu->kpu_fwdata_sz < hdr_sz) { 2040 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 2041 return -EINVAL; 2042 } 2043 2044 fw = rvu->kpu_fwdata; 2045 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 2046 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 2047 fw->signature); 2048 return -EINVAL; 2049 } 2050 /* Verify if the using known profile structure */ 2051 if (NPC_KPU_VER_MAJ(profile->version) > 2052 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 2053 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 2054 NPC_KPU_VER_MAJ(profile->version), 2055 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 2056 return -EINVAL; 2057 } 2058 /* Verify if profile is aligned with the required kernel changes */ 2059 if (NPC_KPU_VER_MIN(profile->version) < 2060 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 2061 dev_warn(rvu->dev, 2062 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 2063 NPC_KPU_VER_MAJ(profile->version), 2064 NPC_KPU_VER_MIN(profile->version), 2065 NPC_KPU_VER_PATCH(profile->version), 2066 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 2067 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 2068 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 2069 return -EINVAL; 2070 } 2071 /* Verify if profile fits the HW */ 2072 if (fw->kpus > rvu->hw->npc_kpus) { 2073 dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus, 2074 rvu->hw->npc_kpus); 2075 return -EINVAL; 2076 } 2077 2078 /* Check if there is enough memory for fw loading. 2079 * Check if there is enough entries for profile->kpu[] to 2080 * set cam_entries2 and action_entries2 2081 */ 2082 if (fw->kpus > profile->kpus) { 2083 dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus, 2084 profile->kpus); 2085 return -EINVAL; 2086 } 2087 2088 *fw_kpus = fw->kpus; 2089 2090 sfw = devm_kcalloc(rvu->dev, 1, sizeof(*sfw), GFP_KERNEL); 2091 if (!sfw) 2092 return -ENOMEM; 2093 2094 memcpy(sfw, fw, sizeof(*sfw)); 2095 2096 profile->custom = 1; 2097 profile->name = sfw->name; 2098 profile->version = le64_to_cpu(fw->version); 2099 profile->mcam_kex_prfl.mkex = &sfw->mkex; 2100 profile->lt_def = &sfw->lt_def; 2101 2102 return from_fs ? npc_apply_custom_kpu_from_fs(rvu, profile) : 2103 npc_apply_custom_kpu_from_fw(rvu, profile); 2104 } 2105 2106 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 2107 u64 prfl_sz, const char *kpu_profile) 2108 { 2109 struct npc_kpu_profile_fwdata *kpu_data = NULL; 2110 int rc = -EINVAL; 2111 2112 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 2113 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 2114 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 2115 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 2116 kpu_profile); 2117 rvu->kpu_fwdata = kpu_data; 2118 rvu->kpu_fwdata_sz = prfl_sz; 2119 rvu->kpu_prfl_addr = prfl_addr; 2120 rc = 0; 2121 } 2122 2123 return rc; 2124 } 2125 2126 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 2127 const char *kpu_profile) 2128 { 2129 struct npc_coalesced_kpu_prfl *img_data = NULL; 2130 int i = 0, rc = -EINVAL; 2131 void __iomem *kpu_prfl_addr; 2132 u32 offset; 2133 2134 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 2135 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 2136 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 2137 /* Loaded profile is a single KPU profile. */ 2138 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 2139 prfl_sz, kpu_profile); 2140 goto done; 2141 } 2142 2143 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 2144 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 2145 (img_data->num_prfl * sizeof(uint16_t)); 2146 /* Check if mapped image is coalesced image. */ 2147 while (i < img_data->num_prfl) { 2148 /* Profile image offsets are rounded up to next 8 multiple.*/ 2149 offset = ALIGN_8B_CEIL(offset); 2150 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 2151 offset); 2152 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 2153 img_data->prfl_sz[i], kpu_profile); 2154 if (!rc) 2155 break; 2156 /* Calculating offset of profile image based on profile size.*/ 2157 offset += img_data->prfl_sz[i]; 2158 i++; 2159 } 2160 done: 2161 return rc; 2162 } 2163 2164 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 2165 { 2166 int ret = -EINVAL; 2167 u64 prfl_sz; 2168 2169 /* Setting up the mapping for NPC profile image */ 2170 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 2171 if (ret < 0) 2172 goto done; 2173 2174 /* Detect if profile is coalesced or single KPU profile and load */ 2175 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 2176 if (ret == 0) 2177 goto done; 2178 2179 /* Cleaning up if KPU profile image from fwdata is not valid. */ 2180 if (rvu->kpu_prfl_addr) { 2181 iounmap(rvu->kpu_prfl_addr); 2182 rvu->kpu_prfl_addr = NULL; 2183 rvu->kpu_fwdata_sz = 0; 2184 rvu->kpu_fwdata = NULL; 2185 } 2186 2187 done: 2188 return ret; 2189 } 2190 2191 static int npc_load_kpu_profile_from_fw(struct rvu *rvu) 2192 { 2193 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2194 const char *kpu_profile = rvu->kpu_pfl_name; 2195 int fw_kpus = 0; 2196 2197 /* Loading the KPU profile using firmware database */ 2198 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 2199 return -EFAULT; 2200 2201 /* Apply profile customization if firmware was loaded. */ 2202 if (!rvu->kpu_fwdata_sz || 2203 npc_apply_custom_kpu(rvu, profile, false, &fw_kpus)) { 2204 /* If image from firmware filesystem fails to load or invalid 2205 * retry with firmware database method. 2206 */ 2207 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 2208 /* Loading image from firmware database failed. */ 2209 if (rvu->kpu_prfl_addr) { 2210 iounmap(rvu->kpu_prfl_addr); 2211 rvu->kpu_prfl_addr = NULL; 2212 } else { 2213 kfree(rvu->kpu_fwdata); 2214 } 2215 rvu->kpu_fwdata = NULL; 2216 rvu->kpu_fwdata_sz = 0; 2217 } 2218 2219 dev_warn(rvu->dev, 2220 "Can't load KPU profile %s. Using default.\n", 2221 kpu_profile); 2222 kfree(rvu->kpu_fwdata); 2223 rvu->kpu_fwdata = NULL; 2224 return -EFAULT; 2225 } 2226 2227 dev_info(rvu->dev, "Using custom profile '%.32s', version %d.%d.%d\n", 2228 profile->name, NPC_KPU_VER_MAJ(profile->version), 2229 NPC_KPU_VER_MIN(profile->version), 2230 NPC_KPU_VER_PATCH(profile->version)); 2231 2232 return 0; 2233 } 2234 2235 static int npc_load_kpu_profile_from_fs(struct rvu *rvu) 2236 { 2237 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2238 const char *kpu_profile = rvu->kpu_pfl_name; 2239 const struct firmware *fw = NULL; 2240 int ret, fw_kpus = 0; 2241 char path[512] = "kpu/"; 2242 2243 if (strlen(kpu_profile) > sizeof(path) - strlen("kpu/") - 1) { 2244 dev_err(rvu->dev, "kpu profile name is too big\n"); 2245 return -ENOSPC; 2246 } 2247 2248 strcat(path, kpu_profile); 2249 2250 if (request_firmware_direct(&fw, path, rvu->dev)) 2251 return -ENOENT; 2252 2253 dev_info(rvu->dev, "Loading KPU profile from filesystem: %s\n", 2254 path); 2255 2256 rvu->kpu_fwdata = fw->data; 2257 rvu->kpu_fwdata_sz = fw->size; 2258 2259 ret = npc_apply_custom_kpu(rvu, profile, true, &fw_kpus); 2260 release_firmware(fw); 2261 rvu->kpu_fwdata = NULL; 2262 2263 if (ret) { 2264 rvu->kpu_fwdata_sz = 0; 2265 dev_err(rvu->dev, 2266 "Loading KPU profile from filesystem failed\n"); 2267 return ret; 2268 } 2269 2270 /* In firmware loading from filesystem method, all entries are from 2271 * same binary blob. 2272 */ 2273 rvu->kpu.kpus = fw_kpus; 2274 profile->kpus = fw_kpus; 2275 profile->from_fs = true; 2276 return 0; 2277 } 2278 2279 void npc_load_kpu_profile(struct rvu *rvu) 2280 { 2281 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2282 const char *kpu_profile = rvu->kpu_pfl_name; 2283 2284 profile->from_fs = false; 2285 2286 npc_prepare_default_kpu(rvu, profile); 2287 2288 /* If user not specified profile customization */ 2289 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 2290 return; 2291 2292 /* Order of preceedence for load loading NPC profile (high to low) 2293 * Firmware binary in filesystem. 2294 * Firmware database method. 2295 * Default KPU profile. 2296 */ 2297 2298 /* Filesystem-based KPU loading is not supported on cn20k. 2299 * npc_prepare_default_kpu() was invoked earlier, but control 2300 * reached this point because the default profile was not selected. 2301 * No need to call it again. 2302 */ 2303 if (!is_cn20k(rvu->pdev)) { 2304 if (!npc_load_kpu_profile_from_fs(rvu)) 2305 return; 2306 } 2307 2308 /* First prepare default KPU, then we'll customize top entries. */ 2309 npc_prepare_default_kpu(rvu, profile); 2310 if (!npc_load_kpu_profile_from_fw(rvu)) 2311 return; 2312 2313 npc_prepare_default_kpu(rvu, profile); 2314 } 2315 2316 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 2317 { 2318 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 2319 struct rvu_hwinfo *hw = rvu->hw; 2320 int num_pkinds, num_kpus, idx; 2321 2322 /* Disable all KPUs and their entries */ 2323 for (idx = 0; idx < hw->npc_kpus; idx++) { 2324 rvu_write64(rvu, blkaddr, 2325 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 2326 rvu_write64(rvu, blkaddr, 2327 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 2328 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 2329 } 2330 2331 /* Load and customize KPU profile. */ 2332 npc_load_kpu_profile(rvu); 2333 2334 /* First program IKPU profile i.e PKIND configs. 2335 * Check HW max count to avoid configuring junk or 2336 * writing to unsupported CSR addresses. 2337 */ 2338 num_pkinds = rvu->kpu.pkinds; 2339 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 2340 2341 for (idx = 0; idx < num_pkinds; idx++) 2342 npc_config_kpuaction(rvu, blkaddr, 2343 npc_get_ikpu_nth_entry(rvu, idx), 2344 0, idx, true); 2345 2346 /* Program KPU CAM and Action profiles */ 2347 num_kpus = rvu->kpu.kpus; 2348 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 2349 2350 for (idx = 0; idx < num_kpus; idx++) 2351 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 2352 2353 if (profile->from_fs) { 2354 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_TYPE(54), 0x03); 2355 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_TYPE(58), 0x03); 2356 } 2357 } 2358 2359 void npc_mcam_rsrcs_deinit(struct rvu *rvu) 2360 { 2361 struct npc_mcam *mcam = &rvu->hw->mcam; 2362 2363 bitmap_free(mcam->bmap); 2364 bitmap_free(mcam->bmap_reverse); 2365 kfree(mcam->entry2pfvf_map); 2366 kfree(mcam->cntr2pfvf_map); 2367 kfree(mcam->entry2cntr_map); 2368 kfree(mcam->cntr_refcnt); 2369 kfree(mcam->entry2target_pffunc); 2370 kfree(mcam->counters.bmap); 2371 } 2372 2373 int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 2374 { 2375 int nixlf_count = rvu_get_nixlf_count(rvu); 2376 struct npc_mcam *mcam = &rvu->hw->mcam; 2377 int rsvd, err; 2378 u16 index; 2379 int cntr; 2380 u64 cfg; 2381 2382 cfg = (rvu_read64(rvu, blkaddr, 2383 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 2384 mcam->keysize = cfg; 2385 2386 /* Number of banks combined per MCAM entry */ 2387 if (is_cn20k(rvu->pdev)) { 2388 /* In cn20k, x2 entries is allowed for x4 profile. 2389 * set total_entries as 8192 * 2 and key size as x2. 2390 */ 2391 mcam->total_entries = mcam->banks * mcam->banksize; 2392 if (cfg == NPC_MCAM_KEY_X2) 2393 mcam->banks_per_entry = 1; 2394 else 2395 mcam->banks_per_entry = 2; 2396 2397 rsvd = 0; 2398 } else { 2399 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * 2400 mcam->banksize; 2401 2402 if (cfg == NPC_MCAM_KEY_X4) 2403 mcam->banks_per_entry = 4; 2404 else if (cfg == NPC_MCAM_KEY_X2) 2405 mcam->banks_per_entry = 2; 2406 else 2407 mcam->banks_per_entry = 1; 2408 2409 /* Reserve one MCAM entry for each of the NIX LF to 2410 * guarantee space to install default matching DMAC rule. 2411 * Also reserve 2 MCAM entries for each PF for default 2412 * channel based matching or 'bcast & promisc' matching to 2413 * support BCAST and PROMISC modes of operation for PFs. 2414 * PF0 is excluded. 2415 */ 2416 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 2417 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 2418 if (mcam->total_entries <= rsvd) { 2419 dev_warn(rvu->dev, 2420 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 2421 mcam->total_entries); 2422 return -ENOMEM; 2423 } 2424 } 2425 2426 mcam->bmap_entries = mcam->total_entries - rsvd; 2427 /* cn20k does not need offsets to alloc mcam entries */ 2428 if (!is_cn20k(rvu->pdev)) { 2429 mcam->nixlf_offset = mcam->bmap_entries; 2430 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 2431 } 2432 2433 /* Allocate bitmaps for managing MCAM entries */ 2434 mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2435 if (!mcam->bmap) 2436 return -ENOMEM; 2437 2438 mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 2439 if (!mcam->bmap_reverse) 2440 goto free_bmap; 2441 2442 mcam->bmap_fcnt = mcam->bmap_entries; 2443 2444 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 2445 mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16), 2446 GFP_KERNEL); 2447 2448 if (!mcam->entry2pfvf_map) 2449 goto free_bmap_reverse; 2450 2451 /* Reserve 1/8th of MCAM entries at the bottom for low priority 2452 * allocations and another 1/8th at the top for high priority 2453 * allocations. 2454 */ 2455 if (!is_cn20k(rvu->pdev)) { 2456 mcam->lprio_count = mcam->bmap_entries / 8; 2457 if (mcam->lprio_count > BITS_PER_LONG) 2458 mcam->lprio_count = round_down(mcam->lprio_count, 2459 BITS_PER_LONG); 2460 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 2461 mcam->hprio_count = mcam->lprio_count; 2462 mcam->hprio_end = mcam->hprio_count; 2463 } 2464 2465 /* Allocate bitmap for managing MCAM counters and memory 2466 * for saving counter to RVU PFFUNC allocation mapping. 2467 */ 2468 err = rvu_alloc_bitmap(&mcam->counters); 2469 if (err) 2470 goto free_entry_map; 2471 2472 mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16), 2473 GFP_KERNEL); 2474 if (!mcam->cntr2pfvf_map) 2475 goto free_cntr_bmap; 2476 2477 /* Alloc memory for MCAM entry to counter mapping and for tracking 2478 * counter's reference count. 2479 */ 2480 mcam->entry2cntr_map = kcalloc(mcam->total_entries, sizeof(u16), 2481 GFP_KERNEL); 2482 if (!mcam->entry2cntr_map) 2483 goto free_cntr_map; 2484 2485 mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16), 2486 GFP_KERNEL); 2487 if (!mcam->cntr_refcnt) 2488 goto free_entry_cntr_map; 2489 2490 /* Alloc memory for saving target device of mcam rule */ 2491 mcam->entry2target_pffunc = kcalloc(mcam->total_entries, 2492 sizeof(u16), GFP_KERNEL); 2493 if (!mcam->entry2target_pffunc) 2494 goto free_cntr_refcnt; 2495 2496 for (index = 0; index < mcam->bmap_entries; index++) 2497 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2498 2499 for (index = 0; index < mcam->total_entries; index++) 2500 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2501 2502 for (cntr = 0; cntr < mcam->counters.max; cntr++) 2503 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2504 2505 mutex_init(&mcam->lock); 2506 2507 return 0; 2508 2509 free_cntr_refcnt: 2510 kfree(mcam->cntr_refcnt); 2511 free_entry_cntr_map: 2512 kfree(mcam->entry2cntr_map); 2513 free_cntr_map: 2514 kfree(mcam->cntr2pfvf_map); 2515 free_cntr_bmap: 2516 kfree(mcam->counters.bmap); 2517 free_entry_map: 2518 kfree(mcam->entry2pfvf_map); 2519 free_bmap_reverse: 2520 bitmap_free(mcam->bmap_reverse); 2521 free_bmap: 2522 bitmap_free(mcam->bmap); 2523 2524 return -ENOMEM; 2525 } 2526 2527 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 2528 { 2529 struct npc_pkind *pkind = &rvu->hw->pkind; 2530 struct npc_mcam *mcam = &rvu->hw->mcam; 2531 struct rvu_hwinfo *hw = rvu->hw; 2532 u64 npc_const, npc_const1; 2533 u64 npc_const2 = 0; 2534 2535 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 2536 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 2537 if (npc_const1 & BIT_ULL(63)) 2538 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 2539 2540 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 2541 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 2542 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 2543 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 2544 /* For Cn20k silicon, check if enhanced parser 2545 * is present, then set the NUM_KPMS = NUM_KPUS / 2 and 2546 * number of LDATA extractors per KEX. 2547 */ 2548 if (is_cn20k(rvu->pdev)) { 2549 if (!(npc_const1 & BIT_ULL(62))) { 2550 WARN(1, "Enhanced parser is not supported\n"); 2551 return; 2552 } 2553 hw->npc_kpms = hw->npc_kpus / 2; 2554 hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL; 2555 } 2556 2557 hw->npc_intfs = npc_const & 0xFULL; 2558 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 2559 2560 mcam->banks = (npc_const >> 44) & 0xFULL; 2561 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 2562 hw->npc_stat_ena = BIT_ULL(9); 2563 /* Extended set */ 2564 if (npc_const2) { 2565 hw->npc_ext_set = true; 2566 /* 96xx supports only match_stats and npc_counters 2567 * reflected in NPC_AF_CONST reg. 2568 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 2569 * 98xx has both match_stat and ext and npc_counter 2570 * reflected in NPC_AF_CONST2 2571 * STAT_SEL_EXT added at [12:14] bit position. 2572 * cn10k supports only ext and hence npc_counters in 2573 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 2574 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 2575 */ 2576 if (!hw->npc_counters) 2577 hw->npc_stat_ena = BIT_ULL(63); 2578 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 2579 mcam->banksize = npc_const2 & 0xFFFFULL; 2580 } 2581 2582 mcam->counters.max = hw->npc_counters; 2583 } 2584 2585 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 2586 { 2587 const struct npc_mcam_kex_extr *mkex_extr; 2588 struct npc_mcam *mcam = &rvu->hw->mcam; 2589 struct rvu_hwinfo *hw = rvu->hw; 2590 const struct npc_mcam_kex *mkex; 2591 u64 nibble_ena, rx_kex, tx_kex; 2592 u64 *keyx_cfg, reg; 2593 u8 intf; 2594 2595 mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr; 2596 mkex = rvu->kpu.mcam_kex_prfl.mkex; 2597 2598 if (is_cn20k(rvu->pdev)) { 2599 keyx_cfg = (u64 *)mkex_extr->keyx_cfg; 2600 } else { 2601 keyx_cfg = (u64 *)mkex->keyx_cfg; 2602 /* Reserve last counter for MCAM RX miss action which is set to 2603 * drop packet. This way we will know how many pkts didn't 2604 * match any MCAM entry. 2605 */ 2606 mcam->counters.max--; 2607 mcam->rx_miss_act_cntr = mcam->counters.max; 2608 } 2609 2610 rx_kex = keyx_cfg[NIX_INTF_RX]; 2611 tx_kex = keyx_cfg[NIX_INTF_TX]; 2612 2613 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 2614 2615 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 2616 if (nibble_ena) { 2617 tx_kex &= ~NPC_PARSE_NIBBLE; 2618 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 2619 keyx_cfg[NIX_INTF_TX] = tx_kex; 2620 } 2621 2622 /* Configure RX interfaces */ 2623 for (intf = 0; intf < hw->npc_intfs; intf++) { 2624 if (is_npc_intf_tx(intf)) 2625 continue; 2626 2627 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 2628 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2629 rx_kex); 2630 2631 if (is_cn20k(rvu->pdev)) 2632 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2633 else 2634 reg = NPC_AF_INTFX_MISS_ACT(intf); 2635 2636 /* If MCAM lookup doesn't result in a match, drop the received 2637 * packet. And map this action to a counter to count dropped 2638 * packets. 2639 */ 2640 rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP); 2641 2642 if (is_cn20k(rvu->pdev)) 2643 continue; 2644 2645 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 2646 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 2647 */ 2648 rvu_write64(rvu, blkaddr, 2649 NPC_AF_INTFX_MISS_STAT_ACT(intf), 2650 ((mcam->rx_miss_act_cntr >> 9) << 12) | 2651 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 2652 } 2653 2654 /* Configure TX interfaces */ 2655 for (intf = 0; intf < hw->npc_intfs; intf++) { 2656 if (is_npc_intf_rx(intf)) 2657 continue; 2658 2659 /* Extract Ltypes LID_LA to LID_LE */ 2660 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2661 tx_kex); 2662 2663 if (is_cn20k(rvu->pdev)) 2664 reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); 2665 else 2666 reg = NPC_AF_INTFX_MISS_ACT(intf); 2667 2668 /* Set TX miss action to UCAST_DEFAULT i.e 2669 * transmit the packet on NIX LF SQ's default channel. 2670 */ 2671 rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT); 2672 } 2673 } 2674 2675 int rvu_npc_init(struct rvu *rvu) 2676 { 2677 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 2678 struct npc_pkind *pkind = &rvu->hw->pkind; 2679 struct npc_mcam *mcam = &rvu->hw->mcam; 2680 int blkaddr, entry, bank, err; 2681 2682 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2683 if (blkaddr < 0) { 2684 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 2685 return -ENODEV; 2686 } 2687 2688 rvu_npc_hw_init(rvu, blkaddr); 2689 2690 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2691 for (bank = 0; bank < mcam->banks; bank++) { 2692 for (entry = 0; entry < mcam->banksize; entry++) 2693 rvu_write64(rvu, blkaddr, 2694 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2695 } 2696 2697 err = rvu_alloc_bitmap(&pkind->rsrc); 2698 if (err) 2699 return err; 2700 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2701 * no need to configure PKIND for all LBKs separately. 2702 */ 2703 rvu_alloc_rsrc(&pkind->rsrc); 2704 2705 /* Allocate mem for pkind to PF and channel mapping info */ 2706 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2707 sizeof(u32), GFP_KERNEL); 2708 if (!pkind->pfchan_map) 2709 return -ENOMEM; 2710 2711 /* Configure KPU profile */ 2712 if (is_cn20k(rvu->pdev)) 2713 npc_cn20k_parser_profile_init(rvu, blkaddr); 2714 else 2715 npc_parser_profile_init(rvu, blkaddr); 2716 2717 /* Config Outer L2, IPv4's NPC layer info */ 2718 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2719 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2720 kpu->lt_def->pck_ol2.ltype_mask); 2721 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2722 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2723 kpu->lt_def->pck_oip4.ltype_mask); 2724 2725 /* Config Inner IPV4 NPC layer info */ 2726 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2727 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2728 kpu->lt_def->pck_iip4.ltype_mask); 2729 2730 /* Enable below for Rx pkts. 2731 * - Outer IPv4 header checksum validation. 2732 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2733 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2734 * - Inner IPv4 header checksum validation. 2735 * - Set non zero checksum error code value 2736 */ 2737 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2738 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2739 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2740 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2741 2742 rvu_npc_setup_interfaces(rvu, blkaddr); 2743 2744 npc_config_secret_key(rvu, blkaddr); 2745 /* Configure MKEX profile */ 2746 if (is_cn20k(rvu->pdev)) 2747 npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2748 else 2749 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2750 2751 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2752 if (err) 2753 return err; 2754 2755 err = npc_flow_steering_init(rvu, blkaddr); 2756 if (err) { 2757 dev_err(rvu->dev, 2758 "Incorrect mkex profile loaded using default mkex\n"); 2759 if (is_cn20k(rvu->pdev)) 2760 npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2761 else 2762 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2763 } 2764 2765 if (is_cn20k(rvu->pdev)) 2766 return npc_cn20k_init(rvu); 2767 2768 return 0; 2769 } 2770 2771 void rvu_npc_freemem(struct rvu *rvu) 2772 { 2773 struct npc_pkind *pkind = &rvu->hw->pkind; 2774 struct npc_mcam *mcam = &rvu->hw->mcam; 2775 2776 kfree(pkind->rsrc.bmap); 2777 npc_mcam_rsrcs_deinit(rvu); 2778 if (rvu->kpu_prfl_addr) 2779 iounmap(rvu->kpu_prfl_addr); 2780 else 2781 kfree(rvu->kpu_fwdata); 2782 mutex_destroy(&mcam->lock); 2783 2784 if (is_cn20k(rvu->pdev)) 2785 npc_cn20k_deinit(rvu); 2786 } 2787 2788 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2789 int blkaddr, int *alloc_cnt, 2790 int *enable_cnt) 2791 { 2792 struct npc_mcam *mcam = &rvu->hw->mcam; 2793 int entry; 2794 2795 *alloc_cnt = 0; 2796 *enable_cnt = 0; 2797 2798 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2799 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2800 (*alloc_cnt)++; 2801 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2802 (*enable_cnt)++; 2803 } 2804 } 2805 } 2806 2807 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2808 int blkaddr, int *alloc_cnt, 2809 int *enable_cnt) 2810 { 2811 struct npc_mcam *mcam = &rvu->hw->mcam; 2812 int cntr; 2813 2814 *alloc_cnt = 0; 2815 *enable_cnt = 0; 2816 2817 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2818 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2819 (*alloc_cnt)++; 2820 if (mcam->cntr_refcnt[cntr]) 2821 (*enable_cnt)++; 2822 } 2823 } 2824 } 2825 2826 int npc_mcam_verify_entry(struct npc_mcam *mcam, 2827 u16 pcifunc, int entry) 2828 { 2829 /* verify AF installed entries */ 2830 if (is_pffunc_af(pcifunc)) 2831 return 0; 2832 /* Verify if entry is valid and if it is indeed 2833 * allocated to the requesting PFFUNC. 2834 */ 2835 if (entry >= mcam->bmap_entries) 2836 return NPC_MCAM_INVALID_REQ; 2837 2838 if (pcifunc != mcam->entry2pfvf_map[entry]) 2839 return NPC_MCAM_PERM_DENIED; 2840 2841 return 0; 2842 } 2843 2844 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2845 u16 pcifunc, int cntr) 2846 { 2847 /* Verify if counter is valid and if it is indeed 2848 * allocated to the requesting PFFUNC. 2849 */ 2850 if (cntr >= mcam->counters.max) 2851 return NPC_MCAM_INVALID_REQ; 2852 2853 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2854 return NPC_MCAM_PERM_DENIED; 2855 2856 return 0; 2857 } 2858 2859 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2860 int blkaddr, u16 entry, u16 cntr) 2861 { 2862 u16 index = entry & (mcam->banksize - 1); 2863 u32 bank = npc_get_bank(mcam, entry); 2864 struct rvu_hwinfo *hw = rvu->hw; 2865 2866 /* Set mapping and increment counter's refcnt */ 2867 mcam->entry2cntr_map[entry] = cntr; 2868 mcam->cntr_refcnt[cntr]++; 2869 2870 if (is_cn20k(rvu->pdev)) 2871 return; 2872 2873 /* Enable stats */ 2874 rvu_write64(rvu, blkaddr, 2875 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2876 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2877 } 2878 2879 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2880 struct npc_mcam *mcam, 2881 int blkaddr, u16 entry, u16 cntr) 2882 { 2883 u16 index = entry & (mcam->banksize - 1); 2884 u32 bank = npc_get_bank(mcam, entry); 2885 2886 /* Remove mapping and reduce counter's refcnt */ 2887 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2888 mcam->cntr_refcnt[cntr]--; 2889 2890 if (is_cn20k(rvu->pdev)) 2891 return; 2892 2893 /* Disable stats */ 2894 rvu_write64(rvu, blkaddr, 2895 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2896 } 2897 2898 /* Sets MCAM entry in bitmap as used. Update 2899 * reverse bitmap too. Should be called with 2900 * 'mcam->lock' held. 2901 */ 2902 void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2903 { 2904 u16 entry, rentry; 2905 2906 entry = index; 2907 rentry = mcam->bmap_entries - index - 1; 2908 2909 __set_bit(entry, mcam->bmap); 2910 __set_bit(rentry, mcam->bmap_reverse); 2911 mcam->bmap_fcnt--; 2912 } 2913 2914 /* Sets MCAM entry in bitmap as free. Update 2915 * reverse bitmap too. Should be called with 2916 * 'mcam->lock' held. 2917 */ 2918 void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2919 { 2920 u16 entry, rentry; 2921 2922 entry = index; 2923 rentry = mcam->bmap_entries - index - 1; 2924 2925 __clear_bit(entry, mcam->bmap); 2926 __clear_bit(rentry, mcam->bmap_reverse); 2927 mcam->bmap_fcnt++; 2928 } 2929 2930 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2931 int blkaddr, u16 pcifunc) 2932 { 2933 u16 dft_idxs[NPC_DFT_RULE_MAX_ID] = {[0 ... NPC_DFT_RULE_MAX_ID - 1] = USHRT_MAX}; 2934 bool cn20k_dft_rl; 2935 u16 index, cntr; 2936 int rc; 2937 2938 npc_cn20k_dft_rules_idx_get(rvu, pcifunc, 2939 &dft_idxs[NPC_DFT_RULE_BCAST_ID], 2940 &dft_idxs[NPC_DFT_RULE_MCAST_ID], 2941 &dft_idxs[NPC_DFT_RULE_PROMISC_ID], 2942 &dft_idxs[NPC_DFT_RULE_UCAST_ID]); 2943 2944 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2945 for (index = 0; index < mcam->bmap_entries; index++) { 2946 if (mcam->entry2pfvf_map[index] != pcifunc) 2947 continue; 2948 2949 cn20k_dft_rl = false; 2950 2951 if (is_cn20k(rvu->pdev)) { 2952 if (dft_idxs[NPC_DFT_RULE_BCAST_ID] == index || 2953 dft_idxs[NPC_DFT_RULE_MCAST_ID] == index || 2954 dft_idxs[NPC_DFT_RULE_PROMISC_ID] == index || 2955 dft_idxs[NPC_DFT_RULE_UCAST_ID] == index) { 2956 cn20k_dft_rl = true; 2957 } 2958 } 2959 2960 if (!cn20k_dft_rl) { 2961 /* Disable the entry */ 2962 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2963 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2964 /* Free the entry in bitmap */ 2965 npc_mcam_clear_bit(mcam, index); 2966 mcam->entry2target_pffunc[index] = 0x0; 2967 } 2968 2969 /* Update entry2counter mapping */ 2970 cntr = mcam->entry2cntr_map[index]; 2971 if (cntr != NPC_MCAM_INVALID_MAP) 2972 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2973 blkaddr, index, 2974 cntr); 2975 2976 if (!is_cn20k(rvu->pdev) || cn20k_dft_rl) 2977 continue; 2978 2979 rc = npc_cn20k_idx_free(rvu, &index, 1); 2980 if (rc) 2981 dev_err(rvu->dev, 2982 "Failed to free mcam idx=%u pcifunc=%#x\n", 2983 index, pcifunc); 2984 } 2985 } 2986 2987 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2988 u16 pcifunc) 2989 { 2990 u16 cntr; 2991 2992 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2993 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2994 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2995 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2996 mcam->cntr_refcnt[cntr] = 0; 2997 rvu_free_rsrc(&mcam->counters, cntr); 2998 /* This API is expected to be called after freeing 2999 * MCAM entries, which inturn will remove 3000 * 'entry to counter' mapping. 3001 * No need to do it again. 3002 */ 3003 } 3004 } 3005 } 3006 3007 /* Find area of contiguous free entries of size 'nr'. 3008 * If not found return max contiguous free entries available. 3009 */ 3010 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 3011 u16 nr, u16 *max_area) 3012 { 3013 u16 max_area_start = 0; 3014 u16 index, next, end; 3015 3016 *max_area = 0; 3017 3018 again: 3019 index = find_next_zero_bit(map, size, start); 3020 if (index >= size) 3021 return max_area_start; 3022 3023 end = ((index + nr) >= size) ? size : index + nr; 3024 next = find_next_bit(map, end, index); 3025 if (*max_area < (next - index)) { 3026 *max_area = next - index; 3027 max_area_start = index; 3028 } 3029 3030 if (next < end) { 3031 start = next + 1; 3032 goto again; 3033 } 3034 3035 return max_area_start; 3036 } 3037 3038 /* Find number of free MCAM entries available 3039 * within range i.e in between 'start' and 'end'. 3040 */ 3041 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 3042 { 3043 u16 index, next; 3044 u16 fcnt = 0; 3045 3046 again: 3047 if (start >= end) 3048 return fcnt; 3049 3050 index = find_next_zero_bit(map, end, start); 3051 if (index >= end) 3052 return fcnt; 3053 3054 next = find_next_bit(map, end, index); 3055 if (next <= end) { 3056 fcnt += next - index; 3057 start = next + 1; 3058 goto again; 3059 } 3060 3061 fcnt += end - index; 3062 return fcnt; 3063 } 3064 3065 static void 3066 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 3067 struct npc_mcam_alloc_entry_req *req, 3068 u16 *start, u16 *end, bool *reverse) 3069 { 3070 u16 fcnt; 3071 3072 if (req->ref_prio == NPC_MCAM_HIGHER_PRIO) 3073 goto hprio; 3074 3075 /* For a low priority entry allocation 3076 * - If reference entry is not in hprio zone then 3077 * search range: ref_entry to end. 3078 * - If reference entry is in hprio zone and if 3079 * request can be accomodated in non-hprio zone then 3080 * search range: 'start of middle zone' to 'end' 3081 * - else search in reverse, so that less number of hprio 3082 * zone entries are allocated. 3083 */ 3084 3085 *reverse = false; 3086 *start = req->ref_entry + 1; 3087 *end = mcam->bmap_entries; 3088 3089 if (req->ref_entry >= mcam->hprio_end) 3090 return; 3091 3092 fcnt = npc_mcam_get_free_count(mcam->bmap, 3093 mcam->hprio_end, mcam->bmap_entries); 3094 if (fcnt > req->count) 3095 *start = mcam->hprio_end; 3096 else 3097 *reverse = true; 3098 return; 3099 3100 hprio: 3101 /* For a high priority entry allocation, search is always 3102 * in reverse to preserve hprio zone entries. 3103 * - If reference entry is not in lprio zone then 3104 * search range: 0 to ref_entry. 3105 * - If reference entry is in lprio zone and if 3106 * request can be accomodated in middle zone then 3107 * search range: 'hprio_end' to 'lprio_start' 3108 */ 3109 3110 *reverse = true; 3111 *start = 0; 3112 *end = req->ref_entry; 3113 3114 if (req->ref_entry <= mcam->lprio_start) 3115 return; 3116 3117 fcnt = npc_mcam_get_free_count(mcam->bmap, 3118 mcam->hprio_end, mcam->lprio_start); 3119 if (fcnt < req->count) 3120 return; 3121 *start = mcam->hprio_end; 3122 *end = mcam->lprio_start; 3123 } 3124 3125 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 3126 struct npc_mcam_alloc_entry_req *req, 3127 struct npc_mcam_alloc_entry_rsp *rsp) 3128 { 3129 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 3130 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 3131 u16 fcnt, hp_fcnt, lp_fcnt; 3132 struct rvu *rvu = hw->rvu; 3133 u16 start, end, index; 3134 int entry, next_start; 3135 bool reverse = false; 3136 unsigned long *bmap; 3137 int ret, limit = 0; 3138 u16 max_contig; 3139 3140 if (!is_cn20k(rvu->pdev)) 3141 goto not_cn20k; 3142 3143 /* Only x2 or x4 key types are accepted */ 3144 if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4) 3145 return NPC_MCAM_INVALID_REQ; 3146 3147 /* The below table is being followed during allocation, 3148 * 3149 * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to 3150 * allocate 0th index 3151 * 2. ref_entry == 0 && prio == HIGH && count > 1 : Invalid request 3152 * 3. ref_entry == 0 && prio == LOW && count >= 1 : limit = 0 3153 * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0 3154 * 5. ref_entry != 0 && prio == LOW && count >=1 : limit = Max 3155 * (X2 2*8192, X4 8192) 3156 */ 3157 if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) { 3158 if (req->kw_type == NPC_MCAM_KEY_X2) 3159 limit = 2 * mcam->bmap_entries; 3160 else 3161 limit = mcam->bmap_entries; 3162 } 3163 3164 ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type, 3165 req->ref_prio, rsp->entry_list, 3166 req->ref_entry, limit, 3167 req->contig, req->count, !!req->virt); 3168 3169 if (ret) { 3170 rsp->count = 0; 3171 return NPC_MCAM_ALLOC_FAILED; 3172 } 3173 3174 rsp->count = req->count; 3175 if (req->contig) 3176 rsp->entry = rsp->entry_list[0]; 3177 3178 /* cn20k, entries allocation algorithm is different. 3179 * This common API updates some bitmap on usage etc, which 3180 * will be used by other functions. So update those for 3181 * cn20k as well. 3182 */ 3183 3184 mutex_lock(&mcam->lock); 3185 /* Mark the allocated entries as used and set nixlf mapping */ 3186 for (entry = 0; entry < rsp->count; entry++) { 3187 index = npc_cn20k_vidx2idx(rsp->entry_list[entry]); 3188 npc_mcam_set_bit(mcam, index); 3189 mcam->entry2pfvf_map[index] = pcifunc; 3190 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 3191 } 3192 3193 /* cn20k, free count is provided thru different mbox message. 3194 * one counter to indicate free x2 slots and free x4 slots 3195 * does not provide any useful information to the user. 3196 */ 3197 rsp->free_count = -1; 3198 mutex_unlock(&mcam->lock); 3199 3200 return 0; 3201 3202 not_cn20k: 3203 mutex_lock(&mcam->lock); 3204 3205 /* Check if there are any free entries */ 3206 if (!mcam->bmap_fcnt) { 3207 mutex_unlock(&mcam->lock); 3208 return NPC_MCAM_ALLOC_FAILED; 3209 } 3210 3211 /* MCAM entries are divided into high priority, middle and 3212 * low priority zones. Idea is to not allocate top and lower 3213 * most entries as much as possible, this is to increase 3214 * probability of honouring priority allocation requests. 3215 * 3216 * Two bitmaps are used for mcam entry management, 3217 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 3218 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 3219 * 3220 * Reverse bitmap is used to allocate entries 3221 * - when a higher priority entry is requested 3222 * - when available free entries are less. 3223 * Lower priority ones out of avaialble free entries are always 3224 * chosen when 'high vs low' question arises. 3225 * 3226 * For a VF base MCAM match rule is set by its PF. And all the 3227 * further MCAM rules installed by VF on its own are 3228 * concatenated with the base rule set by its PF. Hence PF entries 3229 * should be at lower priority compared to VF entries. Otherwise 3230 * base rule is hit always and rules installed by VF will be of 3231 * no use. Hence if the request is from PF then allocate low 3232 * priority entries. 3233 */ 3234 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 3235 goto lprio_alloc; 3236 3237 /* Get the search range for priority allocation request */ 3238 if (req->ref_prio) { 3239 npc_get_mcam_search_range_priority(mcam, req, 3240 &start, &end, &reverse); 3241 goto alloc; 3242 } 3243 3244 /* Find out the search range for non-priority allocation request 3245 * 3246 * Get MCAM free entry count in middle zone. 3247 */ 3248 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 3249 mcam->lprio_start, 3250 mcam->bmap_entries); 3251 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 3252 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 3253 3254 /* Check if request can be accomodated in the middle zone */ 3255 if (fcnt > req->count) { 3256 start = mcam->hprio_end; 3257 end = mcam->lprio_start; 3258 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 3259 /* Expand search zone from half of hprio zone to 3260 * half of lprio zone. 3261 */ 3262 start = mcam->hprio_end / 2; 3263 end = mcam->bmap_entries - (mcam->lprio_count / 2); 3264 reverse = true; 3265 } else { 3266 /* Not enough free entries, search all entries in reverse, 3267 * so that low priority ones will get used up. 3268 */ 3269 lprio_alloc: 3270 reverse = true; 3271 start = 0; 3272 end = mcam->bmap_entries; 3273 /* Ensure PF requests are always at bottom and if PF requests 3274 * for higher/lower priority entry wrt reference entry then 3275 * honour that criteria and start search for entries from bottom 3276 * and not in mid zone. 3277 */ 3278 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 3279 req->ref_prio == NPC_MCAM_HIGHER_PRIO) 3280 end = req->ref_entry; 3281 3282 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 3283 req->ref_prio == NPC_MCAM_LOWER_PRIO) 3284 start = req->ref_entry; 3285 } 3286 3287 alloc: 3288 if (reverse) { 3289 bmap = mcam->bmap_reverse; 3290 start = mcam->bmap_entries - start; 3291 end = mcam->bmap_entries - end; 3292 swap(start, end); 3293 } else { 3294 bmap = mcam->bmap; 3295 } 3296 3297 if (req->contig) { 3298 /* Allocate requested number of contiguous entries, if 3299 * unsuccessful find max contiguous entries available. 3300 */ 3301 index = npc_mcam_find_zero_area(bmap, end, start, 3302 req->count, &max_contig); 3303 rsp->count = max_contig; 3304 if (reverse) 3305 rsp->entry = mcam->bmap_entries - index - max_contig; 3306 else 3307 rsp->entry = index; 3308 } else { 3309 /* Allocate requested number of non-contiguous entries, 3310 * if unsuccessful allocate as many as possible. 3311 */ 3312 rsp->count = 0; 3313 next_start = start; 3314 for (entry = 0; entry < req->count; entry++) { 3315 index = find_next_zero_bit(bmap, end, next_start); 3316 if (index >= end) 3317 break; 3318 3319 next_start = start + (index - start) + 1; 3320 3321 /* Save the entry's index */ 3322 if (reverse) 3323 index = mcam->bmap_entries - index - 1; 3324 entry_list[entry] = index; 3325 rsp->count++; 3326 } 3327 } 3328 3329 /* If allocating requested no of entries is unsucessful, 3330 * expand the search range to full bitmap length and retry. 3331 */ 3332 if (!req->ref_prio && rsp->count < req->count && 3333 ((end - start) != mcam->bmap_entries)) { 3334 reverse = true; 3335 start = 0; 3336 end = mcam->bmap_entries; 3337 goto alloc; 3338 } 3339 3340 /* For priority entry allocation requests, if allocation is 3341 * failed then expand search to max possible range and retry. 3342 */ 3343 if (req->ref_prio && rsp->count < req->count) { 3344 if (req->ref_prio == NPC_MCAM_LOWER_PRIO && 3345 (start != (req->ref_entry + 1))) { 3346 start = req->ref_entry + 1; 3347 end = mcam->bmap_entries; 3348 reverse = false; 3349 goto alloc; 3350 } else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) && 3351 ((end - start) != req->ref_entry)) { 3352 start = 0; 3353 end = req->ref_entry; 3354 reverse = true; 3355 goto alloc; 3356 } 3357 } 3358 3359 /* Copy MCAM entry indices into mbox response entry_list. 3360 * Requester always expects indices in ascending order, so 3361 * reverse the list if reverse bitmap is used for allocation. 3362 */ 3363 if (!req->contig && rsp->count) { 3364 index = 0; 3365 for (entry = rsp->count - 1; entry >= 0; entry--) { 3366 if (reverse) 3367 rsp->entry_list[index++] = entry_list[entry]; 3368 else 3369 rsp->entry_list[entry] = entry_list[entry]; 3370 } 3371 } 3372 3373 /* Mark the allocated entries as used and set nixlf mapping */ 3374 for (entry = 0; entry < rsp->count; entry++) { 3375 index = req->contig ? 3376 (rsp->entry + entry) : rsp->entry_list[entry]; 3377 npc_mcam_set_bit(mcam, index); 3378 mcam->entry2pfvf_map[index] = pcifunc; 3379 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 3380 } 3381 3382 /* Update available free count in mbox response */ 3383 rsp->free_count = mcam->bmap_fcnt; 3384 3385 mutex_unlock(&mcam->lock); 3386 return 0; 3387 } 3388 3389 /* Marks bitmaps to reserved the mcam slot */ 3390 void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx) 3391 { 3392 struct npc_mcam *mcam = &rvu->hw->mcam; 3393 3394 npc_mcam_set_bit(mcam, entry_idx); 3395 } 3396 3397 int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) 3398 { 3399 struct npc_mcam *mcam = &rvu->hw->mcam; 3400 struct npc_install_flow_rsp rsp; 3401 struct rvu_npc_mcam_rule *rule; 3402 int blkaddr; 3403 3404 /* Counter is set for each rule by default */ 3405 if (is_cn20k(rvu->pdev)) 3406 return -EINVAL; 3407 3408 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3409 if (blkaddr < 0) 3410 return -EINVAL; 3411 3412 mutex_lock(&mcam->lock); 3413 list_for_each_entry(rule, &mcam->mcam_rules, list) { 3414 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, rule->entry)) 3415 continue; 3416 if (!rule->default_rule) 3417 continue; 3418 if (enable && !rule->has_cntr) { /* Alloc and map new counter */ 3419 __rvu_mcam_add_counter_to_rule(rvu, rule->owner, 3420 rule, &rsp); 3421 if (rsp.counter < 0) { 3422 dev_err(rvu->dev, 3423 "%s: Failed to allocate cntr for default rule (err=%d)\n", 3424 __func__, rsp.counter); 3425 break; 3426 } 3427 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3428 rule->entry, rsp.counter); 3429 /* Reset counter before use */ 3430 rvu_write64(rvu, blkaddr, 3431 NPC_AF_MATCH_STATX(rule->cntr), 0x0); 3432 } 3433 3434 /* Free and unmap counter */ 3435 if (!enable && rule->has_cntr) 3436 __rvu_mcam_remove_counter_from_rule(rvu, rule->owner, 3437 rule); 3438 } 3439 mutex_unlock(&mcam->lock); 3440 3441 return 0; 3442 } 3443 3444 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 3445 struct npc_mcam_alloc_entry_req *req, 3446 struct npc_mcam_alloc_entry_rsp *rsp) 3447 { 3448 struct npc_mcam *mcam = &rvu->hw->mcam; 3449 u16 pcifunc = req->hdr.pcifunc; 3450 int blkaddr; 3451 3452 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3453 if (blkaddr < 0) 3454 return NPC_MCAM_INVALID_REQ; 3455 3456 rsp->entry = NPC_MCAM_ENTRY_INVALID; 3457 rsp->free_count = 0; 3458 3459 /* Check if ref_entry is greater that the range 3460 * then set it to max value. 3461 */ 3462 if (req->ref_entry > mcam->bmap_entries) 3463 req->ref_entry = mcam->bmap_entries; 3464 3465 /* ref_entry can't be '0' if requested priority is high. 3466 * Can't be last entry if requested priority is low. 3467 */ 3468 if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) || 3469 (req->ref_entry == mcam->bmap_entries && 3470 req->ref_prio == NPC_MCAM_LOWER_PRIO)) 3471 return NPC_MCAM_INVALID_REQ; 3472 3473 /* Since list of allocated indices needs to be sent to requester, 3474 * max number of non-contiguous entries per mbox msg is limited. 3475 */ 3476 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 3477 dev_err(rvu->dev, 3478 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 3479 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 3480 return NPC_MCAM_INVALID_REQ; 3481 } 3482 3483 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 3484 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3485 return NPC_MCAM_ALLOC_DENIED; 3486 3487 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 3488 } 3489 3490 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 3491 struct npc_mcam_free_entry_req *req, 3492 struct msg_rsp *rsp) 3493 { 3494 struct npc_mcam *mcam = &rvu->hw->mcam; 3495 u16 pcifunc = req->hdr.pcifunc; 3496 int blkaddr, rc = 0; 3497 u16 cntr; 3498 3499 req->entry = npc_cn20k_vidx2idx(req->entry); 3500 3501 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3502 if (blkaddr < 0) 3503 return NPC_MCAM_INVALID_REQ; 3504 3505 /* Free request from PFFUNC with no NIXLF attached, ignore */ 3506 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3507 return NPC_MCAM_INVALID_REQ; 3508 3509 mutex_lock(&mcam->lock); 3510 3511 if (req->all) 3512 goto free_all; 3513 3514 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3515 if (rc) 3516 goto exit; 3517 3518 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 3519 mcam->entry2target_pffunc[req->entry] = 0x0; 3520 npc_mcam_clear_bit(mcam, req->entry); 3521 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3522 3523 /* Update entry2counter mapping */ 3524 cntr = mcam->entry2cntr_map[req->entry]; 3525 if (cntr != NPC_MCAM_INVALID_MAP) 3526 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3527 req->entry, cntr); 3528 3529 if (is_cn20k(rvu->pdev)) { 3530 rc = npc_cn20k_idx_free(rvu, &req->entry, 1); 3531 if (rc) 3532 dev_err(rvu->dev, 3533 "Failed to free index=%u\n", 3534 req->entry); 3535 } 3536 3537 goto exit; 3538 3539 free_all: 3540 /* Free up all entries allocated to requesting PFFUNC */ 3541 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 3542 exit: 3543 mutex_unlock(&mcam->lock); 3544 return rc; 3545 } 3546 3547 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 3548 struct npc_mcam_read_entry_req *req, 3549 struct npc_mcam_read_entry_rsp *rsp) 3550 { 3551 struct npc_mcam *mcam = &rvu->hw->mcam; 3552 u16 pcifunc = req->hdr.pcifunc; 3553 int blkaddr, rc; 3554 3555 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3556 if (blkaddr < 0) 3557 return NPC_MCAM_INVALID_REQ; 3558 3559 mutex_lock(&mcam->lock); 3560 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3561 if (!rc) { 3562 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 3563 &rsp->entry_data, 3564 &rsp->intf, &rsp->enable); 3565 } 3566 3567 mutex_unlock(&mcam->lock); 3568 return rc; 3569 } 3570 3571 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 3572 struct npc_mcam_write_entry_req *req, 3573 struct msg_rsp *rsp) 3574 { 3575 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3576 struct npc_mcam *mcam = &rvu->hw->mcam; 3577 u16 pcifunc = req->hdr.pcifunc; 3578 int blkaddr, rc; 3579 u8 nix_intf; 3580 3581 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3582 if (blkaddr < 0) 3583 return NPC_MCAM_INVALID_REQ; 3584 3585 mutex_lock(&mcam->lock); 3586 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3587 if (rc) 3588 goto exit; 3589 3590 if (!is_cn20k(rvu->pdev)) { 3591 /* Verify counter in SoCs other than cn20k */ 3592 if (req->set_cntr && 3593 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 3594 rc = NPC_MCAM_INVALID_REQ; 3595 goto exit; 3596 } 3597 } 3598 3599 if (!is_npc_interface_valid(rvu, req->intf)) { 3600 rc = NPC_MCAM_INVALID_REQ; 3601 goto exit; 3602 } 3603 3604 if (is_npc_intf_tx(req->intf)) 3605 nix_intf = pfvf->nix_tx_intf; 3606 else 3607 nix_intf = pfvf->nix_rx_intf; 3608 3609 /* For AF installed rules, the nix_intf should be set to target NIX */ 3610 if (is_pffunc_af(req->hdr.pcifunc)) 3611 nix_intf = req->intf; 3612 3613 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 3614 &req->entry_data, req->enable_entry); 3615 3616 if (req->set_cntr) 3617 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3618 req->entry, req->cntr); 3619 3620 rc = 0; 3621 exit: 3622 mutex_unlock(&mcam->lock); 3623 return rc; 3624 } 3625 3626 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 3627 struct npc_mcam_ena_dis_entry_req *req, 3628 struct msg_rsp *rsp) 3629 { 3630 struct npc_mcam *mcam = &rvu->hw->mcam; 3631 u16 pcifunc = req->hdr.pcifunc; 3632 int blkaddr, rc; 3633 3634 req->entry = npc_cn20k_vidx2idx(req->entry); 3635 3636 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3637 if (blkaddr < 0) 3638 return NPC_MCAM_INVALID_REQ; 3639 3640 mutex_lock(&mcam->lock); 3641 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3642 mutex_unlock(&mcam->lock); 3643 if (rc) 3644 return rc; 3645 3646 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 3647 3648 return 0; 3649 } 3650 3651 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 3652 struct npc_mcam_ena_dis_entry_req *req, 3653 struct msg_rsp *rsp) 3654 { 3655 struct npc_mcam *mcam = &rvu->hw->mcam; 3656 u16 pcifunc = req->hdr.pcifunc; 3657 int blkaddr, rc; 3658 3659 req->entry = npc_cn20k_vidx2idx(req->entry); 3660 3661 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3662 if (blkaddr < 0) 3663 return NPC_MCAM_INVALID_REQ; 3664 3665 mutex_lock(&mcam->lock); 3666 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 3667 mutex_unlock(&mcam->lock); 3668 if (rc) 3669 return rc; 3670 3671 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 3672 3673 return 0; 3674 } 3675 3676 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 3677 struct npc_mcam_shift_entry_req *req, 3678 struct npc_mcam_shift_entry_rsp *rsp) 3679 { 3680 struct npc_mcam *mcam = &rvu->hw->mcam; 3681 u16 pcifunc = req->hdr.pcifunc; 3682 u16 old_entry, new_entry; 3683 int blkaddr, rc = 0; 3684 u16 index, cntr; 3685 3686 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3687 if (blkaddr < 0) 3688 return NPC_MCAM_INVALID_REQ; 3689 3690 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 3691 return NPC_MCAM_INVALID_REQ; 3692 3693 mutex_lock(&mcam->lock); 3694 for (index = 0; index < req->shift_count; index++) { 3695 old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]); 3696 new_entry = npc_cn20k_vidx2idx(req->new_entry[index]); 3697 3698 /* Check if both old and new entries are valid and 3699 * does belong to this PFFUNC or not. 3700 */ 3701 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 3702 if (rc) 3703 break; 3704 3705 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 3706 if (rc) 3707 break; 3708 3709 /* new_entry should not have a counter mapped */ 3710 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 3711 rc = NPC_MCAM_PERM_DENIED; 3712 break; 3713 } 3714 3715 /* Disable the new_entry */ 3716 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 3717 3718 /* Copy rule from old entry to new entry */ 3719 if (npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry)) { 3720 rc = NPC_MCAM_INVALID_REQ; 3721 break; 3722 } 3723 3724 /* Copy counter mapping, if any */ 3725 cntr = mcam->entry2cntr_map[old_entry]; 3726 if (cntr != NPC_MCAM_INVALID_MAP) { 3727 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3728 old_entry, cntr); 3729 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3730 new_entry, cntr); 3731 } 3732 3733 /* Enable new_entry and disable old_entry */ 3734 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 3735 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 3736 } 3737 3738 /* If shift has failed then report the failed index */ 3739 if (index != req->shift_count) { 3740 if (!rc) 3741 rc = NPC_MCAM_PERM_DENIED; 3742 rsp->failed_entry_idx = index; 3743 } 3744 3745 mutex_unlock(&mcam->lock); 3746 return rc; 3747 } 3748 3749 static int __npc_mcam_alloc_counter(struct rvu *rvu, 3750 struct npc_mcam_alloc_counter_req *req, 3751 struct npc_mcam_alloc_counter_rsp *rsp) 3752 { 3753 struct npc_mcam *mcam = &rvu->hw->mcam; 3754 u16 pcifunc = req->hdr.pcifunc; 3755 u16 max_contig, cntr; 3756 int blkaddr, index; 3757 3758 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3759 if (blkaddr < 0) 3760 return NPC_MCAM_INVALID_REQ; 3761 3762 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 3763 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3764 return NPC_MCAM_INVALID_REQ; 3765 3766 /* Since list of allocated counter IDs needs to be sent to requester, 3767 * max number of non-contiguous counters per mbox msg is limited. 3768 */ 3769 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 3770 return NPC_MCAM_INVALID_REQ; 3771 3772 /* Check if unused counters are available or not */ 3773 if (!rvu_rsrc_free_count(&mcam->counters)) { 3774 return NPC_MCAM_ALLOC_FAILED; 3775 } 3776 3777 rsp->count = 0; 3778 3779 if (req->contig) { 3780 /* Allocate requested number of contiguous counters, if 3781 * unsuccessful find max contiguous entries available. 3782 */ 3783 index = npc_mcam_find_zero_area(mcam->counters.bmap, 3784 mcam->counters.max, 0, 3785 req->count, &max_contig); 3786 rsp->count = max_contig; 3787 rsp->cntr = index; 3788 for (cntr = index; cntr < (index + max_contig); cntr++) { 3789 __set_bit(cntr, mcam->counters.bmap); 3790 mcam->cntr2pfvf_map[cntr] = pcifunc; 3791 } 3792 } else { 3793 /* Allocate requested number of non-contiguous counters, 3794 * if unsuccessful allocate as many as possible. 3795 */ 3796 for (cntr = 0; cntr < req->count; cntr++) { 3797 index = rvu_alloc_rsrc(&mcam->counters); 3798 if (index < 0) 3799 break; 3800 rsp->cntr_list[cntr] = index; 3801 rsp->count++; 3802 mcam->cntr2pfvf_map[index] = pcifunc; 3803 } 3804 } 3805 3806 return 0; 3807 } 3808 3809 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 3810 struct npc_mcam_alloc_counter_req *req, 3811 struct npc_mcam_alloc_counter_rsp *rsp) 3812 { 3813 struct npc_mcam *mcam = &rvu->hw->mcam; 3814 int err; 3815 3816 /* Counter is not supported for CN20K */ 3817 if (is_cn20k(rvu->pdev)) 3818 return NPC_MCAM_INVALID_REQ; 3819 3820 mutex_lock(&mcam->lock); 3821 3822 err = __npc_mcam_alloc_counter(rvu, req, rsp); 3823 3824 mutex_unlock(&mcam->lock); 3825 return err; 3826 } 3827 3828 static int __npc_mcam_free_counter(struct rvu *rvu, 3829 struct npc_mcam_oper_counter_req *req, 3830 struct msg_rsp *rsp) 3831 { 3832 struct npc_mcam *mcam = &rvu->hw->mcam; 3833 u16 index; 3834 int blkaddr, err; 3835 3836 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3837 if (blkaddr < 0) 3838 return NPC_MCAM_INVALID_REQ; 3839 3840 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3841 if (err) { 3842 return err; 3843 } 3844 3845 /* Mark counter as free/unused */ 3846 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 3847 rvu_free_rsrc(&mcam->counters, req->cntr); 3848 3849 /* Disable all MCAM entry's stats which are using this counter. 3850 * Scan the full MCAM index range: AF-reserved rules (e.g. CPT pass-2) 3851 */ 3852 for (index = 0; index < mcam->total_entries; index++) { 3853 if (!mcam->cntr_refcnt[req->cntr]) 3854 break; 3855 if (mcam->entry2cntr_map[index] != req->cntr) 3856 continue; 3857 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, 3858 req->cntr); 3859 } 3860 3861 return 0; 3862 } 3863 3864 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 3865 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3866 { 3867 struct npc_mcam *mcam = &rvu->hw->mcam; 3868 int err; 3869 3870 /* Counter is not supported for CN20K */ 3871 if (is_cn20k(rvu->pdev)) 3872 return NPC_MCAM_INVALID_REQ; 3873 3874 mutex_lock(&mcam->lock); 3875 3876 err = __npc_mcam_free_counter(rvu, req, rsp); 3877 3878 mutex_unlock(&mcam->lock); 3879 3880 return err; 3881 } 3882 3883 void __rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 3884 struct rvu_npc_mcam_rule *rule) 3885 { 3886 struct npc_mcam_oper_counter_req free_req = { 0 }; 3887 struct msg_rsp free_rsp; 3888 3889 if (!rule->has_cntr) 3890 return; 3891 3892 free_req.hdr.pcifunc = pcifunc; 3893 free_req.cntr = rule->cntr; 3894 3895 __npc_mcam_free_counter(rvu, &free_req, &free_rsp); 3896 rule->has_cntr = false; 3897 } 3898 3899 void __rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 3900 struct rvu_npc_mcam_rule *rule, 3901 struct npc_install_flow_rsp *rsp) 3902 { 3903 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 3904 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 3905 int err; 3906 3907 cntr_req.hdr.pcifunc = pcifunc; 3908 cntr_req.contig = true; 3909 cntr_req.count = 1; 3910 3911 /* we try to allocate a counter to track the stats of this 3912 * rule. If counter could not be allocated then proceed 3913 * without counter because counters are limited than entries. 3914 */ 3915 err = __npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3916 if (!err && cntr_rsp.count) { 3917 rule->cntr = cntr_rsp.cntr; 3918 rule->has_cntr = true; 3919 rsp->counter = rule->cntr; 3920 } else { 3921 rsp->counter = err; 3922 } 3923 } 3924 3925 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 3926 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 3927 { 3928 struct npc_mcam *mcam = &rvu->hw->mcam; 3929 u16 index; 3930 int blkaddr, rc; 3931 3932 /* Counter is not supported for CN20K */ 3933 if (is_cn20k(rvu->pdev)) 3934 return NPC_MCAM_INVALID_REQ; 3935 3936 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3937 if (blkaddr < 0) 3938 return NPC_MCAM_INVALID_REQ; 3939 3940 mutex_lock(&mcam->lock); 3941 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3942 if (rc) 3943 goto exit; 3944 3945 /* Unmap the MCAM entry and counter */ 3946 if (!req->all) { 3947 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 3948 if (rc) 3949 goto exit; 3950 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3951 req->entry, req->cntr); 3952 goto exit; 3953 } 3954 3955 /* Disable all MCAM entry's stats which are using this counter */ 3956 for (index = 0; index < mcam->total_entries; index++) { 3957 if (!mcam->cntr_refcnt[req->cntr]) 3958 break; 3959 if (mcam->entry2cntr_map[index] != req->cntr) 3960 continue; 3961 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, index, 3962 req->cntr); 3963 } 3964 exit: 3965 mutex_unlock(&mcam->lock); 3966 return rc; 3967 } 3968 3969 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3970 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3971 { 3972 struct npc_mcam *mcam = &rvu->hw->mcam; 3973 int blkaddr, err, index, bank; 3974 3975 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3976 if (blkaddr < 0) 3977 return NPC_MCAM_INVALID_REQ; 3978 3979 /* For cn20k, npc mcam index is passed as cntr, as each 3980 * mcam entry has corresponding counter. 3981 */ 3982 if (is_cn20k(rvu->pdev)) { 3983 index = req->cntr & (mcam->banksize - 1); 3984 bank = npc_get_bank(mcam, req->cntr); 3985 rvu_write64(rvu, blkaddr, 3986 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); 3987 return 0; 3988 } 3989 3990 mutex_lock(&mcam->lock); 3991 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3992 mutex_unlock(&mcam->lock); 3993 if (err) 3994 return err; 3995 3996 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3997 3998 return 0; 3999 } 4000 4001 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 4002 struct npc_mcam_oper_counter_req *req, 4003 struct npc_mcam_oper_counter_rsp *rsp) 4004 { 4005 struct npc_mcam *mcam = &rvu->hw->mcam; 4006 int blkaddr, err, index, bank; 4007 u64 regval; 4008 4009 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4010 if (blkaddr < 0) 4011 return NPC_MCAM_INVALID_REQ; 4012 4013 /* In CN20K, mcam index is passed cntr. Each cn20k mcam entry 4014 * has its own counter. No need to verify the counter index. 4015 */ 4016 if (is_cn20k(rvu->pdev)) { 4017 index = req->cntr & (mcam->banksize - 1); 4018 bank = npc_get_bank(mcam, req->cntr); 4019 regval = rvu_read64(rvu, blkaddr, 4020 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 4021 bank)); 4022 rsp->stat = regval; 4023 return 0; 4024 } 4025 4026 mutex_lock(&mcam->lock); 4027 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 4028 mutex_unlock(&mcam->lock); 4029 if (err) 4030 return err; 4031 4032 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 4033 rsp->stat &= BIT_ULL(48) - 1; 4034 4035 return 0; 4036 } 4037 4038 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 4039 struct npc_mcam_alloc_and_write_entry_req *req, 4040 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 4041 { 4042 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 4043 struct npc_mcam_alloc_counter_req cntr_req; 4044 struct npc_mcam_alloc_counter_rsp cntr_rsp; 4045 struct npc_mcam_alloc_entry_req entry_req; 4046 struct npc_mcam_alloc_entry_rsp entry_rsp; 4047 struct npc_mcam *mcam = &rvu->hw->mcam; 4048 u16 entry = NPC_MCAM_ENTRY_INVALID; 4049 u16 cntr = NPC_MCAM_ENTRY_INVALID; 4050 int blkaddr, rc; 4051 u8 nix_intf; 4052 4053 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4054 if (blkaddr < 0) 4055 return NPC_MCAM_INVALID_REQ; 4056 4057 if (!is_npc_interface_valid(rvu, req->intf)) 4058 return NPC_MCAM_INVALID_REQ; 4059 4060 /* Try to allocate a MCAM entry */ 4061 entry_req.hdr.pcifunc = req->hdr.pcifunc; 4062 entry_req.contig = true; 4063 entry_req.ref_prio = req->ref_prio; 4064 entry_req.ref_entry = req->ref_entry; 4065 entry_req.count = 1; 4066 4067 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 4068 &entry_req, &entry_rsp); 4069 if (rc) 4070 return rc; 4071 4072 if (!entry_rsp.count) 4073 return NPC_MCAM_ALLOC_FAILED; 4074 4075 entry = entry_rsp.entry; 4076 4077 if (!req->alloc_cntr) 4078 goto write_entry; 4079 4080 /* Now allocate counter */ 4081 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 4082 cntr_req.contig = true; 4083 cntr_req.count = 1; 4084 4085 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 4086 if (rc) { 4087 /* Free allocated MCAM entry */ 4088 mutex_lock(&mcam->lock); 4089 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 4090 npc_mcam_clear_bit(mcam, entry); 4091 mutex_unlock(&mcam->lock); 4092 return rc; 4093 } 4094 4095 cntr = cntr_rsp.cntr; 4096 4097 write_entry: 4098 mutex_lock(&mcam->lock); 4099 4100 if (is_npc_intf_tx(req->intf)) 4101 nix_intf = pfvf->nix_tx_intf; 4102 else 4103 nix_intf = pfvf->nix_rx_intf; 4104 4105 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 4106 &req->entry_data, req->enable_entry); 4107 4108 if (req->alloc_cntr) 4109 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 4110 mutex_unlock(&mcam->lock); 4111 4112 rsp->entry = entry; 4113 rsp->cntr = cntr; 4114 4115 return 0; 4116 } 4117 4118 #define GET_KEX_CFG(intf) \ 4119 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 4120 4121 #define GET_KEX_FLAGS(ld) \ 4122 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 4123 4124 #define GET_KEX_LD(intf, lid, lt, ld) \ 4125 rvu_read64(rvu, BLKADDR_NPC, \ 4126 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 4127 4128 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 4129 rvu_read64(rvu, BLKADDR_NPC, \ 4130 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 4131 4132 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 4133 struct npc_get_kex_cfg_rsp *rsp) 4134 { 4135 int lid, lt, ld, fl; 4136 4137 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 4138 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 4139 for (lid = 0; lid < NPC_MAX_LID; lid++) { 4140 for (lt = 0; lt < NPC_MAX_LT; lt++) { 4141 for (ld = 0; ld < NPC_MAX_LD; ld++) { 4142 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 4143 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 4144 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 4145 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 4146 } 4147 } 4148 } 4149 for (ld = 0; ld < NPC_MAX_LD; ld++) 4150 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 4151 4152 for (ld = 0; ld < NPC_MAX_LD; ld++) { 4153 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 4154 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 4155 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 4156 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 4157 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 4158 } 4159 } 4160 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 4161 return 0; 4162 } 4163 4164 static int 4165 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 4166 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 4167 { 4168 struct npc_kpu_action0 *act0; 4169 u8 shift_count = 0; 4170 int blkaddr; 4171 u64 val; 4172 4173 if (!var_len_off_mask) 4174 return -EINVAL; 4175 4176 if (var_len_off_mask != 0xff) { 4177 if (shift_dir) 4178 shift_count = __ffs(var_len_off_mask); 4179 else 4180 shift_count = (8 - __fls(var_len_off_mask)); 4181 } 4182 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 4183 if (blkaddr < 0) { 4184 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4185 return -EINVAL; 4186 } 4187 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 4188 act0 = (struct npc_kpu_action0 *)&val; 4189 act0->var_len_shift = shift_count; 4190 act0->var_len_right = shift_dir; 4191 act0->var_len_mask = var_len_off_mask; 4192 act0->var_len_offset = var_len_off; 4193 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 4194 return 0; 4195 } 4196 4197 static int npc_set_skip_size_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 4198 u8 skip_size) 4199 { 4200 struct npc_kpu_action0 *act0; 4201 int blkaddr; 4202 u64 val; 4203 4204 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 4205 if (blkaddr < 0) { 4206 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 4207 return -EINVAL; 4208 } 4209 4210 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 4211 act0 = (struct npc_kpu_action0 *)&val; 4212 act0->ptr_advance = skip_size; 4213 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 4214 4215 /* Update CPT_HR new PKIND */ 4216 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind + 4)); 4217 act0 = (struct npc_kpu_action0 *)&val; 4218 act0->ptr_advance = (skip_size + 40); 4219 act0->next_state = NPC_S_KPU1_CPT_HDR; 4220 act0->var_len_offset = (skip_size + 6); 4221 act0->var_len_mask = 0xe0; 4222 act0->var_len_shift = 0x5; 4223 act0->var_len_right = 0x1; 4224 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind + 4), val); 4225 return 0; 4226 } 4227 4228 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 4229 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 4230 u8 shift_dir, u8 skip_size) 4231 { 4232 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 4233 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4234 int blkaddr, nixlf, rc, intf_mode; 4235 u8 cgx_id = 0, lmac_id = 0; 4236 u64 rxpkind, txpkind; 4237 struct cgx *cgxd; 4238 4239 /* use default pkind to disable edsa/higig */ 4240 rxpkind = rvu_npc_get_pkind(rvu, pf); 4241 txpkind = NPC_TX_DEF_PKIND; 4242 intf_mode = NPC_INTF_MODE_DEF; 4243 4244 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 4245 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 4246 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 4247 var_len_off, 4248 var_len_off_mask, 4249 shift_dir); 4250 if (rc) 4251 return rc; 4252 } else if (pkind >= NPC_RX_SKIP_SIZE_PKIND && 4253 pkind <= NPC_RX_SKIP_SIZE_PKIND + 3) { 4254 rc = npc_set_skip_size_pkind(rvu, pcifunc, pkind, 4255 skip_size); 4256 if (rc) 4257 return rc; 4258 } 4259 rxpkind = pkind; 4260 txpkind = pkind; 4261 } 4262 4263 if (dir & PKIND_RX) { 4264 /* rx pkind set req valid only for cgx mapped PFs */ 4265 if (!is_cgx_config_permitted(rvu, pcifunc)) 4266 return 0; 4267 if (!rvu_cgx_check_permission_and_set_pkind(rvu, pcifunc, rxpkind)) 4268 return -EINVAL; 4269 } 4270 4271 if (dir & PKIND_TX) { 4272 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 4273 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4274 if (rc) 4275 return rc; 4276 4277 if (is_pf_cgxmapped(rvu, pf) && is_vf(pcifunc)) { 4278 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, 4279 &lmac_id); 4280 cgxd = rvu_cgx_pdata(cgx_id, rvu); 4281 mutex_lock(&cgxd->lock); 4282 if (rvu_cgx_is_pkind_config_permitted(rvu, pcifunc)) 4283 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 4284 txpkind); 4285 mutex_unlock(&cgxd->lock); 4286 } else { 4287 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 4288 txpkind); 4289 } 4290 } 4291 4292 pfvf->intf_mode = intf_mode; 4293 return 0; 4294 } 4295 4296 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 4297 struct msg_rsp *rsp) 4298 { 4299 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 4300 req->dir, req->pkind, req->var_len_off, 4301 req->var_len_off_mask, req->shift_dir, 4302 req->skip_size); 4303 } 4304 4305 int rvu_mbox_handler_npc_read_default_rule(struct rvu *rvu, 4306 struct msg_req *req, 4307 struct npc_mcam_read_base_rule_rsp *rsp) 4308 { 4309 struct npc_mcam *mcam = &rvu->hw->mcam; 4310 int index, blkaddr, nixlf, rc; 4311 u16 pcifunc = req->hdr.pcifunc; 4312 u8 intf, enable; 4313 4314 if (is_cn20k(rvu->pdev)) 4315 return NPC_MCAM_INVALID_REQ; 4316 4317 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4318 if (blkaddr < 0) 4319 return NPC_MCAM_INVALID_REQ; 4320 4321 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4322 if (rc < 0) 4323 return rc; 4324 4325 /* Read the default ucast entry */ 4326 mutex_lock(&mcam->lock); 4327 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 4328 NIXLF_UCAST_ENTRY); 4329 if (index < 0) { 4330 mutex_unlock(&mcam->lock); 4331 return NIX_AF_ERR_AF_LF_INVALID; 4332 } 4333 4334 /* Read the mcam entry */ 4335 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 4336 &enable); 4337 mutex_unlock(&mcam->lock); 4338 4339 return 0; 4340 } 4341 4342 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 4343 struct msg_req *req, 4344 struct npc_mcam_read_base_rule_rsp *rsp) 4345 { 4346 struct npc_mcam *mcam = &rvu->hw->mcam; 4347 int index, blkaddr, nixlf, rc = 0; 4348 u16 pcifunc = req->hdr.pcifunc; 4349 struct rvu_pfvf *pfvf; 4350 u8 intf, enable; 4351 4352 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4353 if (blkaddr < 0) 4354 return NPC_MCAM_INVALID_REQ; 4355 4356 /* Return the channel number in case of PF */ 4357 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 4358 pfvf = rvu_get_pfvf(rvu, pcifunc); 4359 rsp->entry.kw[0] = pfvf->rx_chan_base; 4360 rsp->entry.kw_mask[0] = 0xFFFULL; 4361 goto out; 4362 } 4363 4364 /* Find the pkt steering rule installed by PF to this VF */ 4365 mutex_lock(&mcam->lock); 4366 for (index = 0; index < mcam->bmap_entries; index++) { 4367 if (mcam->entry2target_pffunc[index] == pcifunc) 4368 goto read_entry; 4369 } 4370 4371 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4372 if (rc < 0) { 4373 mutex_unlock(&mcam->lock); 4374 goto out; 4375 } 4376 /* Read the default ucast entry if there is no pkt steering rule */ 4377 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 4378 NIXLF_UCAST_ENTRY); 4379 if (index < 0) { 4380 mutex_unlock(&mcam->lock); 4381 rc = NIX_AF_ERR_AF_LF_INVALID; 4382 goto out; 4383 } 4384 4385 read_entry: 4386 /* Read the mcam entry */ 4387 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 4388 &enable); 4389 mutex_unlock(&mcam->lock); 4390 out: 4391 return rc; 4392 } 4393 4394 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 4395 struct npc_mcam_get_stats_req *req, 4396 struct npc_mcam_get_stats_rsp *rsp) 4397 { 4398 struct npc_mcam *mcam = &rvu->hw->mcam; 4399 u16 index, cntr; 4400 int blkaddr; 4401 u64 regval; 4402 u32 bank; 4403 4404 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4405 if (blkaddr < 0) 4406 return NPC_MCAM_INVALID_REQ; 4407 4408 req->entry = npc_cn20k_vidx2idx(req->entry); 4409 4410 index = req->entry & (mcam->banksize - 1); 4411 bank = npc_get_bank(mcam, req->entry); 4412 4413 mutex_lock(&mcam->lock); 4414 4415 if (is_cn20k(rvu->pdev)) { 4416 regval = rvu_read64(rvu, blkaddr, 4417 NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, 4418 bank)); 4419 rsp->stat_ena = 1; 4420 rsp->stat = regval; 4421 mutex_unlock(&mcam->lock); 4422 return 0; 4423 } 4424 4425 /* read MCAM entry STAT_ACT register */ 4426 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, 4427 bank)); 4428 4429 if (!(regval & rvu->hw->npc_stat_ena)) { 4430 rsp->stat_ena = 0; 4431 mutex_unlock(&mcam->lock); 4432 return 0; 4433 } 4434 4435 cntr = regval & 0x1FF; 4436 4437 rsp->stat_ena = 1; 4438 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 4439 rsp->stat &= BIT_ULL(48) - 1; 4440 4441 mutex_unlock(&mcam->lock); 4442 4443 return 0; 4444 } 4445 4446 void rvu_npc_clear_ucast_entry(struct rvu *rvu, int pcifunc, int nixlf) 4447 { 4448 struct npc_mcam *mcam = &rvu->hw->mcam; 4449 struct rvu_npc_mcam_rule *rule; 4450 int ucast_idx, blkaddr; 4451 4452 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 4453 if (blkaddr < 0) 4454 return; 4455 4456 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 4457 nixlf, NIXLF_UCAST_ENTRY); 4458 4459 /* In cn20k, default rules are freed before detach rsrc */ 4460 if (ucast_idx < 0) 4461 return; 4462 4463 npc_enable_mcam_entry(rvu, mcam, blkaddr, ucast_idx, false); 4464 4465 npc_set_mcam_action(rvu, mcam, blkaddr, ucast_idx, 0); 4466 4467 npc_clear_mcam_entry(rvu, mcam, blkaddr, ucast_idx); 4468 4469 mutex_lock(&mcam->lock); 4470 list_for_each_entry(rule, &mcam->mcam_rules, list) { 4471 if (rule->entry == ucast_idx) { 4472 list_del(&rule->list); 4473 kfree(rule); 4474 break; 4475 } 4476 } 4477 mutex_unlock(&mcam->lock); 4478 } 4479