1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 #include "rvu_npc_hash.h" 19 20 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 21 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 22 23 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 24 #define NPC_HW_TSTAMP_OFFSET 8ULL 25 #define NPC_KEX_CHAN_MASK 0xFFFULL 26 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 27 28 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 29 30 static const char def_pfl_name[] = "default"; 31 32 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 33 int blkaddr, u16 pcifunc); 34 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 35 u16 pcifunc); 36 37 bool is_npc_intf_tx(u8 intf) 38 { 39 return !!(intf & 0x1); 40 } 41 42 bool is_npc_intf_rx(u8 intf) 43 { 44 return !(intf & 0x1); 45 } 46 47 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 48 { 49 struct rvu_hwinfo *hw = rvu->hw; 50 51 return intf < hw->npc_intfs; 52 } 53 54 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 55 { 56 /* Due to a HW issue in these silicon versions, parse nibble enable 57 * configuration has to be identical for both Rx and Tx interfaces. 58 */ 59 if (is_rvu_96xx_B0(rvu)) 60 return nibble_ena; 61 return 0; 62 } 63 64 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 65 { 66 int blkaddr; 67 u64 val = 0; 68 69 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 70 if (blkaddr < 0) 71 return; 72 73 /* Config CPI base for the PKIND */ 74 val = pkind | 1ULL << 62; 75 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 76 } 77 78 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 79 { 80 struct npc_pkind *pkind = &rvu->hw->pkind; 81 u32 map; 82 int i; 83 84 for (i = 0; i < pkind->rsrc.max; i++) { 85 map = pkind->pfchan_map[i]; 86 if (((map >> 16) & 0x3F) == pf) 87 return i; 88 } 89 return -1; 90 } 91 92 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 93 94 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 95 { 96 int pkind, blkaddr; 97 u64 val; 98 99 pkind = rvu_npc_get_pkind(rvu, pf); 100 if (pkind < 0) { 101 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 102 return -EINVAL; 103 } 104 105 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 106 if (blkaddr < 0) { 107 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 108 return -EINVAL; 109 } 110 111 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 112 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 113 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 114 if (enable) 115 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 116 NPC_HW_TSTAMP_OFFSET); 117 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 118 119 return 0; 120 } 121 122 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 123 int nixlf) 124 { 125 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 126 struct rvu *rvu = hw->rvu; 127 int blkaddr = 0, max = 0; 128 struct rvu_block *block; 129 struct rvu_pfvf *pfvf; 130 131 pfvf = rvu_get_pfvf(rvu, pcifunc); 132 /* Given a PF/VF and NIX LF number calculate the unicast mcam 133 * entry index based on the NIX block assigned to the PF/VF. 134 */ 135 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 136 while (blkaddr) { 137 if (pfvf->nix_blkaddr == blkaddr) 138 break; 139 block = &rvu->hw->block[blkaddr]; 140 max += block->lf.max; 141 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 142 } 143 144 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 145 } 146 147 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 148 u16 pcifunc, int nixlf, int type) 149 { 150 int pf = rvu_get_pf(pcifunc); 151 int index; 152 153 /* Check if this is for a PF */ 154 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 155 /* Reserved entries exclude PF0 */ 156 pf--; 157 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 158 /* Broadcast address matching entry should be first so 159 * that the packet can be replicated to all VFs. 160 */ 161 if (type == NIXLF_BCAST_ENTRY) 162 return index; 163 else if (type == NIXLF_ALLMULTI_ENTRY) 164 return index + 1; 165 else if (type == NIXLF_PROMISC_ENTRY) 166 return index + 2; 167 } 168 169 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 170 } 171 172 int npc_get_bank(struct npc_mcam *mcam, int index) 173 { 174 int bank = index / mcam->banksize; 175 176 /* 0,1 & 2,3 banks are combined for this keysize */ 177 if (mcam->keysize == NPC_MCAM_KEY_X2) 178 return bank ? 2 : 0; 179 180 return bank; 181 } 182 183 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 184 int blkaddr, int index) 185 { 186 int bank = npc_get_bank(mcam, index); 187 u64 cfg; 188 189 index &= (mcam->banksize - 1); 190 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 191 return (cfg & 1); 192 } 193 194 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 195 int blkaddr, int index, bool enable) 196 { 197 int bank = npc_get_bank(mcam, index); 198 int actbank = bank; 199 200 index &= (mcam->banksize - 1); 201 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 202 rvu_write64(rvu, blkaddr, 203 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 204 enable ? 1 : 0); 205 } 206 } 207 208 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 209 int blkaddr, int index) 210 { 211 int bank = npc_get_bank(mcam, index); 212 int actbank = bank; 213 214 index &= (mcam->banksize - 1); 215 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 216 rvu_write64(rvu, blkaddr, 217 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 218 rvu_write64(rvu, blkaddr, 219 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 220 221 rvu_write64(rvu, blkaddr, 222 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 223 rvu_write64(rvu, blkaddr, 224 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 225 226 rvu_write64(rvu, blkaddr, 227 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 228 rvu_write64(rvu, blkaddr, 229 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 230 } 231 } 232 233 static void npc_get_keyword(struct mcam_entry *entry, int idx, 234 u64 *cam0, u64 *cam1) 235 { 236 u64 kw_mask = 0x00; 237 238 #define CAM_MASK(n) (BIT_ULL(n) - 1) 239 240 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 241 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 242 * 243 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 244 */ 245 switch (idx) { 246 case 0: 247 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 248 *cam1 = entry->kw[0]; 249 kw_mask = entry->kw_mask[0]; 250 break; 251 case 1: 252 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 253 *cam1 = entry->kw[1] & CAM_MASK(48); 254 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 255 break; 256 case 2: 257 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 258 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 259 */ 260 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 261 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 262 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 263 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 264 break; 265 case 3: 266 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 267 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 268 */ 269 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 270 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 271 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 272 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 273 break; 274 case 4: 275 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 276 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 277 */ 278 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 279 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 280 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 281 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 282 break; 283 case 5: 284 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 285 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 286 */ 287 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 288 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 289 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 290 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 291 break; 292 case 6: 293 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 294 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 295 */ 296 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 297 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 298 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 299 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 300 break; 301 case 7: 302 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 303 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 304 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 305 break; 306 } 307 308 *cam1 &= kw_mask; 309 *cam0 = ~*cam1 & kw_mask; 310 } 311 312 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 313 u64 cam0, u64 cam1) 314 { 315 /* Similar to npc_get_keyword, but fills mcam_entry structure from 316 * CAM registers. 317 */ 318 switch (idx) { 319 case 0: 320 entry->kw[0] = cam1; 321 entry->kw_mask[0] = cam1 ^ cam0; 322 break; 323 case 1: 324 entry->kw[1] = cam1; 325 entry->kw_mask[1] = cam1 ^ cam0; 326 break; 327 case 2: 328 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 329 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 330 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 331 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 332 break; 333 case 3: 334 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 335 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 336 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 337 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 338 break; 339 case 4: 340 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 341 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 342 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 343 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 344 break; 345 case 5: 346 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 347 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 348 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 349 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 350 break; 351 case 6: 352 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 353 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 354 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 355 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 356 break; 357 case 7: 358 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 359 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 360 break; 361 } 362 } 363 364 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 365 int blkaddr, u16 pf_func) 366 { 367 int bank, nixlf, index; 368 369 /* get ucast entry rule entry index */ 370 if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { 371 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n", 372 __func__, pf_func); 373 /* Action 0 is drop */ 374 return 0; 375 } 376 377 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 378 NIXLF_UCAST_ENTRY); 379 bank = npc_get_bank(mcam, index); 380 index &= (mcam->banksize - 1); 381 382 return rvu_read64(rvu, blkaddr, 383 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 384 } 385 386 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 387 int blkaddr, int index, struct mcam_entry *entry, 388 bool *enable) 389 { 390 struct rvu_npc_mcam_rule *rule; 391 u16 owner, target_func; 392 struct rvu_pfvf *pfvf; 393 u64 rx_action; 394 395 owner = mcam->entry2pfvf_map[index]; 396 target_func = (entry->action >> 4) & 0xffff; 397 /* do nothing when target is LBK/PF or owner is not PF */ 398 if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) || 399 (owner & RVU_PFVF_FUNC_MASK) || 400 !(target_func & RVU_PFVF_FUNC_MASK)) 401 return; 402 403 /* save entry2target_pffunc */ 404 pfvf = rvu_get_pfvf(rvu, target_func); 405 mcam->entry2target_pffunc[index] = target_func; 406 407 /* don't enable rule when nixlf not attached or initialized */ 408 if (!(is_nixlf_attached(rvu, target_func) && 409 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 410 *enable = false; 411 412 /* fix up not needed for the rules added by user(ntuple filters) */ 413 list_for_each_entry(rule, &mcam->mcam_rules, list) { 414 if (rule->entry == index) 415 return; 416 } 417 418 /* AF modifies given action iff PF/VF has requested for it */ 419 if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT) 420 return; 421 422 /* copy VF default entry action to the VF mcam entry */ 423 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 424 target_func); 425 if (rx_action) 426 entry->action = rx_action; 427 } 428 429 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 430 int blkaddr, int index, u8 intf, 431 struct mcam_entry *entry, bool enable) 432 { 433 int bank = npc_get_bank(mcam, index); 434 int kw = 0, actbank, actindex; 435 u8 tx_intf_mask = ~intf & 0x3; 436 u8 tx_intf = intf; 437 u64 cam0, cam1; 438 439 actbank = bank; /* Save bank id, to set action later on */ 440 actindex = index; 441 index &= (mcam->banksize - 1); 442 443 /* Disable before mcam entry update */ 444 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 445 446 /* Clear mcam entry to avoid writes being suppressed by NPC */ 447 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 448 449 /* CAM1 takes the comparison value and 450 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 451 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 452 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 453 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 454 */ 455 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 456 /* Interface should be set in all banks */ 457 if (is_npc_intf_tx(intf)) { 458 /* Last bit must be set and rest don't care 459 * for TX interfaces 460 */ 461 tx_intf_mask = 0x1; 462 tx_intf = intf & tx_intf_mask; 463 tx_intf_mask = ~tx_intf & tx_intf_mask; 464 } 465 466 rvu_write64(rvu, blkaddr, 467 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 468 tx_intf); 469 rvu_write64(rvu, blkaddr, 470 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 471 tx_intf_mask); 472 473 /* Set the match key */ 474 npc_get_keyword(entry, kw, &cam0, &cam1); 475 rvu_write64(rvu, blkaddr, 476 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 477 rvu_write64(rvu, blkaddr, 478 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 479 480 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 481 rvu_write64(rvu, blkaddr, 482 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 483 rvu_write64(rvu, blkaddr, 484 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 485 } 486 487 /* PF installing VF rule */ 488 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 489 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 490 491 /* Set 'action' */ 492 rvu_write64(rvu, blkaddr, 493 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 494 495 /* Set TAG 'action' */ 496 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 497 entry->vtag_action); 498 499 /* Enable the entry */ 500 if (enable) 501 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 502 } 503 504 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 505 int blkaddr, u16 src, 506 struct mcam_entry *entry, u8 *intf, u8 *ena) 507 { 508 int sbank = npc_get_bank(mcam, src); 509 int bank, kw = 0; 510 u64 cam0, cam1; 511 512 src &= (mcam->banksize - 1); 513 bank = sbank; 514 515 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 516 cam1 = rvu_read64(rvu, blkaddr, 517 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 518 cam0 = rvu_read64(rvu, blkaddr, 519 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 520 npc_fill_entryword(entry, kw, cam0, cam1); 521 522 cam1 = rvu_read64(rvu, blkaddr, 523 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 524 cam0 = rvu_read64(rvu, blkaddr, 525 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 526 npc_fill_entryword(entry, kw + 1, cam0, cam1); 527 } 528 529 entry->action = rvu_read64(rvu, blkaddr, 530 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 531 entry->vtag_action = 532 rvu_read64(rvu, blkaddr, 533 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 534 *intf = rvu_read64(rvu, blkaddr, 535 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 536 *ena = rvu_read64(rvu, blkaddr, 537 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 538 } 539 540 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 541 int blkaddr, u16 src, u16 dest) 542 { 543 int dbank = npc_get_bank(mcam, dest); 544 int sbank = npc_get_bank(mcam, src); 545 u64 cfg, sreg, dreg; 546 int bank, i; 547 548 src &= (mcam->banksize - 1); 549 dest &= (mcam->banksize - 1); 550 551 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 552 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 553 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 554 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 555 for (i = 0; i < 6; i++) { 556 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 557 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 558 } 559 } 560 561 /* Copy action */ 562 cfg = rvu_read64(rvu, blkaddr, 563 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 564 rvu_write64(rvu, blkaddr, 565 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 566 567 /* Copy TAG action */ 568 cfg = rvu_read64(rvu, blkaddr, 569 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 570 rvu_write64(rvu, blkaddr, 571 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 572 573 /* Enable or disable */ 574 cfg = rvu_read64(rvu, blkaddr, 575 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 576 rvu_write64(rvu, blkaddr, 577 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 578 } 579 580 u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 581 int blkaddr, int index) 582 { 583 int bank = npc_get_bank(mcam, index); 584 585 index &= (mcam->banksize - 1); 586 return rvu_read64(rvu, blkaddr, 587 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 588 } 589 590 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 591 int blkaddr, int index, u64 cfg) 592 { 593 int bank = npc_get_bank(mcam, index); 594 595 index &= (mcam->banksize - 1); 596 return rvu_write64(rvu, blkaddr, 597 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg); 598 } 599 600 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 601 int nixlf, u64 chan, u8 *mac_addr) 602 { 603 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 604 struct npc_install_flow_req req = { 0 }; 605 struct npc_install_flow_rsp rsp = { 0 }; 606 struct npc_mcam *mcam = &rvu->hw->mcam; 607 struct nix_rx_action action = { 0 }; 608 int blkaddr, index; 609 610 /* AF's and SDP VFs work in promiscuous mode */ 611 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc)) 612 return; 613 614 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 615 if (blkaddr < 0) 616 return; 617 618 /* Ucast rule should not be installed if DMAC 619 * extraction is not supported by the profile. 620 */ 621 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf)) 622 return; 623 624 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 625 nixlf, NIXLF_UCAST_ENTRY); 626 627 /* Don't change the action if entry is already enabled 628 * Otherwise RSS action may get overwritten. 629 */ 630 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 631 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 632 blkaddr, index); 633 } else { 634 action.op = NIX_RX_ACTIONOP_UCAST; 635 action.pf_func = pcifunc; 636 } 637 638 req.default_rule = 1; 639 ether_addr_copy(req.packet.dmac, mac_addr); 640 eth_broadcast_addr((u8 *)&req.mask.dmac); 641 req.features = BIT_ULL(NPC_DMAC); 642 req.channel = chan; 643 req.chan_mask = 0xFFFU; 644 req.intf = pfvf->nix_rx_intf; 645 req.op = action.op; 646 req.hdr.pcifunc = 0; /* AF is requester */ 647 req.vf = action.pf_func; 648 req.index = action.index; 649 req.match_id = action.match_id; 650 req.flow_key_alg = action.flow_key_alg; 651 652 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 653 } 654 655 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 656 int nixlf, u64 chan, u8 chan_cnt) 657 { 658 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 659 struct npc_install_flow_req req = { 0 }; 660 struct npc_install_flow_rsp rsp = { 0 }; 661 struct npc_mcam *mcam = &rvu->hw->mcam; 662 struct rvu_hwinfo *hw = rvu->hw; 663 int blkaddr, ucast_idx, index; 664 struct nix_rx_action action = { 0 }; 665 u64 relaxed_mask; 666 u8 flow_key_alg; 667 668 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 669 return; 670 671 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 672 if (blkaddr < 0) 673 return; 674 675 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 676 nixlf, NIXLF_PROMISC_ENTRY); 677 678 if (is_cgx_vf(rvu, pcifunc)) 679 index = npc_get_nixlf_mcam_index(mcam, 680 pcifunc & ~RVU_PFVF_FUNC_MASK, 681 nixlf, NIXLF_PROMISC_ENTRY); 682 683 /* If the corresponding PF's ucast action is RSS, 684 * use the same action for promisc also 685 */ 686 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 687 nixlf, NIXLF_UCAST_ENTRY); 688 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 689 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 690 blkaddr, ucast_idx); 691 692 if (action.op != NIX_RX_ACTIONOP_RSS) { 693 *(u64 *)&action = 0; 694 action.op = NIX_RX_ACTIONOP_UCAST; 695 } 696 697 flow_key_alg = action.flow_key_alg; 698 699 /* RX_ACTION set to MCAST for CGX PF's */ 700 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 701 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { 702 *(u64 *)&action = 0; 703 action.op = NIX_RX_ACTIONOP_MCAST; 704 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 705 action.index = pfvf->promisc_mce_idx; 706 } 707 708 /* For cn10k the upper two bits of the channel number are 709 * cpt channel number. with masking out these bits in the 710 * mcam entry, same entry used for NIX will allow packets 711 * received from cpt for parsing. 712 */ 713 if (!is_rvu_otx2(rvu)) { 714 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 715 } else { 716 req.chan_mask = 0xFFFU; 717 } 718 719 if (chan_cnt > 1) { 720 if (!is_power_of_2(chan_cnt)) { 721 dev_err(rvu->dev, 722 "%s: channel count more than 1, must be power of 2\n", __func__); 723 return; 724 } 725 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 726 ilog2(chan_cnt)); 727 req.chan_mask &= relaxed_mask; 728 } 729 730 req.channel = chan; 731 req.intf = pfvf->nix_rx_intf; 732 req.entry = index; 733 req.op = action.op; 734 req.hdr.pcifunc = 0; /* AF is requester */ 735 req.vf = pcifunc; 736 req.index = action.index; 737 req.match_id = action.match_id; 738 req.flow_key_alg = flow_key_alg; 739 740 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 741 } 742 743 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 744 int nixlf, bool enable) 745 { 746 struct npc_mcam *mcam = &rvu->hw->mcam; 747 int blkaddr, index; 748 749 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 750 if (blkaddr < 0) 751 return; 752 753 /* Get 'pcifunc' of PF device */ 754 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 755 756 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 757 nixlf, NIXLF_PROMISC_ENTRY); 758 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 759 } 760 761 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 762 int nixlf, u64 chan) 763 { 764 struct rvu_pfvf *pfvf; 765 struct npc_install_flow_req req = { 0 }; 766 struct npc_install_flow_rsp rsp = { 0 }; 767 struct npc_mcam *mcam = &rvu->hw->mcam; 768 struct rvu_hwinfo *hw = rvu->hw; 769 int blkaddr, index; 770 771 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 772 if (blkaddr < 0) 773 return; 774 775 /* Skip LBK VFs */ 776 if (is_lbk_vf(rvu, pcifunc)) 777 return; 778 779 /* If pkt replication is not supported, 780 * then only PF is allowed to add a bcast match entry. 781 */ 782 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 783 return; 784 785 /* Get 'pcifunc' of PF device */ 786 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 787 pfvf = rvu_get_pfvf(rvu, pcifunc); 788 789 /* Bcast rule should not be installed if both DMAC 790 * and LXMB extraction is not supported by the profile. 791 */ 792 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 793 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 794 return; 795 796 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 797 nixlf, NIXLF_BCAST_ENTRY); 798 799 if (!hw->cap.nix_rx_multicast) { 800 /* Early silicon doesn't support pkt replication, 801 * so install entry with UCAST action, so that PF 802 * receives all broadcast packets. 803 */ 804 req.op = NIX_RX_ACTIONOP_UCAST; 805 } else { 806 req.op = NIX_RX_ACTIONOP_MCAST; 807 req.index = pfvf->bcast_mce_idx; 808 } 809 810 eth_broadcast_addr((u8 *)&req.packet.dmac); 811 eth_broadcast_addr((u8 *)&req.mask.dmac); 812 req.features = BIT_ULL(NPC_DMAC); 813 req.channel = chan; 814 req.chan_mask = 0xFFFU; 815 req.intf = pfvf->nix_rx_intf; 816 req.entry = index; 817 req.hdr.pcifunc = 0; /* AF is requester */ 818 req.vf = pcifunc; 819 820 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 821 } 822 823 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 824 bool enable) 825 { 826 struct npc_mcam *mcam = &rvu->hw->mcam; 827 int blkaddr, index; 828 829 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 830 if (blkaddr < 0) 831 return; 832 833 /* Get 'pcifunc' of PF device */ 834 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 835 836 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 837 NIXLF_BCAST_ENTRY); 838 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 839 } 840 841 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 842 u64 chan) 843 { 844 struct npc_install_flow_req req = { 0 }; 845 struct npc_install_flow_rsp rsp = { 0 }; 846 struct npc_mcam *mcam = &rvu->hw->mcam; 847 struct rvu_hwinfo *hw = rvu->hw; 848 int blkaddr, ucast_idx, index; 849 u8 mac_addr[ETH_ALEN] = { 0 }; 850 struct nix_rx_action action = { 0 }; 851 struct rvu_pfvf *pfvf; 852 u8 flow_key_alg; 853 u16 vf_func; 854 855 /* Only CGX PF/VF can add allmulticast entry */ 856 if (is_lbk_vf(rvu, pcifunc) && is_sdp_vf(rvu, pcifunc)) 857 return; 858 859 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 860 if (blkaddr < 0) 861 return; 862 863 /* Get 'pcifunc' of PF device */ 864 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 865 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 866 pfvf = rvu_get_pfvf(rvu, pcifunc); 867 868 /* Mcast rule should not be installed if both DMAC 869 * and LXMB extraction is not supported by the profile. 870 */ 871 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) && 872 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf)) 873 return; 874 875 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 876 nixlf, NIXLF_ALLMULTI_ENTRY); 877 878 /* If the corresponding PF's ucast action is RSS, 879 * use the same action for multicast entry also 880 */ 881 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 882 nixlf, NIXLF_UCAST_ENTRY); 883 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 884 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 885 blkaddr, ucast_idx); 886 887 flow_key_alg = action.flow_key_alg; 888 if (action.op != NIX_RX_ACTIONOP_RSS) { 889 *(u64 *)&action = 0; 890 action.op = NIX_RX_ACTIONOP_UCAST; 891 action.pf_func = pcifunc; 892 } 893 894 /* RX_ACTION set to MCAST for CGX PF's */ 895 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 896 *(u64 *)&action = 0; 897 action.op = NIX_RX_ACTIONOP_MCAST; 898 action.index = pfvf->mcast_mce_idx; 899 } 900 901 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 902 ether_addr_copy(req.packet.dmac, mac_addr); 903 ether_addr_copy(req.mask.dmac, mac_addr); 904 req.features = BIT_ULL(NPC_DMAC); 905 906 /* For cn10k the upper two bits of the channel number are 907 * cpt channel number. with masking out these bits in the 908 * mcam entry, same entry used for NIX will allow packets 909 * received from cpt for parsing. 910 */ 911 if (!is_rvu_otx2(rvu)) 912 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 913 else 914 req.chan_mask = 0xFFFU; 915 916 req.channel = chan; 917 req.intf = pfvf->nix_rx_intf; 918 req.entry = index; 919 req.op = action.op; 920 req.hdr.pcifunc = 0; /* AF is requester */ 921 req.vf = pcifunc | vf_func; 922 req.index = action.index; 923 req.match_id = action.match_id; 924 req.flow_key_alg = flow_key_alg; 925 926 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 927 } 928 929 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 930 bool enable) 931 { 932 struct npc_mcam *mcam = &rvu->hw->mcam; 933 int blkaddr, index; 934 935 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 936 if (blkaddr < 0) 937 return; 938 939 /* Get 'pcifunc' of PF device */ 940 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 941 942 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 943 NIXLF_ALLMULTI_ENTRY); 944 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 945 } 946 947 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 948 int blkaddr, u16 pcifunc, u64 rx_action) 949 { 950 int actindex, index, bank, entry; 951 struct rvu_npc_mcam_rule *rule; 952 bool enable, update; 953 954 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 955 return; 956 957 mutex_lock(&mcam->lock); 958 for (index = 0; index < mcam->bmap_entries; index++) { 959 if (mcam->entry2target_pffunc[index] == pcifunc) { 960 update = true; 961 /* update not needed for the rules added via ntuple filters */ 962 list_for_each_entry(rule, &mcam->mcam_rules, list) { 963 if (rule->entry == index) 964 update = false; 965 } 966 if (!update) 967 continue; 968 bank = npc_get_bank(mcam, index); 969 actindex = index; 970 entry = index & (mcam->banksize - 1); 971 972 /* read vf flow entry enable status */ 973 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 974 actindex); 975 /* disable before mcam entry update */ 976 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 977 false); 978 /* update 'action' */ 979 rvu_write64(rvu, blkaddr, 980 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 981 rx_action); 982 if (enable) 983 npc_enable_mcam_entry(rvu, mcam, blkaddr, 984 actindex, true); 985 } 986 } 987 mutex_unlock(&mcam->lock); 988 } 989 990 static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action, 991 struct rvu_pfvf *pfvf, int mcam_index, int blkaddr, 992 int alg_idx) 993 994 { 995 struct npc_mcam *mcam = &rvu->hw->mcam; 996 struct rvu_hwinfo *hw = rvu->hw; 997 int bank, op_rss; 998 999 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) 1000 return; 1001 1002 op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list); 1003 1004 bank = npc_get_bank(mcam, mcam_index); 1005 mcam_index &= (mcam->banksize - 1); 1006 1007 /* If Rx action is MCAST update only RSS algorithm index */ 1008 if (!op_rss) { 1009 *(u64 *)&action = rvu_read64(rvu, blkaddr, 1010 NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank)); 1011 1012 action.flow_key_alg = alg_idx; 1013 } 1014 rvu_write64(rvu, blkaddr, 1015 NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action); 1016 } 1017 1018 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 1019 int group, int alg_idx, int mcam_index) 1020 { 1021 struct npc_mcam *mcam = &rvu->hw->mcam; 1022 struct nix_rx_action action; 1023 int blkaddr, index, bank; 1024 struct rvu_pfvf *pfvf; 1025 1026 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1027 if (blkaddr < 0) 1028 return; 1029 1030 /* Check if this is for reserved default entry */ 1031 if (mcam_index < 0) { 1032 if (group != DEFAULT_RSS_CONTEXT_GROUP) 1033 return; 1034 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1035 nixlf, NIXLF_UCAST_ENTRY); 1036 } else { 1037 /* TODO: validate this mcam index */ 1038 index = mcam_index; 1039 } 1040 1041 if (index >= mcam->total_entries) 1042 return; 1043 1044 bank = npc_get_bank(mcam, index); 1045 index &= (mcam->banksize - 1); 1046 1047 *(u64 *)&action = rvu_read64(rvu, blkaddr, 1048 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 1049 /* Ignore if no action was set earlier */ 1050 if (!*(u64 *)&action) 1051 return; 1052 1053 action.op = NIX_RX_ACTIONOP_RSS; 1054 action.pf_func = pcifunc; 1055 action.index = group; 1056 action.flow_key_alg = alg_idx; 1057 1058 rvu_write64(rvu, blkaddr, 1059 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 1060 1061 /* update the VF flow rule action with the VF default entry action */ 1062 if (mcam_index < 0) 1063 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1064 *(u64 *)&action); 1065 1066 /* update the action change in default rule */ 1067 pfvf = rvu_get_pfvf(rvu, pcifunc); 1068 if (pfvf->def_ucast_rule) 1069 pfvf->def_ucast_rule->rx_action = action; 1070 1071 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1072 nixlf, NIXLF_PROMISC_ENTRY); 1073 1074 /* If PF's promiscuous entry is enabled, 1075 * Set RSS action for that entry as well 1076 */ 1077 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, 1078 alg_idx); 1079 1080 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1081 nixlf, NIXLF_ALLMULTI_ENTRY); 1082 /* If PF's allmulti entry is enabled, 1083 * Set RSS action for that entry as well 1084 */ 1085 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, 1086 alg_idx); 1087 } 1088 1089 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1090 int nixlf, int type, bool enable) 1091 { 1092 struct npc_mcam *mcam = &rvu->hw->mcam; 1093 struct rvu_hwinfo *hw = rvu->hw; 1094 struct nix_mce_list *mce_list; 1095 int index, blkaddr, mce_idx; 1096 struct rvu_pfvf *pfvf; 1097 1098 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1099 if (blkaddr < 0) 1100 return; 1101 1102 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1103 nixlf, type); 1104 1105 /* disable MCAM entry when packet replication is not supported by hw */ 1106 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1107 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1108 return; 1109 } 1110 1111 /* return incase mce list is not enabled */ 1112 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1113 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1114 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1115 return; 1116 1117 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1118 1119 nix_update_mce_list(rvu, pcifunc, mce_list, 1120 mce_idx, index, enable); 1121 if (enable) 1122 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1123 } 1124 1125 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1126 int nixlf, bool enable) 1127 { 1128 struct npc_mcam *mcam = &rvu->hw->mcam; 1129 int index, blkaddr; 1130 1131 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1132 if (blkaddr < 0) 1133 return; 1134 1135 /* Ucast MCAM match entry of this PF/VF */ 1136 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1137 nixlf, NIXLF_UCAST_ENTRY); 1138 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1139 1140 /* Nothing to do for VFs, on platforms where pkt replication 1141 * is not supported 1142 */ 1143 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1144 return; 1145 1146 /* add/delete pf_func to broadcast MCE list */ 1147 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1148 NIXLF_BCAST_ENTRY, enable); 1149 } 1150 1151 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1152 { 1153 if (nixlf < 0) 1154 return; 1155 1156 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1157 1158 /* Delete multicast and promisc MCAM entries */ 1159 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1160 NIXLF_ALLMULTI_ENTRY, false); 1161 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1162 NIXLF_PROMISC_ENTRY, false); 1163 } 1164 1165 bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable) 1166 { 1167 int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1168 struct npc_mcam *mcam = &rvu->hw->mcam; 1169 struct rvu_npc_mcam_rule *rule, *tmp; 1170 1171 mutex_lock(&mcam->lock); 1172 1173 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1174 if (rule->intf != intf) 1175 continue; 1176 1177 if (rule->entry != entry) 1178 continue; 1179 1180 rule->enable = enable; 1181 mutex_unlock(&mcam->lock); 1182 1183 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1184 entry, enable); 1185 1186 return true; 1187 } 1188 1189 mutex_unlock(&mcam->lock); 1190 return false; 1191 } 1192 1193 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1194 { 1195 if (nixlf < 0) 1196 return; 1197 1198 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1199 * in set_rx_mode mbox handler. 1200 */ 1201 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1202 } 1203 1204 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1205 { 1206 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1207 struct npc_mcam *mcam = &rvu->hw->mcam; 1208 struct rvu_npc_mcam_rule *rule, *tmp; 1209 int blkaddr; 1210 1211 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1212 if (blkaddr < 0) 1213 return; 1214 1215 mutex_lock(&mcam->lock); 1216 1217 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1218 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1219 if (is_npc_intf_rx(rule->intf) && 1220 rule->rx_action.pf_func == pcifunc && 1221 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1222 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1223 rule->entry, false); 1224 rule->enable = false; 1225 /* Indicate that default rule is disabled */ 1226 if (rule->default_rule) { 1227 pfvf->def_ucast_rule = NULL; 1228 list_del(&rule->list); 1229 kfree(rule); 1230 } 1231 } 1232 } 1233 1234 mutex_unlock(&mcam->lock); 1235 1236 npc_mcam_disable_flows(rvu, pcifunc); 1237 1238 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1239 } 1240 1241 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1242 { 1243 struct npc_mcam *mcam = &rvu->hw->mcam; 1244 struct rvu_npc_mcam_rule *rule, *tmp; 1245 int blkaddr; 1246 1247 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1248 if (blkaddr < 0) 1249 return; 1250 1251 mutex_lock(&mcam->lock); 1252 1253 /* Free all MCAM entries owned by this 'pcifunc' */ 1254 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1255 1256 /* Free all MCAM counters owned by this 'pcifunc' */ 1257 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1258 1259 /* Delete MCAM entries owned by this 'pcifunc' */ 1260 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1261 if (rule->owner == pcifunc && !rule->default_rule) { 1262 list_del(&rule->list); 1263 kfree(rule); 1264 } 1265 } 1266 1267 mutex_unlock(&mcam->lock); 1268 1269 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1270 } 1271 1272 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1273 struct npc_mcam_kex *mkex, u8 intf) 1274 { 1275 int lid, lt, ld, fl; 1276 1277 if (is_npc_intf_tx(intf)) 1278 return; 1279 1280 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1281 mkex->keyx_cfg[NIX_INTF_RX]); 1282 1283 /* Program LDATA */ 1284 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1285 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1286 for (ld = 0; ld < NPC_MAX_LD; ld++) 1287 SET_KEX_LD(intf, lid, lt, ld, 1288 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1289 [lid][lt][ld]); 1290 } 1291 } 1292 /* Program LFLAGS */ 1293 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1294 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1295 SET_KEX_LDFLAGS(intf, ld, fl, 1296 mkex->intf_ld_flags[NIX_INTF_RX] 1297 [ld][fl]); 1298 } 1299 } 1300 1301 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1302 struct npc_mcam_kex *mkex, u8 intf) 1303 { 1304 int lid, lt, ld, fl; 1305 1306 if (is_npc_intf_rx(intf)) 1307 return; 1308 1309 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1310 mkex->keyx_cfg[NIX_INTF_TX]); 1311 1312 /* Program LDATA */ 1313 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1314 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1315 for (ld = 0; ld < NPC_MAX_LD; ld++) 1316 SET_KEX_LD(intf, lid, lt, ld, 1317 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1318 [lid][lt][ld]); 1319 } 1320 } 1321 /* Program LFLAGS */ 1322 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1323 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1324 SET_KEX_LDFLAGS(intf, ld, fl, 1325 mkex->intf_ld_flags[NIX_INTF_TX] 1326 [ld][fl]); 1327 } 1328 } 1329 1330 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1331 struct npc_mcam_kex *mkex) 1332 { 1333 struct rvu_hwinfo *hw = rvu->hw; 1334 u8 intf; 1335 int ld; 1336 1337 for (ld = 0; ld < NPC_MAX_LD; ld++) 1338 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1339 mkex->kex_ld_flags[ld]); 1340 1341 for (intf = 0; intf < hw->npc_intfs; intf++) { 1342 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1343 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1344 } 1345 1346 /* Programme mkex hash profile */ 1347 npc_program_mkex_hash(rvu, blkaddr); 1348 } 1349 1350 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1351 u64 *size) 1352 { 1353 u64 prfl_addr, prfl_sz; 1354 1355 if (!rvu->fwdata) 1356 return -EINVAL; 1357 1358 prfl_addr = rvu->fwdata->mcam_addr; 1359 prfl_sz = rvu->fwdata->mcam_sz; 1360 1361 if (!prfl_addr || !prfl_sz) 1362 return -EINVAL; 1363 1364 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1365 if (!(*prfl_img_addr)) 1366 return -ENOMEM; 1367 1368 *size = prfl_sz; 1369 1370 return 0; 1371 } 1372 1373 /* strtoull of "mkexprof" with base:36 */ 1374 #define MKEX_END_SIGN 0xdeadbeef 1375 1376 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1377 const char *mkex_profile) 1378 { 1379 struct device *dev = &rvu->pdev->dev; 1380 struct npc_mcam_kex *mcam_kex; 1381 void __iomem *mkex_prfl_addr = NULL; 1382 u64 prfl_sz; 1383 int ret; 1384 1385 /* If user not selected mkex profile */ 1386 if (rvu->kpu_fwdata_sz || 1387 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1388 goto program_mkex; 1389 1390 /* Setting up the mapping for mkex profile image */ 1391 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1392 if (ret < 0) 1393 goto program_mkex; 1394 1395 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1396 1397 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1398 /* Compare with mkex mod_param name string */ 1399 if (mcam_kex->mkex_sign == MKEX_SIGN && 1400 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1401 /* Due to an errata (35786) in A0/B0 pass silicon, 1402 * parse nibble enable configuration has to be 1403 * identical for both Rx and Tx interfaces. 1404 */ 1405 if (!is_rvu_96xx_B0(rvu) || 1406 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1407 rvu->kpu.mkex = mcam_kex; 1408 goto program_mkex; 1409 } 1410 1411 mcam_kex++; 1412 prfl_sz -= sizeof(struct npc_mcam_kex); 1413 } 1414 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1415 1416 program_mkex: 1417 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1418 /* Program selected mkex profile */ 1419 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1420 if (mkex_prfl_addr) 1421 iounmap(mkex_prfl_addr); 1422 } 1423 1424 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1425 const struct npc_kpu_profile_action *kpuaction, 1426 int kpu, int entry, bool pkind) 1427 { 1428 struct npc_kpu_action0 action0 = {0}; 1429 struct npc_kpu_action1 action1 = {0}; 1430 u64 reg; 1431 1432 action1.errlev = kpuaction->errlev; 1433 action1.errcode = kpuaction->errcode; 1434 action1.dp0_offset = kpuaction->dp0_offset; 1435 action1.dp1_offset = kpuaction->dp1_offset; 1436 action1.dp2_offset = kpuaction->dp2_offset; 1437 1438 if (pkind) 1439 reg = NPC_AF_PKINDX_ACTION1(entry); 1440 else 1441 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1442 1443 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1444 1445 action0.byp_count = kpuaction->bypass_count; 1446 action0.capture_ena = kpuaction->cap_ena; 1447 action0.parse_done = kpuaction->parse_done; 1448 action0.next_state = kpuaction->next_state; 1449 action0.capture_lid = kpuaction->lid; 1450 action0.capture_ltype = kpuaction->ltype; 1451 action0.capture_flags = kpuaction->flags; 1452 action0.ptr_advance = kpuaction->ptr_advance; 1453 action0.var_len_offset = kpuaction->offset; 1454 action0.var_len_mask = kpuaction->mask; 1455 action0.var_len_right = kpuaction->right; 1456 action0.var_len_shift = kpuaction->shift; 1457 1458 if (pkind) 1459 reg = NPC_AF_PKINDX_ACTION0(entry); 1460 else 1461 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1462 1463 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1464 } 1465 1466 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1467 const struct npc_kpu_profile_cam *kpucam, 1468 int kpu, int entry) 1469 { 1470 struct npc_kpu_cam cam0 = {0}; 1471 struct npc_kpu_cam cam1 = {0}; 1472 1473 cam1.state = kpucam->state & kpucam->state_mask; 1474 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1475 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1476 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1477 1478 cam0.state = ~kpucam->state & kpucam->state_mask; 1479 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1480 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1481 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1482 1483 rvu_write64(rvu, blkaddr, 1484 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1485 rvu_write64(rvu, blkaddr, 1486 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1487 } 1488 1489 static inline u64 enable_mask(int count) 1490 { 1491 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1492 } 1493 1494 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1495 const struct npc_kpu_profile *profile) 1496 { 1497 int entry, num_entries, max_entries; 1498 u64 entry_mask; 1499 1500 if (profile->cam_entries != profile->action_entries) { 1501 dev_err(rvu->dev, 1502 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1503 kpu, profile->cam_entries, profile->action_entries); 1504 } 1505 1506 max_entries = rvu->hw->npc_kpu_entries; 1507 1508 /* Program CAM match entries for previous KPU extracted data */ 1509 num_entries = min_t(int, profile->cam_entries, max_entries); 1510 for (entry = 0; entry < num_entries; entry++) 1511 npc_config_kpucam(rvu, blkaddr, 1512 &profile->cam[entry], kpu, entry); 1513 1514 /* Program this KPU's actions */ 1515 num_entries = min_t(int, profile->action_entries, max_entries); 1516 for (entry = 0; entry < num_entries; entry++) 1517 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1518 kpu, entry, false); 1519 1520 /* Enable all programmed entries */ 1521 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1522 entry_mask = enable_mask(num_entries); 1523 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1524 if (!rvu->kpu.custom) 1525 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1526 rvu_write64(rvu, blkaddr, 1527 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1528 if (num_entries > 64) { 1529 rvu_write64(rvu, blkaddr, 1530 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1531 enable_mask(num_entries - 64)); 1532 } 1533 1534 /* Enable this KPU */ 1535 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1536 } 1537 1538 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1539 { 1540 profile->custom = 0; 1541 profile->name = def_pfl_name; 1542 profile->version = NPC_KPU_PROFILE_VER; 1543 profile->ikpu = ikpu_action_entries; 1544 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1545 profile->kpu = npc_kpu_profiles; 1546 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1547 profile->lt_def = &npc_lt_defaults; 1548 profile->mkex = &npc_mkex_default; 1549 profile->mkex_hash = &npc_mkex_hash_default; 1550 1551 return 0; 1552 } 1553 1554 static int npc_apply_custom_kpu(struct rvu *rvu, 1555 struct npc_kpu_profile_adapter *profile) 1556 { 1557 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1558 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1559 struct npc_kpu_profile_action *action; 1560 struct npc_kpu_profile_cam *cam; 1561 struct npc_kpu_fwdata *fw_kpu; 1562 int entries; 1563 u16 kpu, entry; 1564 1565 if (rvu->kpu_fwdata_sz < hdr_sz) { 1566 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1567 return -EINVAL; 1568 } 1569 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1570 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1571 fw->signature); 1572 return -EINVAL; 1573 } 1574 /* Verify if the using known profile structure */ 1575 if (NPC_KPU_VER_MAJ(profile->version) > 1576 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1577 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1578 NPC_KPU_VER_MAJ(profile->version), 1579 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1580 return -EINVAL; 1581 } 1582 /* Verify if profile is aligned with the required kernel changes */ 1583 if (NPC_KPU_VER_MIN(profile->version) < 1584 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1585 dev_warn(rvu->dev, 1586 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1587 NPC_KPU_VER_MAJ(profile->version), 1588 NPC_KPU_VER_MIN(profile->version), 1589 NPC_KPU_VER_PATCH(profile->version), 1590 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1591 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1592 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1593 return -EINVAL; 1594 } 1595 /* Verify if profile fits the HW */ 1596 if (fw->kpus > profile->kpus) { 1597 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1598 profile->kpus); 1599 return -EINVAL; 1600 } 1601 1602 profile->custom = 1; 1603 profile->name = fw->name; 1604 profile->version = le64_to_cpu(fw->version); 1605 profile->mkex = &fw->mkex; 1606 profile->lt_def = &fw->lt_def; 1607 1608 for (kpu = 0; kpu < fw->kpus; kpu++) { 1609 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1610 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1611 dev_warn(rvu->dev, 1612 "Too many custom entries on KPU%d: %d > %d\n", 1613 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1614 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1615 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1616 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1617 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1618 offset += fw_kpu->entries * sizeof(*action); 1619 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1620 dev_warn(rvu->dev, 1621 "Profile size mismatch on KPU%i parsing.\n", 1622 kpu + 1); 1623 return -EINVAL; 1624 } 1625 for (entry = 0; entry < entries; entry++) { 1626 profile->kpu[kpu].cam[entry] = cam[entry]; 1627 profile->kpu[kpu].action[entry] = action[entry]; 1628 } 1629 } 1630 1631 return 0; 1632 } 1633 1634 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1635 u64 prfl_sz, const char *kpu_profile) 1636 { 1637 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1638 int rc = -EINVAL; 1639 1640 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1641 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1642 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1643 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1644 kpu_profile); 1645 rvu->kpu_fwdata = kpu_data; 1646 rvu->kpu_fwdata_sz = prfl_sz; 1647 rvu->kpu_prfl_addr = prfl_addr; 1648 rc = 0; 1649 } 1650 1651 return rc; 1652 } 1653 1654 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1655 const char *kpu_profile) 1656 { 1657 struct npc_coalesced_kpu_prfl *img_data = NULL; 1658 int i = 0, rc = -EINVAL; 1659 void __iomem *kpu_prfl_addr; 1660 u32 offset; 1661 1662 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1663 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1664 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1665 /* Loaded profile is a single KPU profile. */ 1666 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1667 prfl_sz, kpu_profile); 1668 goto done; 1669 } 1670 1671 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1672 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1673 (img_data->num_prfl * sizeof(uint16_t)); 1674 /* Check if mapped image is coalesced image. */ 1675 while (i < img_data->num_prfl) { 1676 /* Profile image offsets are rounded up to next 8 multiple.*/ 1677 offset = ALIGN_8B_CEIL(offset); 1678 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1679 offset); 1680 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1681 img_data->prfl_sz[i], kpu_profile); 1682 if (!rc) 1683 break; 1684 /* Calculating offset of profile image based on profile size.*/ 1685 offset += img_data->prfl_sz[i]; 1686 i++; 1687 } 1688 done: 1689 return rc; 1690 } 1691 1692 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1693 { 1694 int ret = -EINVAL; 1695 u64 prfl_sz; 1696 1697 /* Setting up the mapping for NPC profile image */ 1698 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1699 if (ret < 0) 1700 goto done; 1701 1702 /* Detect if profile is coalesced or single KPU profile and load */ 1703 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1704 if (ret == 0) 1705 goto done; 1706 1707 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1708 if (rvu->kpu_prfl_addr) { 1709 iounmap(rvu->kpu_prfl_addr); 1710 rvu->kpu_prfl_addr = NULL; 1711 rvu->kpu_fwdata_sz = 0; 1712 rvu->kpu_fwdata = NULL; 1713 } 1714 1715 done: 1716 return ret; 1717 } 1718 1719 static void npc_load_kpu_profile(struct rvu *rvu) 1720 { 1721 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1722 const char *kpu_profile = rvu->kpu_pfl_name; 1723 const struct firmware *fw = NULL; 1724 bool retry_fwdb = false; 1725 1726 /* If user not specified profile customization */ 1727 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1728 goto revert_to_default; 1729 /* First prepare default KPU, then we'll customize top entries. */ 1730 npc_prepare_default_kpu(profile); 1731 1732 /* Order of preceedence for load loading NPC profile (high to low) 1733 * Firmware binary in filesystem. 1734 * Firmware database method. 1735 * Default KPU profile. 1736 */ 1737 if (!request_firmware_direct(&fw, kpu_profile, rvu->dev)) { 1738 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1739 kpu_profile); 1740 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1741 if (rvu->kpu_fwdata) { 1742 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1743 rvu->kpu_fwdata_sz = fw->size; 1744 } 1745 release_firmware(fw); 1746 retry_fwdb = true; 1747 goto program_kpu; 1748 } 1749 1750 load_image_fwdb: 1751 /* Loading the KPU profile using firmware database */ 1752 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1753 goto revert_to_default; 1754 1755 program_kpu: 1756 /* Apply profile customization if firmware was loaded. */ 1757 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1758 /* If image from firmware filesystem fails to load or invalid 1759 * retry with firmware database method. 1760 */ 1761 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1762 /* Loading image from firmware database failed. */ 1763 if (rvu->kpu_prfl_addr) { 1764 iounmap(rvu->kpu_prfl_addr); 1765 rvu->kpu_prfl_addr = NULL; 1766 } else { 1767 kfree(rvu->kpu_fwdata); 1768 } 1769 rvu->kpu_fwdata = NULL; 1770 rvu->kpu_fwdata_sz = 0; 1771 if (retry_fwdb) { 1772 retry_fwdb = false; 1773 goto load_image_fwdb; 1774 } 1775 } 1776 1777 dev_warn(rvu->dev, 1778 "Can't load KPU profile %s. Using default.\n", 1779 kpu_profile); 1780 kfree(rvu->kpu_fwdata); 1781 rvu->kpu_fwdata = NULL; 1782 goto revert_to_default; 1783 } 1784 1785 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1786 profile->name, NPC_KPU_VER_MAJ(profile->version), 1787 NPC_KPU_VER_MIN(profile->version), 1788 NPC_KPU_VER_PATCH(profile->version)); 1789 1790 return; 1791 1792 revert_to_default: 1793 npc_prepare_default_kpu(profile); 1794 } 1795 1796 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1797 { 1798 struct rvu_hwinfo *hw = rvu->hw; 1799 int num_pkinds, num_kpus, idx; 1800 1801 /* Disable all KPUs and their entries */ 1802 for (idx = 0; idx < hw->npc_kpus; idx++) { 1803 rvu_write64(rvu, blkaddr, 1804 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1805 rvu_write64(rvu, blkaddr, 1806 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1807 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1808 } 1809 1810 /* Load and customize KPU profile. */ 1811 npc_load_kpu_profile(rvu); 1812 1813 /* First program IKPU profile i.e PKIND configs. 1814 * Check HW max count to avoid configuring junk or 1815 * writing to unsupported CSR addresses. 1816 */ 1817 num_pkinds = rvu->kpu.pkinds; 1818 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 1819 1820 for (idx = 0; idx < num_pkinds; idx++) 1821 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1822 1823 /* Program KPU CAM and Action profiles */ 1824 num_kpus = rvu->kpu.kpus; 1825 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1826 1827 for (idx = 0; idx < num_kpus; idx++) 1828 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1829 } 1830 1831 void npc_mcam_rsrcs_deinit(struct rvu *rvu) 1832 { 1833 struct npc_mcam *mcam = &rvu->hw->mcam; 1834 1835 bitmap_free(mcam->bmap); 1836 bitmap_free(mcam->bmap_reverse); 1837 kfree(mcam->entry2pfvf_map); 1838 kfree(mcam->cntr2pfvf_map); 1839 kfree(mcam->entry2cntr_map); 1840 kfree(mcam->cntr_refcnt); 1841 kfree(mcam->entry2target_pffunc); 1842 kfree(mcam->counters.bmap); 1843 } 1844 1845 int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1846 { 1847 int nixlf_count = rvu_get_nixlf_count(rvu); 1848 struct npc_mcam *mcam = &rvu->hw->mcam; 1849 int rsvd, err; 1850 u16 index; 1851 int cntr; 1852 u64 cfg; 1853 1854 /* Actual number of MCAM entries vary by entry size */ 1855 cfg = (rvu_read64(rvu, blkaddr, 1856 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1857 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1858 mcam->keysize = cfg; 1859 1860 /* Number of banks combined per MCAM entry */ 1861 if (cfg == NPC_MCAM_KEY_X4) 1862 mcam->banks_per_entry = 4; 1863 else if (cfg == NPC_MCAM_KEY_X2) 1864 mcam->banks_per_entry = 2; 1865 else 1866 mcam->banks_per_entry = 1; 1867 1868 /* Reserve one MCAM entry for each of the NIX LF to 1869 * guarantee space to install default matching DMAC rule. 1870 * Also reserve 2 MCAM entries for each PF for default 1871 * channel based matching or 'bcast & promisc' matching to 1872 * support BCAST and PROMISC modes of operation for PFs. 1873 * PF0 is excluded. 1874 */ 1875 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1876 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1877 if (mcam->total_entries <= rsvd) { 1878 dev_warn(rvu->dev, 1879 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1880 mcam->total_entries); 1881 return -ENOMEM; 1882 } 1883 1884 mcam->bmap_entries = mcam->total_entries - rsvd; 1885 mcam->nixlf_offset = mcam->bmap_entries; 1886 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1887 1888 /* Allocate bitmaps for managing MCAM entries */ 1889 mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 1890 if (!mcam->bmap) 1891 return -ENOMEM; 1892 1893 mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); 1894 if (!mcam->bmap_reverse) 1895 goto free_bmap; 1896 1897 mcam->bmap_fcnt = mcam->bmap_entries; 1898 1899 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1900 mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16), 1901 GFP_KERNEL); 1902 1903 if (!mcam->entry2pfvf_map) 1904 goto free_bmap_reverse; 1905 1906 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1907 * allocations and another 1/8th at the top for high priority 1908 * allocations. 1909 */ 1910 mcam->lprio_count = mcam->bmap_entries / 8; 1911 if (mcam->lprio_count > BITS_PER_LONG) 1912 mcam->lprio_count = round_down(mcam->lprio_count, 1913 BITS_PER_LONG); 1914 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1915 mcam->hprio_count = mcam->lprio_count; 1916 mcam->hprio_end = mcam->hprio_count; 1917 1918 /* Allocate bitmap for managing MCAM counters and memory 1919 * for saving counter to RVU PFFUNC allocation mapping. 1920 */ 1921 err = rvu_alloc_bitmap(&mcam->counters); 1922 if (err) 1923 goto free_entry_map; 1924 1925 mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16), 1926 GFP_KERNEL); 1927 if (!mcam->cntr2pfvf_map) 1928 goto free_cntr_bmap; 1929 1930 /* Alloc memory for MCAM entry to counter mapping and for tracking 1931 * counter's reference count. 1932 */ 1933 mcam->entry2cntr_map = kcalloc(mcam->bmap_entries, sizeof(u16), 1934 GFP_KERNEL); 1935 if (!mcam->entry2cntr_map) 1936 goto free_cntr_map; 1937 1938 mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16), 1939 GFP_KERNEL); 1940 if (!mcam->cntr_refcnt) 1941 goto free_entry_cntr_map; 1942 1943 /* Alloc memory for saving target device of mcam rule */ 1944 mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries, 1945 sizeof(u16), GFP_KERNEL); 1946 if (!mcam->entry2target_pffunc) 1947 goto free_cntr_refcnt; 1948 1949 for (index = 0; index < mcam->bmap_entries; index++) { 1950 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1951 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1952 } 1953 1954 for (cntr = 0; cntr < mcam->counters.max; cntr++) 1955 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1956 1957 mutex_init(&mcam->lock); 1958 1959 return 0; 1960 1961 free_cntr_refcnt: 1962 kfree(mcam->cntr_refcnt); 1963 free_entry_cntr_map: 1964 kfree(mcam->entry2cntr_map); 1965 free_cntr_map: 1966 kfree(mcam->cntr2pfvf_map); 1967 free_cntr_bmap: 1968 kfree(mcam->counters.bmap); 1969 free_entry_map: 1970 kfree(mcam->entry2pfvf_map); 1971 free_bmap_reverse: 1972 bitmap_free(mcam->bmap_reverse); 1973 free_bmap: 1974 bitmap_free(mcam->bmap); 1975 1976 return -ENOMEM; 1977 } 1978 1979 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1980 { 1981 struct npc_pkind *pkind = &rvu->hw->pkind; 1982 struct npc_mcam *mcam = &rvu->hw->mcam; 1983 struct rvu_hwinfo *hw = rvu->hw; 1984 u64 npc_const, npc_const1; 1985 u64 npc_const2 = 0; 1986 1987 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1988 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1989 if (npc_const1 & BIT_ULL(63)) 1990 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1991 1992 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 1993 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 1994 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1995 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1996 hw->npc_intfs = npc_const & 0xFULL; 1997 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1998 1999 mcam->banks = (npc_const >> 44) & 0xFULL; 2000 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 2001 hw->npc_stat_ena = BIT_ULL(9); 2002 /* Extended set */ 2003 if (npc_const2) { 2004 hw->npc_ext_set = true; 2005 /* 96xx supports only match_stats and npc_counters 2006 * reflected in NPC_AF_CONST reg. 2007 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 2008 * 98xx has both match_stat and ext and npc_counter 2009 * reflected in NPC_AF_CONST2 2010 * STAT_SEL_EXT added at [12:14] bit position. 2011 * cn10k supports only ext and hence npc_counters in 2012 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 2013 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 2014 */ 2015 if (!hw->npc_counters) 2016 hw->npc_stat_ena = BIT_ULL(63); 2017 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 2018 mcam->banksize = npc_const2 & 0xFFFFULL; 2019 } 2020 2021 mcam->counters.max = hw->npc_counters; 2022 } 2023 2024 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 2025 { 2026 struct npc_mcam_kex *mkex = rvu->kpu.mkex; 2027 struct npc_mcam *mcam = &rvu->hw->mcam; 2028 struct rvu_hwinfo *hw = rvu->hw; 2029 u64 nibble_ena, rx_kex, tx_kex; 2030 u8 intf; 2031 2032 /* Reserve last counter for MCAM RX miss action which is set to 2033 * drop packet. This way we will know how many pkts didn't match 2034 * any MCAM entry. 2035 */ 2036 mcam->counters.max--; 2037 mcam->rx_miss_act_cntr = mcam->counters.max; 2038 2039 rx_kex = mkex->keyx_cfg[NIX_INTF_RX]; 2040 tx_kex = mkex->keyx_cfg[NIX_INTF_TX]; 2041 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 2042 2043 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 2044 if (nibble_ena) { 2045 tx_kex &= ~NPC_PARSE_NIBBLE; 2046 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 2047 mkex->keyx_cfg[NIX_INTF_TX] = tx_kex; 2048 } 2049 2050 /* Configure RX interfaces */ 2051 for (intf = 0; intf < hw->npc_intfs; intf++) { 2052 if (is_npc_intf_tx(intf)) 2053 continue; 2054 2055 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 2056 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2057 rx_kex); 2058 2059 /* If MCAM lookup doesn't result in a match, drop the received 2060 * packet. And map this action to a counter to count dropped 2061 * packets. 2062 */ 2063 rvu_write64(rvu, blkaddr, 2064 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 2065 2066 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 2067 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 2068 */ 2069 rvu_write64(rvu, blkaddr, 2070 NPC_AF_INTFX_MISS_STAT_ACT(intf), 2071 ((mcam->rx_miss_act_cntr >> 9) << 12) | 2072 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 2073 } 2074 2075 /* Configure TX interfaces */ 2076 for (intf = 0; intf < hw->npc_intfs; intf++) { 2077 if (is_npc_intf_rx(intf)) 2078 continue; 2079 2080 /* Extract Ltypes LID_LA to LID_LE */ 2081 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 2082 tx_kex); 2083 2084 /* Set TX miss action to UCAST_DEFAULT i.e 2085 * transmit the packet on NIX LF SQ's default channel. 2086 */ 2087 rvu_write64(rvu, blkaddr, 2088 NPC_AF_INTFX_MISS_ACT(intf), 2089 NIX_TX_ACTIONOP_UCAST_DEFAULT); 2090 } 2091 } 2092 2093 int rvu_npc_init(struct rvu *rvu) 2094 { 2095 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 2096 struct npc_pkind *pkind = &rvu->hw->pkind; 2097 struct npc_mcam *mcam = &rvu->hw->mcam; 2098 int blkaddr, entry, bank, err; 2099 2100 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2101 if (blkaddr < 0) { 2102 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 2103 return -ENODEV; 2104 } 2105 2106 rvu_npc_hw_init(rvu, blkaddr); 2107 2108 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2109 for (bank = 0; bank < mcam->banks; bank++) { 2110 for (entry = 0; entry < mcam->banksize; entry++) 2111 rvu_write64(rvu, blkaddr, 2112 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2113 } 2114 2115 err = rvu_alloc_bitmap(&pkind->rsrc); 2116 if (err) 2117 return err; 2118 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2119 * no need to configure PKIND for all LBKs separately. 2120 */ 2121 rvu_alloc_rsrc(&pkind->rsrc); 2122 2123 /* Allocate mem for pkind to PF and channel mapping info */ 2124 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2125 sizeof(u32), GFP_KERNEL); 2126 if (!pkind->pfchan_map) 2127 return -ENOMEM; 2128 2129 /* Configure KPU profile */ 2130 npc_parser_profile_init(rvu, blkaddr); 2131 2132 /* Config Outer L2, IPv4's NPC layer info */ 2133 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2134 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2135 kpu->lt_def->pck_ol2.ltype_mask); 2136 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2137 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2138 kpu->lt_def->pck_oip4.ltype_mask); 2139 2140 /* Config Inner IPV4 NPC layer info */ 2141 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2142 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2143 kpu->lt_def->pck_iip4.ltype_mask); 2144 2145 /* Enable below for Rx pkts. 2146 * - Outer IPv4 header checksum validation. 2147 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2148 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2149 * - Inner IPv4 header checksum validation. 2150 * - Set non zero checksum error code value 2151 */ 2152 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2153 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2154 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2155 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2156 2157 rvu_npc_setup_interfaces(rvu, blkaddr); 2158 2159 npc_config_secret_key(rvu, blkaddr); 2160 /* Configure MKEX profile */ 2161 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2162 2163 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2164 if (err) 2165 return err; 2166 2167 err = npc_flow_steering_init(rvu, blkaddr); 2168 if (err) { 2169 dev_err(rvu->dev, 2170 "Incorrect mkex profile loaded using default mkex\n"); 2171 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2172 } 2173 2174 return 0; 2175 } 2176 2177 void rvu_npc_freemem(struct rvu *rvu) 2178 { 2179 struct npc_pkind *pkind = &rvu->hw->pkind; 2180 struct npc_mcam *mcam = &rvu->hw->mcam; 2181 2182 kfree(pkind->rsrc.bmap); 2183 npc_mcam_rsrcs_deinit(rvu); 2184 if (rvu->kpu_prfl_addr) 2185 iounmap(rvu->kpu_prfl_addr); 2186 else 2187 kfree(rvu->kpu_fwdata); 2188 mutex_destroy(&mcam->lock); 2189 } 2190 2191 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2192 int blkaddr, int *alloc_cnt, 2193 int *enable_cnt) 2194 { 2195 struct npc_mcam *mcam = &rvu->hw->mcam; 2196 int entry; 2197 2198 *alloc_cnt = 0; 2199 *enable_cnt = 0; 2200 2201 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2202 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2203 (*alloc_cnt)++; 2204 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2205 (*enable_cnt)++; 2206 } 2207 } 2208 } 2209 2210 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2211 int blkaddr, int *alloc_cnt, 2212 int *enable_cnt) 2213 { 2214 struct npc_mcam *mcam = &rvu->hw->mcam; 2215 int cntr; 2216 2217 *alloc_cnt = 0; 2218 *enable_cnt = 0; 2219 2220 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2221 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2222 (*alloc_cnt)++; 2223 if (mcam->cntr_refcnt[cntr]) 2224 (*enable_cnt)++; 2225 } 2226 } 2227 } 2228 2229 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 2230 u16 pcifunc, int entry) 2231 { 2232 /* verify AF installed entries */ 2233 if (is_pffunc_af(pcifunc)) 2234 return 0; 2235 /* Verify if entry is valid and if it is indeed 2236 * allocated to the requesting PFFUNC. 2237 */ 2238 if (entry >= mcam->bmap_entries) 2239 return NPC_MCAM_INVALID_REQ; 2240 2241 if (pcifunc != mcam->entry2pfvf_map[entry]) 2242 return NPC_MCAM_PERM_DENIED; 2243 2244 return 0; 2245 } 2246 2247 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2248 u16 pcifunc, int cntr) 2249 { 2250 /* Verify if counter is valid and if it is indeed 2251 * allocated to the requesting PFFUNC. 2252 */ 2253 if (cntr >= mcam->counters.max) 2254 return NPC_MCAM_INVALID_REQ; 2255 2256 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2257 return NPC_MCAM_PERM_DENIED; 2258 2259 return 0; 2260 } 2261 2262 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2263 int blkaddr, u16 entry, u16 cntr) 2264 { 2265 u16 index = entry & (mcam->banksize - 1); 2266 u32 bank = npc_get_bank(mcam, entry); 2267 struct rvu_hwinfo *hw = rvu->hw; 2268 2269 /* Set mapping and increment counter's refcnt */ 2270 mcam->entry2cntr_map[entry] = cntr; 2271 mcam->cntr_refcnt[cntr]++; 2272 /* Enable stats */ 2273 rvu_write64(rvu, blkaddr, 2274 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2275 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2276 } 2277 2278 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2279 struct npc_mcam *mcam, 2280 int blkaddr, u16 entry, u16 cntr) 2281 { 2282 u16 index = entry & (mcam->banksize - 1); 2283 u32 bank = npc_get_bank(mcam, entry); 2284 2285 /* Remove mapping and reduce counter's refcnt */ 2286 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2287 mcam->cntr_refcnt[cntr]--; 2288 /* Disable stats */ 2289 rvu_write64(rvu, blkaddr, 2290 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2291 } 2292 2293 /* Sets MCAM entry in bitmap as used. Update 2294 * reverse bitmap too. Should be called with 2295 * 'mcam->lock' held. 2296 */ 2297 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2298 { 2299 u16 entry, rentry; 2300 2301 entry = index; 2302 rentry = mcam->bmap_entries - index - 1; 2303 2304 __set_bit(entry, mcam->bmap); 2305 __set_bit(rentry, mcam->bmap_reverse); 2306 mcam->bmap_fcnt--; 2307 } 2308 2309 /* Sets MCAM entry in bitmap as free. Update 2310 * reverse bitmap too. Should be called with 2311 * 'mcam->lock' held. 2312 */ 2313 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2314 { 2315 u16 entry, rentry; 2316 2317 entry = index; 2318 rentry = mcam->bmap_entries - index - 1; 2319 2320 __clear_bit(entry, mcam->bmap); 2321 __clear_bit(rentry, mcam->bmap_reverse); 2322 mcam->bmap_fcnt++; 2323 } 2324 2325 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2326 int blkaddr, u16 pcifunc) 2327 { 2328 u16 index, cntr; 2329 2330 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2331 for (index = 0; index < mcam->bmap_entries; index++) { 2332 if (mcam->entry2pfvf_map[index] == pcifunc) { 2333 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2334 /* Free the entry in bitmap */ 2335 npc_mcam_clear_bit(mcam, index); 2336 /* Disable the entry */ 2337 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2338 2339 /* Update entry2counter mapping */ 2340 cntr = mcam->entry2cntr_map[index]; 2341 if (cntr != NPC_MCAM_INVALID_MAP) 2342 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2343 blkaddr, index, 2344 cntr); 2345 mcam->entry2target_pffunc[index] = 0x0; 2346 } 2347 } 2348 } 2349 2350 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2351 u16 pcifunc) 2352 { 2353 u16 cntr; 2354 2355 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2356 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2357 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2358 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2359 mcam->cntr_refcnt[cntr] = 0; 2360 rvu_free_rsrc(&mcam->counters, cntr); 2361 /* This API is expected to be called after freeing 2362 * MCAM entries, which inturn will remove 2363 * 'entry to counter' mapping. 2364 * No need to do it again. 2365 */ 2366 } 2367 } 2368 } 2369 2370 /* Find area of contiguous free entries of size 'nr'. 2371 * If not found return max contiguous free entries available. 2372 */ 2373 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2374 u16 nr, u16 *max_area) 2375 { 2376 u16 max_area_start = 0; 2377 u16 index, next, end; 2378 2379 *max_area = 0; 2380 2381 again: 2382 index = find_next_zero_bit(map, size, start); 2383 if (index >= size) 2384 return max_area_start; 2385 2386 end = ((index + nr) >= size) ? size : index + nr; 2387 next = find_next_bit(map, end, index); 2388 if (*max_area < (next - index)) { 2389 *max_area = next - index; 2390 max_area_start = index; 2391 } 2392 2393 if (next < end) { 2394 start = next + 1; 2395 goto again; 2396 } 2397 2398 return max_area_start; 2399 } 2400 2401 /* Find number of free MCAM entries available 2402 * within range i.e in between 'start' and 'end'. 2403 */ 2404 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2405 { 2406 u16 index, next; 2407 u16 fcnt = 0; 2408 2409 again: 2410 if (start >= end) 2411 return fcnt; 2412 2413 index = find_next_zero_bit(map, end, start); 2414 if (index >= end) 2415 return fcnt; 2416 2417 next = find_next_bit(map, end, index); 2418 if (next <= end) { 2419 fcnt += next - index; 2420 start = next + 1; 2421 goto again; 2422 } 2423 2424 fcnt += end - index; 2425 return fcnt; 2426 } 2427 2428 static void 2429 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2430 struct npc_mcam_alloc_entry_req *req, 2431 u16 *start, u16 *end, bool *reverse) 2432 { 2433 u16 fcnt; 2434 2435 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2436 goto hprio; 2437 2438 /* For a low priority entry allocation 2439 * - If reference entry is not in hprio zone then 2440 * search range: ref_entry to end. 2441 * - If reference entry is in hprio zone and if 2442 * request can be accomodated in non-hprio zone then 2443 * search range: 'start of middle zone' to 'end' 2444 * - else search in reverse, so that less number of hprio 2445 * zone entries are allocated. 2446 */ 2447 2448 *reverse = false; 2449 *start = req->ref_entry + 1; 2450 *end = mcam->bmap_entries; 2451 2452 if (req->ref_entry >= mcam->hprio_end) 2453 return; 2454 2455 fcnt = npc_mcam_get_free_count(mcam->bmap, 2456 mcam->hprio_end, mcam->bmap_entries); 2457 if (fcnt > req->count) 2458 *start = mcam->hprio_end; 2459 else 2460 *reverse = true; 2461 return; 2462 2463 hprio: 2464 /* For a high priority entry allocation, search is always 2465 * in reverse to preserve hprio zone entries. 2466 * - If reference entry is not in lprio zone then 2467 * search range: 0 to ref_entry. 2468 * - If reference entry is in lprio zone and if 2469 * request can be accomodated in middle zone then 2470 * search range: 'hprio_end' to 'lprio_start' 2471 */ 2472 2473 *reverse = true; 2474 *start = 0; 2475 *end = req->ref_entry; 2476 2477 if (req->ref_entry <= mcam->lprio_start) 2478 return; 2479 2480 fcnt = npc_mcam_get_free_count(mcam->bmap, 2481 mcam->hprio_end, mcam->lprio_start); 2482 if (fcnt < req->count) 2483 return; 2484 *start = mcam->hprio_end; 2485 *end = mcam->lprio_start; 2486 } 2487 2488 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2489 struct npc_mcam_alloc_entry_req *req, 2490 struct npc_mcam_alloc_entry_rsp *rsp) 2491 { 2492 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2493 u16 fcnt, hp_fcnt, lp_fcnt; 2494 u16 start, end, index; 2495 int entry, next_start; 2496 bool reverse = false; 2497 unsigned long *bmap; 2498 u16 max_contig; 2499 2500 mutex_lock(&mcam->lock); 2501 2502 /* Check if there are any free entries */ 2503 if (!mcam->bmap_fcnt) { 2504 mutex_unlock(&mcam->lock); 2505 return NPC_MCAM_ALLOC_FAILED; 2506 } 2507 2508 /* MCAM entries are divided into high priority, middle and 2509 * low priority zones. Idea is to not allocate top and lower 2510 * most entries as much as possible, this is to increase 2511 * probability of honouring priority allocation requests. 2512 * 2513 * Two bitmaps are used for mcam entry management, 2514 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2515 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2516 * 2517 * Reverse bitmap is used to allocate entries 2518 * - when a higher priority entry is requested 2519 * - when available free entries are less. 2520 * Lower priority ones out of avaialble free entries are always 2521 * chosen when 'high vs low' question arises. 2522 * 2523 * For a VF base MCAM match rule is set by its PF. And all the 2524 * further MCAM rules installed by VF on its own are 2525 * concatenated with the base rule set by its PF. Hence PF entries 2526 * should be at lower priority compared to VF entries. Otherwise 2527 * base rule is hit always and rules installed by VF will be of 2528 * no use. Hence if the request is from PF then allocate low 2529 * priority entries. 2530 */ 2531 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 2532 goto lprio_alloc; 2533 2534 /* Get the search range for priority allocation request */ 2535 if (req->priority) { 2536 npc_get_mcam_search_range_priority(mcam, req, 2537 &start, &end, &reverse); 2538 goto alloc; 2539 } 2540 2541 /* Find out the search range for non-priority allocation request 2542 * 2543 * Get MCAM free entry count in middle zone. 2544 */ 2545 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2546 mcam->lprio_start, 2547 mcam->bmap_entries); 2548 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2549 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2550 2551 /* Check if request can be accomodated in the middle zone */ 2552 if (fcnt > req->count) { 2553 start = mcam->hprio_end; 2554 end = mcam->lprio_start; 2555 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2556 /* Expand search zone from half of hprio zone to 2557 * half of lprio zone. 2558 */ 2559 start = mcam->hprio_end / 2; 2560 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2561 reverse = true; 2562 } else { 2563 /* Not enough free entries, search all entries in reverse, 2564 * so that low priority ones will get used up. 2565 */ 2566 lprio_alloc: 2567 reverse = true; 2568 start = 0; 2569 end = mcam->bmap_entries; 2570 /* Ensure PF requests are always at bottom and if PF requests 2571 * for higher/lower priority entry wrt reference entry then 2572 * honour that criteria and start search for entries from bottom 2573 * and not in mid zone. 2574 */ 2575 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 2576 req->priority == NPC_MCAM_HIGHER_PRIO) 2577 end = req->ref_entry; 2578 2579 if (!(pcifunc & RVU_PFVF_FUNC_MASK) && 2580 req->priority == NPC_MCAM_LOWER_PRIO) 2581 start = req->ref_entry; 2582 } 2583 2584 alloc: 2585 if (reverse) { 2586 bmap = mcam->bmap_reverse; 2587 start = mcam->bmap_entries - start; 2588 end = mcam->bmap_entries - end; 2589 swap(start, end); 2590 } else { 2591 bmap = mcam->bmap; 2592 } 2593 2594 if (req->contig) { 2595 /* Allocate requested number of contiguous entries, if 2596 * unsuccessful find max contiguous entries available. 2597 */ 2598 index = npc_mcam_find_zero_area(bmap, end, start, 2599 req->count, &max_contig); 2600 rsp->count = max_contig; 2601 if (reverse) 2602 rsp->entry = mcam->bmap_entries - index - max_contig; 2603 else 2604 rsp->entry = index; 2605 } else { 2606 /* Allocate requested number of non-contiguous entries, 2607 * if unsuccessful allocate as many as possible. 2608 */ 2609 rsp->count = 0; 2610 next_start = start; 2611 for (entry = 0; entry < req->count; entry++) { 2612 index = find_next_zero_bit(bmap, end, next_start); 2613 if (index >= end) 2614 break; 2615 2616 next_start = start + (index - start) + 1; 2617 2618 /* Save the entry's index */ 2619 if (reverse) 2620 index = mcam->bmap_entries - index - 1; 2621 entry_list[entry] = index; 2622 rsp->count++; 2623 } 2624 } 2625 2626 /* If allocating requested no of entries is unsucessful, 2627 * expand the search range to full bitmap length and retry. 2628 */ 2629 if (!req->priority && (rsp->count < req->count) && 2630 ((end - start) != mcam->bmap_entries)) { 2631 reverse = true; 2632 start = 0; 2633 end = mcam->bmap_entries; 2634 goto alloc; 2635 } 2636 2637 /* For priority entry allocation requests, if allocation is 2638 * failed then expand search to max possible range and retry. 2639 */ 2640 if (req->priority && rsp->count < req->count) { 2641 if (req->priority == NPC_MCAM_LOWER_PRIO && 2642 (start != (req->ref_entry + 1))) { 2643 start = req->ref_entry + 1; 2644 end = mcam->bmap_entries; 2645 reverse = false; 2646 goto alloc; 2647 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2648 ((end - start) != req->ref_entry)) { 2649 start = 0; 2650 end = req->ref_entry; 2651 reverse = true; 2652 goto alloc; 2653 } 2654 } 2655 2656 /* Copy MCAM entry indices into mbox response entry_list. 2657 * Requester always expects indices in ascending order, so 2658 * reverse the list if reverse bitmap is used for allocation. 2659 */ 2660 if (!req->contig && rsp->count) { 2661 index = 0; 2662 for (entry = rsp->count - 1; entry >= 0; entry--) { 2663 if (reverse) 2664 rsp->entry_list[index++] = entry_list[entry]; 2665 else 2666 rsp->entry_list[entry] = entry_list[entry]; 2667 } 2668 } 2669 2670 /* Mark the allocated entries as used and set nixlf mapping */ 2671 for (entry = 0; entry < rsp->count; entry++) { 2672 index = req->contig ? 2673 (rsp->entry + entry) : rsp->entry_list[entry]; 2674 npc_mcam_set_bit(mcam, index); 2675 mcam->entry2pfvf_map[index] = pcifunc; 2676 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2677 } 2678 2679 /* Update available free count in mbox response */ 2680 rsp->free_count = mcam->bmap_fcnt; 2681 2682 mutex_unlock(&mcam->lock); 2683 return 0; 2684 } 2685 2686 /* Marks bitmaps to reserved the mcam slot */ 2687 void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx) 2688 { 2689 struct npc_mcam *mcam = &rvu->hw->mcam; 2690 2691 npc_mcam_set_bit(mcam, entry_idx); 2692 } 2693 2694 int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) 2695 { 2696 struct npc_mcam *mcam = &rvu->hw->mcam; 2697 struct npc_install_flow_rsp rsp; 2698 struct rvu_npc_mcam_rule *rule; 2699 int blkaddr; 2700 2701 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2702 if (blkaddr < 0) 2703 return -EINVAL; 2704 2705 mutex_lock(&mcam->lock); 2706 list_for_each_entry(rule, &mcam->mcam_rules, list) { 2707 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, rule->entry)) 2708 continue; 2709 if (!rule->default_rule) 2710 continue; 2711 if (enable && !rule->has_cntr) { /* Alloc and map new counter */ 2712 __rvu_mcam_add_counter_to_rule(rvu, rule->owner, 2713 rule, &rsp); 2714 if (rsp.counter < 0) { 2715 dev_err(rvu->dev, 2716 "%s: Failed to allocate cntr for default rule (err=%d)\n", 2717 __func__, rsp.counter); 2718 break; 2719 } 2720 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2721 rule->entry, rsp.counter); 2722 /* Reset counter before use */ 2723 rvu_write64(rvu, blkaddr, 2724 NPC_AF_MATCH_STATX(rule->cntr), 0x0); 2725 } 2726 2727 /* Free and unmap counter */ 2728 if (!enable && rule->has_cntr) 2729 __rvu_mcam_remove_counter_from_rule(rvu, rule->owner, 2730 rule); 2731 } 2732 mutex_unlock(&mcam->lock); 2733 2734 return 0; 2735 } 2736 2737 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2738 struct npc_mcam_alloc_entry_req *req, 2739 struct npc_mcam_alloc_entry_rsp *rsp) 2740 { 2741 struct npc_mcam *mcam = &rvu->hw->mcam; 2742 u16 pcifunc = req->hdr.pcifunc; 2743 int blkaddr; 2744 2745 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2746 if (blkaddr < 0) 2747 return NPC_MCAM_INVALID_REQ; 2748 2749 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2750 rsp->free_count = 0; 2751 2752 /* Check if ref_entry is greater that the range 2753 * then set it to max value. 2754 */ 2755 if (req->ref_entry > mcam->bmap_entries) 2756 req->ref_entry = mcam->bmap_entries; 2757 2758 /* ref_entry can't be '0' if requested priority is high. 2759 * Can't be last entry if requested priority is low. 2760 */ 2761 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2762 ((req->ref_entry == mcam->bmap_entries) && 2763 req->priority == NPC_MCAM_LOWER_PRIO)) 2764 return NPC_MCAM_INVALID_REQ; 2765 2766 /* Since list of allocated indices needs to be sent to requester, 2767 * max number of non-contiguous entries per mbox msg is limited. 2768 */ 2769 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 2770 dev_err(rvu->dev, 2771 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 2772 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 2773 return NPC_MCAM_INVALID_REQ; 2774 } 2775 2776 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2777 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2778 return NPC_MCAM_ALLOC_DENIED; 2779 2780 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2781 } 2782 2783 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2784 struct npc_mcam_free_entry_req *req, 2785 struct msg_rsp *rsp) 2786 { 2787 struct npc_mcam *mcam = &rvu->hw->mcam; 2788 u16 pcifunc = req->hdr.pcifunc; 2789 int blkaddr, rc = 0; 2790 u16 cntr; 2791 2792 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2793 if (blkaddr < 0) 2794 return NPC_MCAM_INVALID_REQ; 2795 2796 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2797 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2798 return NPC_MCAM_INVALID_REQ; 2799 2800 mutex_lock(&mcam->lock); 2801 2802 if (req->all) 2803 goto free_all; 2804 2805 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2806 if (rc) 2807 goto exit; 2808 2809 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 2810 mcam->entry2target_pffunc[req->entry] = 0x0; 2811 npc_mcam_clear_bit(mcam, req->entry); 2812 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2813 2814 /* Update entry2counter mapping */ 2815 cntr = mcam->entry2cntr_map[req->entry]; 2816 if (cntr != NPC_MCAM_INVALID_MAP) 2817 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2818 req->entry, cntr); 2819 2820 goto exit; 2821 2822 free_all: 2823 /* Free up all entries allocated to requesting PFFUNC */ 2824 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2825 exit: 2826 mutex_unlock(&mcam->lock); 2827 return rc; 2828 } 2829 2830 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2831 struct npc_mcam_read_entry_req *req, 2832 struct npc_mcam_read_entry_rsp *rsp) 2833 { 2834 struct npc_mcam *mcam = &rvu->hw->mcam; 2835 u16 pcifunc = req->hdr.pcifunc; 2836 int blkaddr, rc; 2837 2838 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2839 if (blkaddr < 0) 2840 return NPC_MCAM_INVALID_REQ; 2841 2842 mutex_lock(&mcam->lock); 2843 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2844 if (!rc) { 2845 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2846 &rsp->entry_data, 2847 &rsp->intf, &rsp->enable); 2848 } 2849 2850 mutex_unlock(&mcam->lock); 2851 return rc; 2852 } 2853 2854 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2855 struct npc_mcam_write_entry_req *req, 2856 struct msg_rsp *rsp) 2857 { 2858 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2859 struct npc_mcam *mcam = &rvu->hw->mcam; 2860 u16 pcifunc = req->hdr.pcifunc; 2861 int blkaddr, rc; 2862 u8 nix_intf; 2863 2864 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2865 if (blkaddr < 0) 2866 return NPC_MCAM_INVALID_REQ; 2867 2868 mutex_lock(&mcam->lock); 2869 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2870 if (rc) 2871 goto exit; 2872 2873 if (req->set_cntr && 2874 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2875 rc = NPC_MCAM_INVALID_REQ; 2876 goto exit; 2877 } 2878 2879 if (!is_npc_interface_valid(rvu, req->intf)) { 2880 rc = NPC_MCAM_INVALID_REQ; 2881 goto exit; 2882 } 2883 2884 if (is_npc_intf_tx(req->intf)) 2885 nix_intf = pfvf->nix_tx_intf; 2886 else 2887 nix_intf = pfvf->nix_rx_intf; 2888 2889 /* For AF installed rules, the nix_intf should be set to target NIX */ 2890 if (is_pffunc_af(req->hdr.pcifunc)) 2891 nix_intf = req->intf; 2892 2893 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2894 &req->entry_data, req->enable_entry); 2895 2896 if (req->set_cntr) 2897 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2898 req->entry, req->cntr); 2899 2900 rc = 0; 2901 exit: 2902 mutex_unlock(&mcam->lock); 2903 return rc; 2904 } 2905 2906 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2907 struct npc_mcam_ena_dis_entry_req *req, 2908 struct msg_rsp *rsp) 2909 { 2910 struct npc_mcam *mcam = &rvu->hw->mcam; 2911 u16 pcifunc = req->hdr.pcifunc; 2912 int blkaddr, rc; 2913 2914 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2915 if (blkaddr < 0) 2916 return NPC_MCAM_INVALID_REQ; 2917 2918 mutex_lock(&mcam->lock); 2919 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2920 mutex_unlock(&mcam->lock); 2921 if (rc) 2922 return rc; 2923 2924 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2925 2926 return 0; 2927 } 2928 2929 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2930 struct npc_mcam_ena_dis_entry_req *req, 2931 struct msg_rsp *rsp) 2932 { 2933 struct npc_mcam *mcam = &rvu->hw->mcam; 2934 u16 pcifunc = req->hdr.pcifunc; 2935 int blkaddr, rc; 2936 2937 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2938 if (blkaddr < 0) 2939 return NPC_MCAM_INVALID_REQ; 2940 2941 mutex_lock(&mcam->lock); 2942 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2943 mutex_unlock(&mcam->lock); 2944 if (rc) 2945 return rc; 2946 2947 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2948 2949 return 0; 2950 } 2951 2952 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2953 struct npc_mcam_shift_entry_req *req, 2954 struct npc_mcam_shift_entry_rsp *rsp) 2955 { 2956 struct npc_mcam *mcam = &rvu->hw->mcam; 2957 u16 pcifunc = req->hdr.pcifunc; 2958 u16 old_entry, new_entry; 2959 int blkaddr, rc = 0; 2960 u16 index, cntr; 2961 2962 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2963 if (blkaddr < 0) 2964 return NPC_MCAM_INVALID_REQ; 2965 2966 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2967 return NPC_MCAM_INVALID_REQ; 2968 2969 mutex_lock(&mcam->lock); 2970 for (index = 0; index < req->shift_count; index++) { 2971 old_entry = req->curr_entry[index]; 2972 new_entry = req->new_entry[index]; 2973 2974 /* Check if both old and new entries are valid and 2975 * does belong to this PFFUNC or not. 2976 */ 2977 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2978 if (rc) 2979 break; 2980 2981 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2982 if (rc) 2983 break; 2984 2985 /* new_entry should not have a counter mapped */ 2986 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2987 rc = NPC_MCAM_PERM_DENIED; 2988 break; 2989 } 2990 2991 /* Disable the new_entry */ 2992 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2993 2994 /* Copy rule from old entry to new entry */ 2995 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2996 2997 /* Copy counter mapping, if any */ 2998 cntr = mcam->entry2cntr_map[old_entry]; 2999 if (cntr != NPC_MCAM_INVALID_MAP) { 3000 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3001 old_entry, cntr); 3002 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3003 new_entry, cntr); 3004 } 3005 3006 /* Enable new_entry and disable old_entry */ 3007 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 3008 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 3009 } 3010 3011 /* If shift has failed then report the failed index */ 3012 if (index != req->shift_count) { 3013 rc = NPC_MCAM_PERM_DENIED; 3014 rsp->failed_entry_idx = index; 3015 } 3016 3017 mutex_unlock(&mcam->lock); 3018 return rc; 3019 } 3020 3021 static int __npc_mcam_alloc_counter(struct rvu *rvu, 3022 struct npc_mcam_alloc_counter_req *req, 3023 struct npc_mcam_alloc_counter_rsp *rsp) 3024 { 3025 struct npc_mcam *mcam = &rvu->hw->mcam; 3026 u16 pcifunc = req->hdr.pcifunc; 3027 u16 max_contig, cntr; 3028 int blkaddr, index; 3029 3030 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3031 if (blkaddr < 0) 3032 return NPC_MCAM_INVALID_REQ; 3033 3034 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 3035 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 3036 return NPC_MCAM_INVALID_REQ; 3037 3038 /* Since list of allocated counter IDs needs to be sent to requester, 3039 * max number of non-contiguous counters per mbox msg is limited. 3040 */ 3041 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 3042 return NPC_MCAM_INVALID_REQ; 3043 3044 3045 /* Check if unused counters are available or not */ 3046 if (!rvu_rsrc_free_count(&mcam->counters)) { 3047 return NPC_MCAM_ALLOC_FAILED; 3048 } 3049 3050 rsp->count = 0; 3051 3052 if (req->contig) { 3053 /* Allocate requested number of contiguous counters, if 3054 * unsuccessful find max contiguous entries available. 3055 */ 3056 index = npc_mcam_find_zero_area(mcam->counters.bmap, 3057 mcam->counters.max, 0, 3058 req->count, &max_contig); 3059 rsp->count = max_contig; 3060 rsp->cntr = index; 3061 for (cntr = index; cntr < (index + max_contig); cntr++) { 3062 __set_bit(cntr, mcam->counters.bmap); 3063 mcam->cntr2pfvf_map[cntr] = pcifunc; 3064 } 3065 } else { 3066 /* Allocate requested number of non-contiguous counters, 3067 * if unsuccessful allocate as many as possible. 3068 */ 3069 for (cntr = 0; cntr < req->count; cntr++) { 3070 index = rvu_alloc_rsrc(&mcam->counters); 3071 if (index < 0) 3072 break; 3073 rsp->cntr_list[cntr] = index; 3074 rsp->count++; 3075 mcam->cntr2pfvf_map[index] = pcifunc; 3076 } 3077 } 3078 3079 return 0; 3080 } 3081 3082 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 3083 struct npc_mcam_alloc_counter_req *req, 3084 struct npc_mcam_alloc_counter_rsp *rsp) 3085 { 3086 struct npc_mcam *mcam = &rvu->hw->mcam; 3087 int err; 3088 3089 mutex_lock(&mcam->lock); 3090 3091 err = __npc_mcam_alloc_counter(rvu, req, rsp); 3092 3093 mutex_unlock(&mcam->lock); 3094 return err; 3095 } 3096 3097 static int __npc_mcam_free_counter(struct rvu *rvu, 3098 struct npc_mcam_oper_counter_req *req, 3099 struct msg_rsp *rsp) 3100 { 3101 struct npc_mcam *mcam = &rvu->hw->mcam; 3102 u16 index, entry = 0; 3103 int blkaddr, err; 3104 3105 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3106 if (blkaddr < 0) 3107 return NPC_MCAM_INVALID_REQ; 3108 3109 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3110 if (err) { 3111 return err; 3112 } 3113 3114 /* Mark counter as free/unused */ 3115 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 3116 rvu_free_rsrc(&mcam->counters, req->cntr); 3117 3118 /* Disable all MCAM entry's stats which are using this counter */ 3119 while (entry < mcam->bmap_entries) { 3120 if (!mcam->cntr_refcnt[req->cntr]) 3121 break; 3122 3123 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 3124 if (index >= mcam->bmap_entries) 3125 break; 3126 entry = index + 1; 3127 if (mcam->entry2cntr_map[index] != req->cntr) 3128 continue; 3129 3130 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3131 index, req->cntr); 3132 } 3133 3134 return 0; 3135 } 3136 3137 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 3138 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3139 { 3140 struct npc_mcam *mcam = &rvu->hw->mcam; 3141 int err; 3142 3143 mutex_lock(&mcam->lock); 3144 3145 err = __npc_mcam_free_counter(rvu, req, rsp); 3146 3147 mutex_unlock(&mcam->lock); 3148 3149 return err; 3150 } 3151 3152 void __rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 3153 struct rvu_npc_mcam_rule *rule) 3154 { 3155 struct npc_mcam_oper_counter_req free_req = { 0 }; 3156 struct msg_rsp free_rsp; 3157 3158 if (!rule->has_cntr) 3159 return; 3160 3161 free_req.hdr.pcifunc = pcifunc; 3162 free_req.cntr = rule->cntr; 3163 3164 __npc_mcam_free_counter(rvu, &free_req, &free_rsp); 3165 rule->has_cntr = false; 3166 } 3167 3168 void __rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 3169 struct rvu_npc_mcam_rule *rule, 3170 struct npc_install_flow_rsp *rsp) 3171 { 3172 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 3173 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 3174 int err; 3175 3176 cntr_req.hdr.pcifunc = pcifunc; 3177 cntr_req.contig = true; 3178 cntr_req.count = 1; 3179 3180 /* we try to allocate a counter to track the stats of this 3181 * rule. If counter could not be allocated then proceed 3182 * without counter because counters are limited than entries. 3183 */ 3184 err = __npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3185 if (!err && cntr_rsp.count) { 3186 rule->cntr = cntr_rsp.cntr; 3187 rule->has_cntr = true; 3188 rsp->counter = rule->cntr; 3189 } else { 3190 rsp->counter = err; 3191 } 3192 } 3193 3194 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 3195 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 3196 { 3197 struct npc_mcam *mcam = &rvu->hw->mcam; 3198 u16 index, entry = 0; 3199 int blkaddr, rc; 3200 3201 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3202 if (blkaddr < 0) 3203 return NPC_MCAM_INVALID_REQ; 3204 3205 mutex_lock(&mcam->lock); 3206 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3207 if (rc) 3208 goto exit; 3209 3210 /* Unmap the MCAM entry and counter */ 3211 if (!req->all) { 3212 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 3213 if (rc) 3214 goto exit; 3215 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3216 req->entry, req->cntr); 3217 goto exit; 3218 } 3219 3220 /* Disable all MCAM entry's stats which are using this counter */ 3221 while (entry < mcam->bmap_entries) { 3222 if (!mcam->cntr_refcnt[req->cntr]) 3223 break; 3224 3225 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 3226 if (index >= mcam->bmap_entries) 3227 break; 3228 entry = index + 1; 3229 3230 if (mcam->entry2cntr_map[index] != req->cntr) 3231 continue; 3232 3233 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3234 index, req->cntr); 3235 } 3236 exit: 3237 mutex_unlock(&mcam->lock); 3238 return rc; 3239 } 3240 3241 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3242 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3243 { 3244 struct npc_mcam *mcam = &rvu->hw->mcam; 3245 int blkaddr, err; 3246 3247 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3248 if (blkaddr < 0) 3249 return NPC_MCAM_INVALID_REQ; 3250 3251 mutex_lock(&mcam->lock); 3252 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3253 mutex_unlock(&mcam->lock); 3254 if (err) 3255 return err; 3256 3257 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3258 3259 return 0; 3260 } 3261 3262 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 3263 struct npc_mcam_oper_counter_req *req, 3264 struct npc_mcam_oper_counter_rsp *rsp) 3265 { 3266 struct npc_mcam *mcam = &rvu->hw->mcam; 3267 int blkaddr, err; 3268 3269 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3270 if (blkaddr < 0) 3271 return NPC_MCAM_INVALID_REQ; 3272 3273 mutex_lock(&mcam->lock); 3274 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3275 mutex_unlock(&mcam->lock); 3276 if (err) 3277 return err; 3278 3279 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3280 rsp->stat &= BIT_ULL(48) - 1; 3281 3282 return 0; 3283 } 3284 3285 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3286 struct npc_mcam_alloc_and_write_entry_req *req, 3287 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3288 { 3289 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3290 struct npc_mcam_alloc_counter_req cntr_req; 3291 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3292 struct npc_mcam_alloc_entry_req entry_req; 3293 struct npc_mcam_alloc_entry_rsp entry_rsp; 3294 struct npc_mcam *mcam = &rvu->hw->mcam; 3295 u16 entry = NPC_MCAM_ENTRY_INVALID; 3296 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3297 int blkaddr, rc; 3298 u8 nix_intf; 3299 3300 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3301 if (blkaddr < 0) 3302 return NPC_MCAM_INVALID_REQ; 3303 3304 if (!is_npc_interface_valid(rvu, req->intf)) 3305 return NPC_MCAM_INVALID_REQ; 3306 3307 /* Try to allocate a MCAM entry */ 3308 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3309 entry_req.contig = true; 3310 entry_req.priority = req->priority; 3311 entry_req.ref_entry = req->ref_entry; 3312 entry_req.count = 1; 3313 3314 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3315 &entry_req, &entry_rsp); 3316 if (rc) 3317 return rc; 3318 3319 if (!entry_rsp.count) 3320 return NPC_MCAM_ALLOC_FAILED; 3321 3322 entry = entry_rsp.entry; 3323 3324 if (!req->alloc_cntr) 3325 goto write_entry; 3326 3327 /* Now allocate counter */ 3328 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3329 cntr_req.contig = true; 3330 cntr_req.count = 1; 3331 3332 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3333 if (rc) { 3334 /* Free allocated MCAM entry */ 3335 mutex_lock(&mcam->lock); 3336 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 3337 npc_mcam_clear_bit(mcam, entry); 3338 mutex_unlock(&mcam->lock); 3339 return rc; 3340 } 3341 3342 cntr = cntr_rsp.cntr; 3343 3344 write_entry: 3345 mutex_lock(&mcam->lock); 3346 3347 if (is_npc_intf_tx(req->intf)) 3348 nix_intf = pfvf->nix_tx_intf; 3349 else 3350 nix_intf = pfvf->nix_rx_intf; 3351 3352 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3353 &req->entry_data, req->enable_entry); 3354 3355 if (req->alloc_cntr) 3356 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3357 mutex_unlock(&mcam->lock); 3358 3359 rsp->entry = entry; 3360 rsp->cntr = cntr; 3361 3362 return 0; 3363 } 3364 3365 #define GET_KEX_CFG(intf) \ 3366 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3367 3368 #define GET_KEX_FLAGS(ld) \ 3369 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3370 3371 #define GET_KEX_LD(intf, lid, lt, ld) \ 3372 rvu_read64(rvu, BLKADDR_NPC, \ 3373 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3374 3375 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3376 rvu_read64(rvu, BLKADDR_NPC, \ 3377 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3378 3379 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3380 struct npc_get_kex_cfg_rsp *rsp) 3381 { 3382 int lid, lt, ld, fl; 3383 3384 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3385 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3386 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3387 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3388 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3389 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3390 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3391 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3392 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3393 } 3394 } 3395 } 3396 for (ld = 0; ld < NPC_MAX_LD; ld++) 3397 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3398 3399 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3400 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3401 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3402 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3403 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3404 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3405 } 3406 } 3407 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3408 return 0; 3409 } 3410 3411 static int 3412 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 3413 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 3414 { 3415 struct npc_kpu_action0 *act0; 3416 u8 shift_count = 0; 3417 int blkaddr; 3418 u64 val; 3419 3420 if (!var_len_off_mask) 3421 return -EINVAL; 3422 3423 if (var_len_off_mask != 0xff) { 3424 if (shift_dir) 3425 shift_count = __ffs(var_len_off_mask); 3426 else 3427 shift_count = (8 - __fls(var_len_off_mask)); 3428 } 3429 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 3430 if (blkaddr < 0) { 3431 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 3432 return -EINVAL; 3433 } 3434 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 3435 act0 = (struct npc_kpu_action0 *)&val; 3436 act0->var_len_shift = shift_count; 3437 act0->var_len_right = shift_dir; 3438 act0->var_len_mask = var_len_off_mask; 3439 act0->var_len_offset = var_len_off; 3440 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 3441 return 0; 3442 } 3443 3444 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 3445 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 3446 u8 shift_dir) 3447 3448 { 3449 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 3450 int blkaddr, nixlf, rc, intf_mode; 3451 int pf = rvu_get_pf(pcifunc); 3452 u64 rxpkind, txpkind; 3453 u8 cgx_id, lmac_id; 3454 3455 /* use default pkind to disable edsa/higig */ 3456 rxpkind = rvu_npc_get_pkind(rvu, pf); 3457 txpkind = NPC_TX_DEF_PKIND; 3458 intf_mode = NPC_INTF_MODE_DEF; 3459 3460 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 3461 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 3462 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 3463 var_len_off, 3464 var_len_off_mask, 3465 shift_dir); 3466 if (rc) 3467 return rc; 3468 } 3469 rxpkind = pkind; 3470 txpkind = pkind; 3471 } 3472 3473 if (dir & PKIND_RX) { 3474 /* rx pkind set req valid only for cgx mapped PFs */ 3475 if (!is_cgx_config_permitted(rvu, pcifunc)) 3476 return 0; 3477 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3478 3479 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 3480 rxpkind); 3481 if (rc) 3482 return rc; 3483 } 3484 3485 if (dir & PKIND_TX) { 3486 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 3487 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3488 if (rc) 3489 return rc; 3490 3491 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 3492 txpkind); 3493 } 3494 3495 pfvf->intf_mode = intf_mode; 3496 return 0; 3497 } 3498 3499 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 3500 struct msg_rsp *rsp) 3501 { 3502 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 3503 req->dir, req->pkind, req->var_len_off, 3504 req->var_len_off_mask, req->shift_dir); 3505 } 3506 3507 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3508 struct msg_req *req, 3509 struct npc_mcam_read_base_rule_rsp *rsp) 3510 { 3511 struct npc_mcam *mcam = &rvu->hw->mcam; 3512 int index, blkaddr, nixlf, rc = 0; 3513 u16 pcifunc = req->hdr.pcifunc; 3514 struct rvu_pfvf *pfvf; 3515 u8 intf, enable; 3516 3517 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3518 if (blkaddr < 0) 3519 return NPC_MCAM_INVALID_REQ; 3520 3521 /* Return the channel number in case of PF */ 3522 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3523 pfvf = rvu_get_pfvf(rvu, pcifunc); 3524 rsp->entry.kw[0] = pfvf->rx_chan_base; 3525 rsp->entry.kw_mask[0] = 0xFFFULL; 3526 goto out; 3527 } 3528 3529 /* Find the pkt steering rule installed by PF to this VF */ 3530 mutex_lock(&mcam->lock); 3531 for (index = 0; index < mcam->bmap_entries; index++) { 3532 if (mcam->entry2target_pffunc[index] == pcifunc) 3533 goto read_entry; 3534 } 3535 3536 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3537 if (rc < 0) { 3538 mutex_unlock(&mcam->lock); 3539 goto out; 3540 } 3541 /* Read the default ucast entry if there is no pkt steering rule */ 3542 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3543 NIXLF_UCAST_ENTRY); 3544 read_entry: 3545 /* Read the mcam entry */ 3546 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3547 &enable); 3548 mutex_unlock(&mcam->lock); 3549 out: 3550 return rc; 3551 } 3552 3553 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3554 struct npc_mcam_get_stats_req *req, 3555 struct npc_mcam_get_stats_rsp *rsp) 3556 { 3557 struct npc_mcam *mcam = &rvu->hw->mcam; 3558 u16 index, cntr; 3559 int blkaddr; 3560 u64 regval; 3561 u32 bank; 3562 3563 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3564 if (blkaddr < 0) 3565 return NPC_MCAM_INVALID_REQ; 3566 3567 mutex_lock(&mcam->lock); 3568 3569 index = req->entry & (mcam->banksize - 1); 3570 bank = npc_get_bank(mcam, req->entry); 3571 3572 /* read MCAM entry STAT_ACT register */ 3573 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3574 3575 if (!(regval & rvu->hw->npc_stat_ena)) { 3576 rsp->stat_ena = 0; 3577 mutex_unlock(&mcam->lock); 3578 return 0; 3579 } 3580 3581 cntr = regval & 0x1FF; 3582 3583 rsp->stat_ena = 1; 3584 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3585 rsp->stat &= BIT_ULL(48) - 1; 3586 3587 mutex_unlock(&mcam->lock); 3588 3589 return 0; 3590 } 3591