1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "cgx.h" 20 #include "npc_profile.h" 21 22 #define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ 23 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 24 25 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 26 #define NPC_HW_TSTAMP_OFFSET 8 27 #define NPC_KEX_CHAN_MASK 0xFFFULL 28 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 29 30 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 31 32 static const char def_pfl_name[] = "default"; 33 34 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 35 int blkaddr, u16 pcifunc); 36 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 37 u16 pcifunc); 38 39 bool is_npc_intf_tx(u8 intf) 40 { 41 return !!(intf & 0x1); 42 } 43 44 bool is_npc_intf_rx(u8 intf) 45 { 46 return !(intf & 0x1); 47 } 48 49 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 50 { 51 struct rvu_hwinfo *hw = rvu->hw; 52 53 return intf < hw->npc_intfs; 54 } 55 56 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 57 { 58 /* Due to a HW issue in these silicon versions, parse nibble enable 59 * configuration has to be identical for both Rx and Tx interfaces. 60 */ 61 if (is_rvu_96xx_B0(rvu)) 62 return nibble_ena; 63 return 0; 64 } 65 66 static int npc_mcam_verify_pf_func(struct rvu *rvu, 67 struct mcam_entry *entry_data, u8 intf, 68 u16 pcifunc) 69 { 70 u16 pf_func, pf_func_mask; 71 72 if (is_npc_intf_rx(intf)) 73 return 0; 74 75 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 76 NPC_KEX_PF_FUNC_MASK; 77 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 78 79 pf_func = be16_to_cpu((__force __be16)pf_func); 80 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 81 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 82 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 83 return -EINVAL; 84 85 return 0; 86 } 87 88 int npc_mcam_verify_channel(struct rvu *rvu, u16 pcifunc, u8 intf, u16 channel) 89 { 90 int pf = rvu_get_pf(pcifunc); 91 u8 cgx_id, lmac_id; 92 int base = 0, end; 93 94 if (is_npc_intf_tx(intf)) 95 return 0; 96 97 /* return in case of AF installed rules */ 98 if (is_pffunc_af(pcifunc)) 99 return 0; 100 101 if (is_afvf(pcifunc)) { 102 end = rvu_get_num_lbk_chans(); 103 if (end < 0) 104 return -EINVAL; 105 } else { 106 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 107 base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0x0); 108 /* CGX mapped functions has maximum of 16 channels */ 109 end = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0xF); 110 } 111 112 if (channel < base || channel > end) 113 return -EINVAL; 114 115 return 0; 116 } 117 118 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 119 { 120 int blkaddr; 121 u64 val = 0; 122 123 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 124 if (blkaddr < 0) 125 return; 126 127 /* Config CPI base for the PKIND */ 128 val = pkind | 1ULL << 62; 129 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 130 } 131 132 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 133 { 134 struct npc_pkind *pkind = &rvu->hw->pkind; 135 u32 map; 136 int i; 137 138 for (i = 0; i < pkind->rsrc.max; i++) { 139 map = pkind->pfchan_map[i]; 140 if (((map >> 16) & 0x3F) == pf) 141 return i; 142 } 143 return -1; 144 } 145 146 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 147 148 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 149 { 150 int pkind, blkaddr; 151 u64 val; 152 153 pkind = rvu_npc_get_pkind(rvu, pf); 154 if (pkind < 0) { 155 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 156 return -EINVAL; 157 } 158 159 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 160 if (blkaddr < 0) { 161 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 162 return -EINVAL; 163 } 164 165 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 166 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 167 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 168 if (enable) 169 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 170 NPC_HW_TSTAMP_OFFSET); 171 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 172 173 return 0; 174 } 175 176 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 177 int nixlf) 178 { 179 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 180 struct rvu *rvu = hw->rvu; 181 int blkaddr = 0, max = 0; 182 struct rvu_block *block; 183 struct rvu_pfvf *pfvf; 184 185 pfvf = rvu_get_pfvf(rvu, pcifunc); 186 /* Given a PF/VF and NIX LF number calculate the unicast mcam 187 * entry index based on the NIX block assigned to the PF/VF. 188 */ 189 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 190 while (blkaddr) { 191 if (pfvf->nix_blkaddr == blkaddr) 192 break; 193 block = &rvu->hw->block[blkaddr]; 194 max += block->lf.max; 195 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 196 } 197 198 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 199 } 200 201 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 202 u16 pcifunc, int nixlf, int type) 203 { 204 int pf = rvu_get_pf(pcifunc); 205 int index; 206 207 /* Check if this is for a PF */ 208 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 209 /* Reserved entries exclude PF0 */ 210 pf--; 211 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 212 /* Broadcast address matching entry should be first so 213 * that the packet can be replicated to all VFs. 214 */ 215 if (type == NIXLF_BCAST_ENTRY) 216 return index; 217 else if (type == NIXLF_PROMISC_ENTRY) 218 return index + 1; 219 } 220 221 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 222 } 223 224 int npc_get_bank(struct npc_mcam *mcam, int index) 225 { 226 int bank = index / mcam->banksize; 227 228 /* 0,1 & 2,3 banks are combined for this keysize */ 229 if (mcam->keysize == NPC_MCAM_KEY_X2) 230 return bank ? 2 : 0; 231 232 return bank; 233 } 234 235 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 236 int blkaddr, int index) 237 { 238 int bank = npc_get_bank(mcam, index); 239 u64 cfg; 240 241 index &= (mcam->banksize - 1); 242 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 243 return (cfg & 1); 244 } 245 246 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 247 int blkaddr, int index, bool enable) 248 { 249 int bank = npc_get_bank(mcam, index); 250 int actbank = bank; 251 252 index &= (mcam->banksize - 1); 253 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 254 rvu_write64(rvu, blkaddr, 255 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 256 enable ? 1 : 0); 257 } 258 } 259 260 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 261 int blkaddr, int index) 262 { 263 int bank = npc_get_bank(mcam, index); 264 int actbank = bank; 265 266 index &= (mcam->banksize - 1); 267 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 268 rvu_write64(rvu, blkaddr, 269 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 270 rvu_write64(rvu, blkaddr, 271 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 272 273 rvu_write64(rvu, blkaddr, 274 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 275 rvu_write64(rvu, blkaddr, 276 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 277 278 rvu_write64(rvu, blkaddr, 279 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 280 rvu_write64(rvu, blkaddr, 281 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 282 } 283 } 284 285 static void npc_get_keyword(struct mcam_entry *entry, int idx, 286 u64 *cam0, u64 *cam1) 287 { 288 u64 kw_mask = 0x00; 289 290 #define CAM_MASK(n) (BIT_ULL(n) - 1) 291 292 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 293 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 294 * 295 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 296 */ 297 switch (idx) { 298 case 0: 299 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 300 *cam1 = entry->kw[0]; 301 kw_mask = entry->kw_mask[0]; 302 break; 303 case 1: 304 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 305 *cam1 = entry->kw[1] & CAM_MASK(48); 306 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 307 break; 308 case 2: 309 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 310 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 311 */ 312 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 313 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 314 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 315 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 316 break; 317 case 3: 318 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 319 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 320 */ 321 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 322 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 323 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 324 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 325 break; 326 case 4: 327 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 328 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 329 */ 330 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 331 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 332 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 333 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 334 break; 335 case 5: 336 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 337 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 338 */ 339 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 340 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 341 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 342 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 343 break; 344 case 6: 345 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 346 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 347 */ 348 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 349 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 350 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 351 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 352 break; 353 case 7: 354 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 355 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 356 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 357 break; 358 } 359 360 *cam1 &= kw_mask; 361 *cam0 = ~*cam1 & kw_mask; 362 } 363 364 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 365 u64 cam0, u64 cam1) 366 { 367 /* Similar to npc_get_keyword, but fills mcam_entry structure from 368 * CAM registers. 369 */ 370 switch (idx) { 371 case 0: 372 entry->kw[0] = cam1; 373 entry->kw_mask[0] = cam1 ^ cam0; 374 break; 375 case 1: 376 entry->kw[1] = cam1; 377 entry->kw_mask[1] = cam1 ^ cam0; 378 break; 379 case 2: 380 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 381 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 382 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 383 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 384 break; 385 case 3: 386 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 387 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 388 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 389 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 390 break; 391 case 4: 392 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 393 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 394 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 395 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 396 break; 397 case 5: 398 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 399 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 400 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 401 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 402 break; 403 case 6: 404 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 405 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 406 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 407 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 408 break; 409 case 7: 410 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 411 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 412 break; 413 } 414 } 415 416 static void npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 417 int blkaddr, int index, 418 struct mcam_entry *entry) 419 { 420 u16 owner, target_func; 421 struct rvu_pfvf *pfvf; 422 int bank, nixlf; 423 u64 rx_action; 424 425 owner = mcam->entry2pfvf_map[index]; 426 target_func = (entry->action >> 4) & 0xffff; 427 /* return incase target is PF or LBK or rule owner is not PF */ 428 if (is_afvf(target_func) || (owner & RVU_PFVF_FUNC_MASK) || 429 !(target_func & RVU_PFVF_FUNC_MASK)) 430 return; 431 432 pfvf = rvu_get_pfvf(rvu, target_func); 433 mcam->entry2target_pffunc[index] = target_func; 434 /* return if nixlf is not attached or initialized */ 435 if (!is_nixlf_attached(rvu, target_func) || !pfvf->def_ucast_rule) 436 return; 437 438 /* get VF ucast entry rule */ 439 nix_get_nixlf(rvu, target_func, &nixlf, NULL); 440 index = npc_get_nixlf_mcam_index(mcam, target_func, 441 nixlf, NIXLF_UCAST_ENTRY); 442 bank = npc_get_bank(mcam, index); 443 index &= (mcam->banksize - 1); 444 445 rx_action = rvu_read64(rvu, blkaddr, 446 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 447 if (rx_action) 448 entry->action = rx_action; 449 } 450 451 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 452 int blkaddr, int index, u8 intf, 453 struct mcam_entry *entry, bool enable) 454 { 455 int bank = npc_get_bank(mcam, index); 456 int kw = 0, actbank, actindex; 457 u64 cam0, cam1; 458 459 actbank = bank; /* Save bank id, to set action later on */ 460 actindex = index; 461 index &= (mcam->banksize - 1); 462 463 /* Disable before mcam entry update */ 464 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 465 466 /* Clear mcam entry to avoid writes being suppressed by NPC */ 467 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 468 469 /* CAM1 takes the comparison value and 470 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 471 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 472 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 473 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 474 */ 475 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 476 /* Interface should be set in all banks */ 477 rvu_write64(rvu, blkaddr, 478 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 479 intf); 480 rvu_write64(rvu, blkaddr, 481 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 482 ~intf & 0x3); 483 484 /* Set the match key */ 485 npc_get_keyword(entry, kw, &cam0, &cam1); 486 rvu_write64(rvu, blkaddr, 487 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 488 rvu_write64(rvu, blkaddr, 489 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 490 491 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 492 rvu_write64(rvu, blkaddr, 493 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 494 rvu_write64(rvu, blkaddr, 495 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 496 } 497 498 /* copy VF default entry action to the VF mcam entry */ 499 if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) 500 npc_get_default_entry_action(rvu, mcam, blkaddr, actindex, 501 entry); 502 503 /* Set 'action' */ 504 rvu_write64(rvu, blkaddr, 505 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 506 507 /* Set TAG 'action' */ 508 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 509 entry->vtag_action); 510 511 /* Enable the entry */ 512 if (enable) 513 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 514 } 515 516 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 517 int blkaddr, u16 src, 518 struct mcam_entry *entry, u8 *intf, u8 *ena) 519 { 520 int sbank = npc_get_bank(mcam, src); 521 int bank, kw = 0; 522 u64 cam0, cam1; 523 524 src &= (mcam->banksize - 1); 525 bank = sbank; 526 527 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 528 cam1 = rvu_read64(rvu, blkaddr, 529 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 530 cam0 = rvu_read64(rvu, blkaddr, 531 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 532 npc_fill_entryword(entry, kw, cam0, cam1); 533 534 cam1 = rvu_read64(rvu, blkaddr, 535 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 536 cam0 = rvu_read64(rvu, blkaddr, 537 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 538 npc_fill_entryword(entry, kw + 1, cam0, cam1); 539 } 540 541 entry->action = rvu_read64(rvu, blkaddr, 542 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 543 entry->vtag_action = 544 rvu_read64(rvu, blkaddr, 545 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 546 *intf = rvu_read64(rvu, blkaddr, 547 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 548 *ena = rvu_read64(rvu, blkaddr, 549 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 550 } 551 552 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 553 int blkaddr, u16 src, u16 dest) 554 { 555 int dbank = npc_get_bank(mcam, dest); 556 int sbank = npc_get_bank(mcam, src); 557 u64 cfg, sreg, dreg; 558 int bank, i; 559 560 src &= (mcam->banksize - 1); 561 dest &= (mcam->banksize - 1); 562 563 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 564 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 565 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 566 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 567 for (i = 0; i < 6; i++) { 568 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 569 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 570 } 571 } 572 573 /* Copy action */ 574 cfg = rvu_read64(rvu, blkaddr, 575 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 576 rvu_write64(rvu, blkaddr, 577 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 578 579 /* Copy TAG action */ 580 cfg = rvu_read64(rvu, blkaddr, 581 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 582 rvu_write64(rvu, blkaddr, 583 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 584 585 /* Enable or disable */ 586 cfg = rvu_read64(rvu, blkaddr, 587 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 588 rvu_write64(rvu, blkaddr, 589 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 590 } 591 592 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 593 int blkaddr, int index) 594 { 595 int bank = npc_get_bank(mcam, index); 596 597 index &= (mcam->banksize - 1); 598 return rvu_read64(rvu, blkaddr, 599 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 600 } 601 602 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 603 int nixlf, u64 chan, u8 *mac_addr) 604 { 605 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 606 struct npc_install_flow_req req = { 0 }; 607 struct npc_install_flow_rsp rsp = { 0 }; 608 struct npc_mcam *mcam = &rvu->hw->mcam; 609 struct nix_rx_action action; 610 int blkaddr, index; 611 612 /* AF's VFs work in promiscuous mode */ 613 if (is_afvf(pcifunc)) 614 return; 615 616 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 617 if (blkaddr < 0) 618 return; 619 620 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 621 nixlf, NIXLF_UCAST_ENTRY); 622 623 /* Don't change the action if entry is already enabled 624 * Otherwise RSS action may get overwritten. 625 */ 626 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 627 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 628 blkaddr, index); 629 } else { 630 *(u64 *)&action = 0x00; 631 action.op = NIX_RX_ACTIONOP_UCAST; 632 action.pf_func = pcifunc; 633 } 634 635 req.default_rule = 1; 636 ether_addr_copy(req.packet.dmac, mac_addr); 637 eth_broadcast_addr((u8 *)&req.mask.dmac); 638 req.features = BIT_ULL(NPC_DMAC); 639 req.channel = chan; 640 req.intf = pfvf->nix_rx_intf; 641 req.op = action.op; 642 req.hdr.pcifunc = 0; /* AF is requester */ 643 req.vf = action.pf_func; 644 req.index = action.index; 645 req.match_id = action.match_id; 646 req.flow_key_alg = action.flow_key_alg; 647 648 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 649 } 650 651 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 652 int nixlf, u64 chan, u8 chan_cnt, 653 bool allmulti) 654 { 655 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 656 struct npc_install_flow_req req = { 0 }; 657 struct npc_install_flow_rsp rsp = { 0 }; 658 struct npc_mcam *mcam = &rvu->hw->mcam; 659 int blkaddr, ucast_idx, index; 660 u8 mac_addr[ETH_ALEN] = { 0 }; 661 struct nix_rx_action action; 662 u64 relaxed_mask; 663 664 /* Only PF or AF VF can add a promiscuous entry */ 665 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) 666 return; 667 668 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 669 if (blkaddr < 0) 670 return; 671 672 *(u64 *)&action = 0x00; 673 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 674 nixlf, NIXLF_PROMISC_ENTRY); 675 676 /* If the corresponding PF's ucast action is RSS, 677 * use the same action for promisc also 678 */ 679 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 680 nixlf, NIXLF_UCAST_ENTRY); 681 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 682 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 683 blkaddr, ucast_idx); 684 685 if (action.op != NIX_RX_ACTIONOP_RSS) { 686 *(u64 *)&action = 0x00; 687 action.op = NIX_RX_ACTIONOP_UCAST; 688 action.pf_func = pcifunc; 689 } 690 691 if (allmulti) { 692 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 693 ether_addr_copy(req.packet.dmac, mac_addr); 694 ether_addr_copy(req.mask.dmac, mac_addr); 695 req.features = BIT_ULL(NPC_DMAC); 696 } 697 698 req.chan_mask = 0xFFFU; 699 if (chan_cnt > 1) { 700 if (!is_power_of_2(chan_cnt)) { 701 dev_err(rvu->dev, 702 "%s: channel count more than 1, must be power of 2\n", __func__); 703 return; 704 } 705 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 706 ilog2(chan_cnt)); 707 req.chan_mask &= relaxed_mask; 708 } 709 710 req.channel = chan; 711 req.intf = pfvf->nix_rx_intf; 712 req.entry = index; 713 req.op = action.op; 714 req.hdr.pcifunc = 0; /* AF is requester */ 715 req.vf = pcifunc; 716 req.index = action.index; 717 req.match_id = action.match_id; 718 req.flow_key_alg = action.flow_key_alg; 719 720 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 721 } 722 723 static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, 724 int nixlf, bool enable) 725 { 726 struct npc_mcam *mcam = &rvu->hw->mcam; 727 int blkaddr, index; 728 729 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 730 if (blkaddr < 0) 731 return; 732 733 /* Only PF's have a promiscuous entry */ 734 if (pcifunc & RVU_PFVF_FUNC_MASK) 735 return; 736 737 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 738 nixlf, NIXLF_PROMISC_ENTRY); 739 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 740 } 741 742 void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 743 { 744 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); 745 } 746 747 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 748 { 749 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); 750 } 751 752 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 753 int nixlf, u64 chan) 754 { 755 struct rvu_pfvf *pfvf; 756 struct npc_install_flow_req req = { 0 }; 757 struct npc_install_flow_rsp rsp = { 0 }; 758 struct npc_mcam *mcam = &rvu->hw->mcam; 759 struct rvu_hwinfo *hw = rvu->hw; 760 int blkaddr, index; 761 u32 req_index = 0; 762 u8 op; 763 764 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 765 if (blkaddr < 0) 766 return; 767 768 /* Skip LBK VFs */ 769 if (is_afvf(pcifunc)) 770 return; 771 772 /* If pkt replication is not supported, 773 * then only PF is allowed to add a bcast match entry. 774 */ 775 if (!hw->cap.nix_rx_multicast && pcifunc & RVU_PFVF_FUNC_MASK) 776 return; 777 778 /* Get 'pcifunc' of PF device */ 779 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 780 pfvf = rvu_get_pfvf(rvu, pcifunc); 781 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 782 nixlf, NIXLF_BCAST_ENTRY); 783 784 if (!hw->cap.nix_rx_multicast) { 785 /* Early silicon doesn't support pkt replication, 786 * so install entry with UCAST action, so that PF 787 * receives all broadcast packets. 788 */ 789 op = NIX_RX_ACTIONOP_UCAST; 790 } else { 791 op = NIX_RX_ACTIONOP_MCAST; 792 req_index = pfvf->bcast_mce_idx; 793 } 794 795 eth_broadcast_addr((u8 *)&req.packet.dmac); 796 eth_broadcast_addr((u8 *)&req.mask.dmac); 797 req.features = BIT_ULL(NPC_DMAC); 798 req.channel = chan; 799 req.intf = pfvf->nix_rx_intf; 800 req.entry = index; 801 req.op = op; 802 req.hdr.pcifunc = 0; /* AF is requester */ 803 req.vf = pcifunc; 804 req.index = req_index; 805 806 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 807 } 808 809 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable) 810 { 811 struct npc_mcam *mcam = &rvu->hw->mcam; 812 int blkaddr, index; 813 814 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 815 if (blkaddr < 0) 816 return; 817 818 /* Get 'pcifunc' of PF device */ 819 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 820 821 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 0, NIXLF_BCAST_ENTRY); 822 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 823 } 824 825 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 826 int blkaddr, u16 pcifunc, u64 rx_action) 827 { 828 int actindex, index, bank; 829 bool enable; 830 831 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 832 return; 833 834 mutex_lock(&mcam->lock); 835 for (index = 0; index < mcam->bmap_entries; index++) { 836 if (mcam->entry2target_pffunc[index] == pcifunc) { 837 bank = npc_get_bank(mcam, index); 838 actindex = index; 839 index &= (mcam->banksize - 1); 840 841 /* read vf flow entry enable status */ 842 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 843 actindex); 844 /* disable before mcam entry update */ 845 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 846 false); 847 /* update 'action' */ 848 rvu_write64(rvu, blkaddr, 849 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 850 rx_action); 851 if (enable) 852 npc_enable_mcam_entry(rvu, mcam, blkaddr, 853 actindex, true); 854 } 855 } 856 mutex_unlock(&mcam->lock); 857 } 858 859 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 860 int group, int alg_idx, int mcam_index) 861 { 862 struct npc_mcam *mcam = &rvu->hw->mcam; 863 struct nix_rx_action action; 864 int blkaddr, index, bank; 865 struct rvu_pfvf *pfvf; 866 867 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 868 if (blkaddr < 0) 869 return; 870 871 /* Check if this is for reserved default entry */ 872 if (mcam_index < 0) { 873 if (group != DEFAULT_RSS_CONTEXT_GROUP) 874 return; 875 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 876 nixlf, NIXLF_UCAST_ENTRY); 877 } else { 878 /* TODO: validate this mcam index */ 879 index = mcam_index; 880 } 881 882 if (index >= mcam->total_entries) 883 return; 884 885 bank = npc_get_bank(mcam, index); 886 index &= (mcam->banksize - 1); 887 888 *(u64 *)&action = rvu_read64(rvu, blkaddr, 889 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 890 /* Ignore if no action was set earlier */ 891 if (!*(u64 *)&action) 892 return; 893 894 action.op = NIX_RX_ACTIONOP_RSS; 895 action.pf_func = pcifunc; 896 action.index = group; 897 action.flow_key_alg = alg_idx; 898 899 rvu_write64(rvu, blkaddr, 900 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 901 902 /* update the VF flow rule action with the VF default entry action */ 903 if (mcam_index < 0) 904 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 905 *(u64 *)&action); 906 907 /* update the action change in default rule */ 908 pfvf = rvu_get_pfvf(rvu, pcifunc); 909 if (pfvf->def_ucast_rule) 910 pfvf->def_ucast_rule->rx_action = action; 911 912 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 913 nixlf, NIXLF_PROMISC_ENTRY); 914 915 /* If PF's promiscuous entry is enabled, 916 * Set RSS action for that entry as well 917 */ 918 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 919 bank = npc_get_bank(mcam, index); 920 index &= (mcam->banksize - 1); 921 922 rvu_write64(rvu, blkaddr, 923 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 924 *(u64 *)&action); 925 } 926 } 927 928 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 929 int nixlf, bool enable) 930 { 931 struct npc_mcam *mcam = &rvu->hw->mcam; 932 struct nix_rx_action action; 933 int index, bank, blkaddr; 934 935 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 936 if (blkaddr < 0) 937 return; 938 939 /* Ucast MCAM match entry of this PF/VF */ 940 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 941 nixlf, NIXLF_UCAST_ENTRY); 942 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 943 944 /* For PF, ena/dis promisc and bcast MCAM match entries. 945 * For VFs add/delete from bcast list when RX multicast 946 * feature is present. 947 */ 948 if (pcifunc & RVU_PFVF_FUNC_MASK && !rvu->hw->cap.nix_rx_multicast) 949 return; 950 951 /* For bcast, enable/disable only if it's action is not 952 * packet replication, incase if action is replication 953 * then this PF/VF's nixlf is removed from bcast replication 954 * list. 955 */ 956 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 957 nixlf, NIXLF_BCAST_ENTRY); 958 bank = npc_get_bank(mcam, index); 959 *(u64 *)&action = rvu_read64(rvu, blkaddr, 960 NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); 961 962 /* VFs will not have BCAST entry */ 963 if (action.op != NIX_RX_ACTIONOP_MCAST && 964 !(pcifunc & RVU_PFVF_FUNC_MASK)) { 965 npc_enable_mcam_entry(rvu, mcam, 966 blkaddr, index, enable); 967 } else { 968 nix_update_bcast_mce_list(rvu, pcifunc, enable); 969 /* Enable PF's BCAST entry for packet replication */ 970 rvu_npc_enable_bcast_entry(rvu, pcifunc, enable); 971 } 972 973 if (enable) 974 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); 975 else 976 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); 977 } 978 979 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 980 { 981 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 982 } 983 984 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 985 { 986 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 987 } 988 989 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 990 { 991 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 992 struct npc_mcam *mcam = &rvu->hw->mcam; 993 struct rvu_npc_mcam_rule *rule, *tmp; 994 int blkaddr; 995 996 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 997 if (blkaddr < 0) 998 return; 999 1000 mutex_lock(&mcam->lock); 1001 1002 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1003 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1004 if (is_npc_intf_rx(rule->intf) && 1005 rule->rx_action.pf_func == pcifunc) { 1006 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1007 rule->entry, false); 1008 rule->enable = false; 1009 /* Indicate that default rule is disabled */ 1010 if (rule->default_rule) { 1011 pfvf->def_ucast_rule = NULL; 1012 list_del(&rule->list); 1013 kfree(rule); 1014 } 1015 } 1016 } 1017 1018 mutex_unlock(&mcam->lock); 1019 1020 npc_mcam_disable_flows(rvu, pcifunc); 1021 1022 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1023 } 1024 1025 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1026 { 1027 struct npc_mcam *mcam = &rvu->hw->mcam; 1028 struct rvu_npc_mcam_rule *rule, *tmp; 1029 int blkaddr; 1030 1031 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1032 if (blkaddr < 0) 1033 return; 1034 1035 mutex_lock(&mcam->lock); 1036 1037 /* Free all MCAM entries owned by this 'pcifunc' */ 1038 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1039 1040 /* Free all MCAM counters owned by this 'pcifunc' */ 1041 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1042 1043 /* Delete MCAM entries owned by this 'pcifunc' */ 1044 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1045 if (rule->owner == pcifunc && !rule->default_rule) { 1046 list_del(&rule->list); 1047 kfree(rule); 1048 } 1049 } 1050 1051 mutex_unlock(&mcam->lock); 1052 1053 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1054 } 1055 1056 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1057 rvu_write64(rvu, blkaddr, \ 1058 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1059 1060 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1061 rvu_write64(rvu, blkaddr, \ 1062 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1063 1064 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1065 struct npc_mcam_kex *mkex, u8 intf) 1066 { 1067 int lid, lt, ld, fl; 1068 1069 if (is_npc_intf_tx(intf)) 1070 return; 1071 1072 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1073 mkex->keyx_cfg[NIX_INTF_RX]); 1074 1075 /* Program LDATA */ 1076 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1077 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1078 for (ld = 0; ld < NPC_MAX_LD; ld++) 1079 SET_KEX_LD(intf, lid, lt, ld, 1080 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1081 [lid][lt][ld]); 1082 } 1083 } 1084 /* Program LFLAGS */ 1085 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1086 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1087 SET_KEX_LDFLAGS(intf, ld, fl, 1088 mkex->intf_ld_flags[NIX_INTF_RX] 1089 [ld][fl]); 1090 } 1091 } 1092 1093 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1094 struct npc_mcam_kex *mkex, u8 intf) 1095 { 1096 int lid, lt, ld, fl; 1097 1098 if (is_npc_intf_rx(intf)) 1099 return; 1100 1101 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1102 mkex->keyx_cfg[NIX_INTF_TX]); 1103 1104 /* Program LDATA */ 1105 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1106 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1107 for (ld = 0; ld < NPC_MAX_LD; ld++) 1108 SET_KEX_LD(intf, lid, lt, ld, 1109 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1110 [lid][lt][ld]); 1111 } 1112 } 1113 /* Program LFLAGS */ 1114 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1115 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1116 SET_KEX_LDFLAGS(intf, ld, fl, 1117 mkex->intf_ld_flags[NIX_INTF_TX] 1118 [ld][fl]); 1119 } 1120 } 1121 1122 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1123 struct npc_mcam_kex *mkex) 1124 { 1125 struct rvu_hwinfo *hw = rvu->hw; 1126 u8 intf; 1127 int ld; 1128 1129 for (ld = 0; ld < NPC_MAX_LD; ld++) 1130 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1131 mkex->kex_ld_flags[ld]); 1132 1133 for (intf = 0; intf < hw->npc_intfs; intf++) { 1134 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1135 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1136 } 1137 } 1138 1139 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1140 u64 *size) 1141 { 1142 u64 prfl_addr, prfl_sz; 1143 1144 if (!rvu->fwdata) 1145 return -EINVAL; 1146 1147 prfl_addr = rvu->fwdata->mcam_addr; 1148 prfl_sz = rvu->fwdata->mcam_sz; 1149 1150 if (!prfl_addr || !prfl_sz) 1151 return -EINVAL; 1152 1153 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1154 if (!(*prfl_img_addr)) 1155 return -ENOMEM; 1156 1157 *size = prfl_sz; 1158 1159 return 0; 1160 } 1161 1162 /* strtoull of "mkexprof" with base:36 */ 1163 #define MKEX_END_SIGN 0xdeadbeef 1164 1165 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1166 const char *mkex_profile) 1167 { 1168 struct device *dev = &rvu->pdev->dev; 1169 struct npc_mcam_kex *mcam_kex; 1170 void __iomem *mkex_prfl_addr = NULL; 1171 u64 prfl_sz; 1172 int ret; 1173 1174 /* If user not selected mkex profile */ 1175 if (rvu->kpu_fwdata_sz || 1176 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1177 goto program_mkex; 1178 1179 /* Setting up the mapping for mkex profile image */ 1180 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1181 if (ret < 0) 1182 goto program_mkex; 1183 1184 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1185 1186 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1187 /* Compare with mkex mod_param name string */ 1188 if (mcam_kex->mkex_sign == MKEX_SIGN && 1189 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1190 /* Due to an errata (35786) in A0/B0 pass silicon, 1191 * parse nibble enable configuration has to be 1192 * identical for both Rx and Tx interfaces. 1193 */ 1194 if (!is_rvu_96xx_B0(rvu) || 1195 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1196 rvu->kpu.mkex = mcam_kex; 1197 goto program_mkex; 1198 } 1199 1200 mcam_kex++; 1201 prfl_sz -= sizeof(struct npc_mcam_kex); 1202 } 1203 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1204 1205 program_mkex: 1206 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1207 /* Program selected mkex profile */ 1208 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1209 if (mkex_prfl_addr) 1210 iounmap(mkex_prfl_addr); 1211 } 1212 1213 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1214 const struct npc_kpu_profile_action *kpuaction, 1215 int kpu, int entry, bool pkind) 1216 { 1217 struct npc_kpu_action0 action0 = {0}; 1218 struct npc_kpu_action1 action1 = {0}; 1219 u64 reg; 1220 1221 action1.errlev = kpuaction->errlev; 1222 action1.errcode = kpuaction->errcode; 1223 action1.dp0_offset = kpuaction->dp0_offset; 1224 action1.dp1_offset = kpuaction->dp1_offset; 1225 action1.dp2_offset = kpuaction->dp2_offset; 1226 1227 if (pkind) 1228 reg = NPC_AF_PKINDX_ACTION1(entry); 1229 else 1230 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1231 1232 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1233 1234 action0.byp_count = kpuaction->bypass_count; 1235 action0.capture_ena = kpuaction->cap_ena; 1236 action0.parse_done = kpuaction->parse_done; 1237 action0.next_state = kpuaction->next_state; 1238 action0.capture_lid = kpuaction->lid; 1239 action0.capture_ltype = kpuaction->ltype; 1240 action0.capture_flags = kpuaction->flags; 1241 action0.ptr_advance = kpuaction->ptr_advance; 1242 action0.var_len_offset = kpuaction->offset; 1243 action0.var_len_mask = kpuaction->mask; 1244 action0.var_len_right = kpuaction->right; 1245 action0.var_len_shift = kpuaction->shift; 1246 1247 if (pkind) 1248 reg = NPC_AF_PKINDX_ACTION0(entry); 1249 else 1250 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1251 1252 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1253 } 1254 1255 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1256 const struct npc_kpu_profile_cam *kpucam, 1257 int kpu, int entry) 1258 { 1259 struct npc_kpu_cam cam0 = {0}; 1260 struct npc_kpu_cam cam1 = {0}; 1261 1262 cam1.state = kpucam->state & kpucam->state_mask; 1263 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1264 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1265 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1266 1267 cam0.state = ~kpucam->state & kpucam->state_mask; 1268 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1269 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1270 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1271 1272 rvu_write64(rvu, blkaddr, 1273 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1274 rvu_write64(rvu, blkaddr, 1275 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1276 } 1277 1278 static inline u64 enable_mask(int count) 1279 { 1280 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1281 } 1282 1283 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1284 const struct npc_kpu_profile *profile) 1285 { 1286 int entry, num_entries, max_entries; 1287 u64 entry_mask; 1288 1289 if (profile->cam_entries != profile->action_entries) { 1290 dev_err(rvu->dev, 1291 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1292 kpu, profile->cam_entries, profile->action_entries); 1293 } 1294 1295 max_entries = rvu->hw->npc_kpu_entries; 1296 1297 /* Program CAM match entries for previous KPU extracted data */ 1298 num_entries = min_t(int, profile->cam_entries, max_entries); 1299 for (entry = 0; entry < num_entries; entry++) 1300 npc_config_kpucam(rvu, blkaddr, 1301 &profile->cam[entry], kpu, entry); 1302 1303 /* Program this KPU's actions */ 1304 num_entries = min_t(int, profile->action_entries, max_entries); 1305 for (entry = 0; entry < num_entries; entry++) 1306 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1307 kpu, entry, false); 1308 1309 /* Enable all programmed entries */ 1310 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1311 entry_mask = enable_mask(num_entries); 1312 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1313 if (!rvu->kpu.custom) 1314 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1315 rvu_write64(rvu, blkaddr, 1316 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1317 if (num_entries > 64) { 1318 rvu_write64(rvu, blkaddr, 1319 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1320 enable_mask(num_entries - 64)); 1321 } 1322 1323 /* Enable this KPU */ 1324 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1325 } 1326 1327 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1328 { 1329 profile->custom = 0; 1330 profile->name = def_pfl_name; 1331 profile->version = NPC_KPU_PROFILE_VER; 1332 profile->ikpu = ikpu_action_entries; 1333 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1334 profile->kpu = npc_kpu_profiles; 1335 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1336 profile->lt_def = &npc_lt_defaults; 1337 profile->mkex = &npc_mkex_default; 1338 1339 return 0; 1340 } 1341 1342 static int npc_apply_custom_kpu(struct rvu *rvu, 1343 struct npc_kpu_profile_adapter *profile) 1344 { 1345 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1346 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1347 struct npc_kpu_profile_action *action; 1348 struct npc_kpu_profile_cam *cam; 1349 struct npc_kpu_fwdata *fw_kpu; 1350 int entries; 1351 u16 kpu, entry; 1352 1353 if (rvu->kpu_fwdata_sz < hdr_sz) { 1354 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1355 return -EINVAL; 1356 } 1357 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1358 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1359 fw->signature); 1360 return -EINVAL; 1361 } 1362 /* Verify if the using known profile structure */ 1363 if (NPC_KPU_VER_MAJ(profile->version) > 1364 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1365 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1366 NPC_KPU_VER_MAJ(profile->version), 1367 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1368 return -EINVAL; 1369 } 1370 /* Verify if profile is aligned with the required kernel changes */ 1371 if (NPC_KPU_VER_MIN(profile->version) < 1372 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1373 dev_warn(rvu->dev, 1374 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1375 NPC_KPU_VER_MAJ(profile->version), 1376 NPC_KPU_VER_MIN(profile->version), 1377 NPC_KPU_VER_PATCH(profile->version), 1378 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1379 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1380 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1381 return -EINVAL; 1382 } 1383 /* Verify if profile fits the HW */ 1384 if (fw->kpus > profile->kpus) { 1385 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1386 profile->kpus); 1387 return -EINVAL; 1388 } 1389 1390 profile->custom = 1; 1391 profile->name = fw->name; 1392 profile->version = le64_to_cpu(fw->version); 1393 profile->mkex = &fw->mkex; 1394 profile->lt_def = &fw->lt_def; 1395 1396 for (kpu = 0; kpu < fw->kpus; kpu++) { 1397 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1398 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1399 dev_warn(rvu->dev, 1400 "Too many custom entries on KPU%d: %d > %d\n", 1401 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1402 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1403 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1404 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1405 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1406 offset += fw_kpu->entries * sizeof(*action); 1407 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1408 dev_warn(rvu->dev, 1409 "Profile size mismatch on KPU%i parsing.\n", 1410 kpu + 1); 1411 return -EINVAL; 1412 } 1413 for (entry = 0; entry < entries; entry++) { 1414 profile->kpu[kpu].cam[entry] = cam[entry]; 1415 profile->kpu[kpu].action[entry] = action[entry]; 1416 } 1417 } 1418 1419 return 0; 1420 } 1421 1422 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1423 u64 prfl_sz, const char *kpu_profile) 1424 { 1425 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1426 int rc = -EINVAL; 1427 1428 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1429 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1430 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1431 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1432 kpu_profile); 1433 rvu->kpu_fwdata = kpu_data; 1434 rvu->kpu_fwdata_sz = prfl_sz; 1435 rvu->kpu_prfl_addr = prfl_addr; 1436 rc = 0; 1437 } 1438 1439 return rc; 1440 } 1441 1442 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1443 const char *kpu_profile) 1444 { 1445 struct npc_coalesced_kpu_prfl *img_data = NULL; 1446 int i = 0, rc = -EINVAL; 1447 void __iomem *kpu_prfl_addr; 1448 u16 offset; 1449 1450 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1451 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1452 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1453 /* Loaded profile is a single KPU profile. */ 1454 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1455 prfl_sz, kpu_profile); 1456 goto done; 1457 } 1458 1459 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1460 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1461 (img_data->num_prfl * sizeof(uint16_t)); 1462 /* Check if mapped image is coalesced image. */ 1463 while (i < img_data->num_prfl) { 1464 /* Profile image offsets are rounded up to next 8 multiple.*/ 1465 offset = ALIGN_8B_CEIL(offset); 1466 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1467 offset); 1468 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1469 img_data->prfl_sz[i], kpu_profile); 1470 if (!rc) 1471 break; 1472 /* Calculating offset of profile image based on profile size.*/ 1473 offset += img_data->prfl_sz[i]; 1474 i++; 1475 } 1476 done: 1477 return rc; 1478 } 1479 1480 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1481 { 1482 int ret = -EINVAL; 1483 u64 prfl_sz; 1484 1485 /* Setting up the mapping for NPC profile image */ 1486 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1487 if (ret < 0) 1488 goto done; 1489 1490 /* Detect if profile is coalesced or single KPU profile and load */ 1491 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1492 if (ret == 0) 1493 goto done; 1494 1495 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1496 if (rvu->kpu_prfl_addr) { 1497 iounmap(rvu->kpu_prfl_addr); 1498 rvu->kpu_prfl_addr = NULL; 1499 rvu->kpu_fwdata_sz = 0; 1500 rvu->kpu_fwdata = NULL; 1501 } 1502 1503 done: 1504 return ret; 1505 } 1506 1507 static void npc_load_kpu_profile(struct rvu *rvu) 1508 { 1509 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1510 const char *kpu_profile = rvu->kpu_pfl_name; 1511 const struct firmware *fw = NULL; 1512 bool retry_fwdb = false; 1513 1514 /* If user not specified profile customization */ 1515 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1516 goto revert_to_default; 1517 /* First prepare default KPU, then we'll customize top entries. */ 1518 npc_prepare_default_kpu(profile); 1519 1520 /* Order of preceedence for load loading NPC profile (high to low) 1521 * Firmware binary in filesystem. 1522 * Firmware database method. 1523 * Default KPU profile. 1524 */ 1525 if (!request_firmware(&fw, kpu_profile, rvu->dev)) { 1526 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1527 kpu_profile); 1528 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1529 if (rvu->kpu_fwdata) { 1530 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1531 rvu->kpu_fwdata_sz = fw->size; 1532 } 1533 release_firmware(fw); 1534 retry_fwdb = true; 1535 goto program_kpu; 1536 } 1537 1538 load_image_fwdb: 1539 /* Loading the KPU profile using firmware database */ 1540 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1541 goto revert_to_default; 1542 1543 program_kpu: 1544 /* Apply profile customization if firmware was loaded. */ 1545 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1546 /* If image from firmware filesystem fails to load or invalid 1547 * retry with firmware database method. 1548 */ 1549 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1550 /* Loading image from firmware database failed. */ 1551 if (rvu->kpu_prfl_addr) { 1552 iounmap(rvu->kpu_prfl_addr); 1553 rvu->kpu_prfl_addr = NULL; 1554 } else { 1555 kfree(rvu->kpu_fwdata); 1556 } 1557 rvu->kpu_fwdata = NULL; 1558 rvu->kpu_fwdata_sz = 0; 1559 if (retry_fwdb) { 1560 retry_fwdb = false; 1561 goto load_image_fwdb; 1562 } 1563 } 1564 1565 dev_warn(rvu->dev, 1566 "Can't load KPU profile %s. Using default.\n", 1567 kpu_profile); 1568 kfree(rvu->kpu_fwdata); 1569 rvu->kpu_fwdata = NULL; 1570 goto revert_to_default; 1571 } 1572 1573 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1574 profile->name, NPC_KPU_VER_MAJ(profile->version), 1575 NPC_KPU_VER_MIN(profile->version), 1576 NPC_KPU_VER_PATCH(profile->version)); 1577 1578 return; 1579 1580 revert_to_default: 1581 npc_prepare_default_kpu(profile); 1582 } 1583 1584 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1585 { 1586 struct rvu_hwinfo *hw = rvu->hw; 1587 int num_pkinds, num_kpus, idx; 1588 struct npc_pkind *pkind; 1589 1590 /* Disable all KPUs and their entries */ 1591 for (idx = 0; idx < hw->npc_kpus; idx++) { 1592 rvu_write64(rvu, blkaddr, 1593 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1594 rvu_write64(rvu, blkaddr, 1595 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1596 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1597 } 1598 1599 /* Load and customize KPU profile. */ 1600 npc_load_kpu_profile(rvu); 1601 1602 /* First program IKPU profile i.e PKIND configs. 1603 * Check HW max count to avoid configuring junk or 1604 * writing to unsupported CSR addresses. 1605 */ 1606 pkind = &hw->pkind; 1607 num_pkinds = rvu->kpu.pkinds; 1608 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 1609 1610 for (idx = 0; idx < num_pkinds; idx++) 1611 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1612 1613 /* Program KPU CAM and Action profiles */ 1614 num_kpus = rvu->kpu.kpus; 1615 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1616 1617 for (idx = 0; idx < num_kpus; idx++) 1618 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1619 } 1620 1621 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1622 { 1623 int nixlf_count = rvu_get_nixlf_count(rvu); 1624 struct npc_mcam *mcam = &rvu->hw->mcam; 1625 int rsvd, err; 1626 u64 cfg; 1627 1628 /* Actual number of MCAM entries vary by entry size */ 1629 cfg = (rvu_read64(rvu, blkaddr, 1630 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1631 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1632 mcam->keysize = cfg; 1633 1634 /* Number of banks combined per MCAM entry */ 1635 if (cfg == NPC_MCAM_KEY_X4) 1636 mcam->banks_per_entry = 4; 1637 else if (cfg == NPC_MCAM_KEY_X2) 1638 mcam->banks_per_entry = 2; 1639 else 1640 mcam->banks_per_entry = 1; 1641 1642 /* Reserve one MCAM entry for each of the NIX LF to 1643 * guarantee space to install default matching DMAC rule. 1644 * Also reserve 2 MCAM entries for each PF for default 1645 * channel based matching or 'bcast & promisc' matching to 1646 * support BCAST and PROMISC modes of operation for PFs. 1647 * PF0 is excluded. 1648 */ 1649 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1650 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1651 if (mcam->total_entries <= rsvd) { 1652 dev_warn(rvu->dev, 1653 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1654 mcam->total_entries); 1655 return -ENOMEM; 1656 } 1657 1658 mcam->bmap_entries = mcam->total_entries - rsvd; 1659 mcam->nixlf_offset = mcam->bmap_entries; 1660 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1661 1662 /* Allocate bitmaps for managing MCAM entries */ 1663 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1664 sizeof(long), GFP_KERNEL); 1665 if (!mcam->bmap) 1666 return -ENOMEM; 1667 1668 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1669 BITS_TO_LONGS(mcam->bmap_entries), 1670 sizeof(long), GFP_KERNEL); 1671 if (!mcam->bmap_reverse) 1672 return -ENOMEM; 1673 1674 mcam->bmap_fcnt = mcam->bmap_entries; 1675 1676 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1677 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1678 sizeof(u16), GFP_KERNEL); 1679 if (!mcam->entry2pfvf_map) 1680 return -ENOMEM; 1681 1682 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1683 * allocations and another 1/8th at the top for high priority 1684 * allocations. 1685 */ 1686 mcam->lprio_count = mcam->bmap_entries / 8; 1687 if (mcam->lprio_count > BITS_PER_LONG) 1688 mcam->lprio_count = round_down(mcam->lprio_count, 1689 BITS_PER_LONG); 1690 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1691 mcam->hprio_count = mcam->lprio_count; 1692 mcam->hprio_end = mcam->hprio_count; 1693 1694 1695 /* Allocate bitmap for managing MCAM counters and memory 1696 * for saving counter to RVU PFFUNC allocation mapping. 1697 */ 1698 err = rvu_alloc_bitmap(&mcam->counters); 1699 if (err) 1700 return err; 1701 1702 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1703 sizeof(u16), GFP_KERNEL); 1704 if (!mcam->cntr2pfvf_map) 1705 goto free_mem; 1706 1707 /* Alloc memory for MCAM entry to counter mapping and for tracking 1708 * counter's reference count. 1709 */ 1710 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1711 sizeof(u16), GFP_KERNEL); 1712 if (!mcam->entry2cntr_map) 1713 goto free_mem; 1714 1715 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1716 sizeof(u16), GFP_KERNEL); 1717 if (!mcam->cntr_refcnt) 1718 goto free_mem; 1719 1720 /* Alloc memory for saving target device of mcam rule */ 1721 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1722 sizeof(u16), GFP_KERNEL); 1723 if (!mcam->entry2target_pffunc) 1724 goto free_mem; 1725 1726 mutex_init(&mcam->lock); 1727 1728 return 0; 1729 1730 free_mem: 1731 kfree(mcam->counters.bmap); 1732 return -ENOMEM; 1733 } 1734 1735 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1736 { 1737 struct npc_pkind *pkind = &rvu->hw->pkind; 1738 struct npc_mcam *mcam = &rvu->hw->mcam; 1739 struct rvu_hwinfo *hw = rvu->hw; 1740 u64 npc_const, npc_const1; 1741 u64 npc_const2 = 0; 1742 1743 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1744 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1745 if (npc_const1 & BIT_ULL(63)) 1746 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1747 1748 pkind->rsrc.max = (npc_const1 >> 12) & 0xFFULL; 1749 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1750 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1751 hw->npc_intfs = npc_const & 0xFULL; 1752 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1753 1754 mcam->banks = (npc_const >> 44) & 0xFULL; 1755 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1756 /* Extended set */ 1757 if (npc_const2) { 1758 hw->npc_ext_set = true; 1759 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1760 mcam->banksize = npc_const2 & 0xFFFFULL; 1761 } 1762 1763 mcam->counters.max = hw->npc_counters; 1764 } 1765 1766 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1767 { 1768 struct npc_mcam *mcam = &rvu->hw->mcam; 1769 struct rvu_hwinfo *hw = rvu->hw; 1770 u64 nibble_ena, rx_kex, tx_kex; 1771 u8 intf; 1772 1773 /* Reserve last counter for MCAM RX miss action which is set to 1774 * drop packet. This way we will know how many pkts didn't match 1775 * any MCAM entry. 1776 */ 1777 mcam->counters.max--; 1778 mcam->rx_miss_act_cntr = mcam->counters.max; 1779 1780 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1781 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1782 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1783 1784 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1785 if (nibble_ena) { 1786 tx_kex &= ~NPC_PARSE_NIBBLE; 1787 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1788 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1789 } 1790 1791 /* Configure RX interfaces */ 1792 for (intf = 0; intf < hw->npc_intfs; intf++) { 1793 if (is_npc_intf_tx(intf)) 1794 continue; 1795 1796 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1797 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1798 rx_kex); 1799 1800 /* If MCAM lookup doesn't result in a match, drop the received 1801 * packet. And map this action to a counter to count dropped 1802 * packets. 1803 */ 1804 rvu_write64(rvu, blkaddr, 1805 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1806 1807 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1808 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1809 */ 1810 rvu_write64(rvu, blkaddr, 1811 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1812 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1813 BIT_ULL(9) | mcam->rx_miss_act_cntr); 1814 } 1815 1816 /* Configure TX interfaces */ 1817 for (intf = 0; intf < hw->npc_intfs; intf++) { 1818 if (is_npc_intf_rx(intf)) 1819 continue; 1820 1821 /* Extract Ltypes LID_LA to LID_LE */ 1822 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1823 tx_kex); 1824 1825 /* Set TX miss action to UCAST_DEFAULT i.e 1826 * transmit the packet on NIX LF SQ's default channel. 1827 */ 1828 rvu_write64(rvu, blkaddr, 1829 NPC_AF_INTFX_MISS_ACT(intf), 1830 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1831 } 1832 } 1833 1834 int rvu_npc_init(struct rvu *rvu) 1835 { 1836 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1837 struct npc_pkind *pkind = &rvu->hw->pkind; 1838 struct npc_mcam *mcam = &rvu->hw->mcam; 1839 int blkaddr, entry, bank, err; 1840 1841 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1842 if (blkaddr < 0) { 1843 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1844 return -ENODEV; 1845 } 1846 1847 rvu_npc_hw_init(rvu, blkaddr); 1848 1849 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1850 for (bank = 0; bank < mcam->banks; bank++) { 1851 for (entry = 0; entry < mcam->banksize; entry++) 1852 rvu_write64(rvu, blkaddr, 1853 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1854 } 1855 1856 err = rvu_alloc_bitmap(&pkind->rsrc); 1857 if (err) 1858 return err; 1859 1860 /* Allocate mem for pkind to PF and channel mapping info */ 1861 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 1862 sizeof(u32), GFP_KERNEL); 1863 if (!pkind->pfchan_map) 1864 return -ENOMEM; 1865 1866 /* Configure KPU profile */ 1867 npc_parser_profile_init(rvu, blkaddr); 1868 1869 /* Config Outer L2, IPv4's NPC layer info */ 1870 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1871 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 1872 kpu->lt_def->pck_ol2.ltype_mask); 1873 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1874 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 1875 kpu->lt_def->pck_oip4.ltype_mask); 1876 1877 /* Config Inner IPV4 NPC layer info */ 1878 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 1879 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 1880 kpu->lt_def->pck_iip4.ltype_mask); 1881 1882 /* Enable below for Rx pkts. 1883 * - Outer IPv4 header checksum validation. 1884 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 1885 * - Inner IPv4 header checksum validation. 1886 * - Set non zero checksum error code value 1887 */ 1888 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 1889 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 1890 BIT_ULL(32) | BIT_ULL(24) | BIT_ULL(6) | 1891 BIT_ULL(2) | BIT_ULL(1)); 1892 1893 rvu_npc_setup_interfaces(rvu, blkaddr); 1894 1895 /* Configure MKEX profile */ 1896 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 1897 1898 err = npc_mcam_rsrcs_init(rvu, blkaddr); 1899 if (err) 1900 return err; 1901 1902 err = npc_flow_steering_init(rvu, blkaddr); 1903 if (err) { 1904 dev_err(rvu->dev, 1905 "Incorrect mkex profile loaded using default mkex\n"); 1906 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 1907 } 1908 1909 return 0; 1910 } 1911 1912 void rvu_npc_freemem(struct rvu *rvu) 1913 { 1914 struct npc_pkind *pkind = &rvu->hw->pkind; 1915 struct npc_mcam *mcam = &rvu->hw->mcam; 1916 1917 kfree(pkind->rsrc.bmap); 1918 kfree(mcam->counters.bmap); 1919 if (rvu->kpu_prfl_addr) 1920 iounmap(rvu->kpu_prfl_addr); 1921 else 1922 kfree(rvu->kpu_fwdata); 1923 mutex_destroy(&mcam->lock); 1924 } 1925 1926 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 1927 int blkaddr, int *alloc_cnt, 1928 int *enable_cnt) 1929 { 1930 struct npc_mcam *mcam = &rvu->hw->mcam; 1931 int entry; 1932 1933 *alloc_cnt = 0; 1934 *enable_cnt = 0; 1935 1936 for (entry = 0; entry < mcam->bmap_entries; entry++) { 1937 if (mcam->entry2pfvf_map[entry] == pcifunc) { 1938 (*alloc_cnt)++; 1939 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 1940 (*enable_cnt)++; 1941 } 1942 } 1943 } 1944 1945 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 1946 int blkaddr, int *alloc_cnt, 1947 int *enable_cnt) 1948 { 1949 struct npc_mcam *mcam = &rvu->hw->mcam; 1950 int cntr; 1951 1952 *alloc_cnt = 0; 1953 *enable_cnt = 0; 1954 1955 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1956 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1957 (*alloc_cnt)++; 1958 if (mcam->cntr_refcnt[cntr]) 1959 (*enable_cnt)++; 1960 } 1961 } 1962 } 1963 1964 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 1965 u16 pcifunc, int entry) 1966 { 1967 /* verify AF installed entries */ 1968 if (is_pffunc_af(pcifunc)) 1969 return 0; 1970 /* Verify if entry is valid and if it is indeed 1971 * allocated to the requesting PFFUNC. 1972 */ 1973 if (entry >= mcam->bmap_entries) 1974 return NPC_MCAM_INVALID_REQ; 1975 1976 if (pcifunc != mcam->entry2pfvf_map[entry]) 1977 return NPC_MCAM_PERM_DENIED; 1978 1979 return 0; 1980 } 1981 1982 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 1983 u16 pcifunc, int cntr) 1984 { 1985 /* Verify if counter is valid and if it is indeed 1986 * allocated to the requesting PFFUNC. 1987 */ 1988 if (cntr >= mcam->counters.max) 1989 return NPC_MCAM_INVALID_REQ; 1990 1991 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 1992 return NPC_MCAM_PERM_DENIED; 1993 1994 return 0; 1995 } 1996 1997 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 1998 int blkaddr, u16 entry, u16 cntr) 1999 { 2000 u16 index = entry & (mcam->banksize - 1); 2001 u16 bank = npc_get_bank(mcam, entry); 2002 2003 /* Set mapping and increment counter's refcnt */ 2004 mcam->entry2cntr_map[entry] = cntr; 2005 mcam->cntr_refcnt[cntr]++; 2006 /* Enable stats 2007 * NPC_AF_MCAMEX_BANKX_STAT_ACT[14:12] - counter[11:9] 2008 * NPC_AF_MCAMEX_BANKX_STAT_ACT[8:0] - counter[8:0] 2009 */ 2010 rvu_write64(rvu, blkaddr, 2011 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2012 ((cntr >> 9) << 12) | BIT_ULL(9) | cntr); 2013 } 2014 2015 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2016 struct npc_mcam *mcam, 2017 int blkaddr, u16 entry, u16 cntr) 2018 { 2019 u16 index = entry & (mcam->banksize - 1); 2020 u16 bank = npc_get_bank(mcam, entry); 2021 2022 /* Remove mapping and reduce counter's refcnt */ 2023 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2024 mcam->cntr_refcnt[cntr]--; 2025 /* Disable stats */ 2026 rvu_write64(rvu, blkaddr, 2027 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2028 } 2029 2030 /* Sets MCAM entry in bitmap as used. Update 2031 * reverse bitmap too. Should be called with 2032 * 'mcam->lock' held. 2033 */ 2034 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2035 { 2036 u16 entry, rentry; 2037 2038 entry = index; 2039 rentry = mcam->bmap_entries - index - 1; 2040 2041 __set_bit(entry, mcam->bmap); 2042 __set_bit(rentry, mcam->bmap_reverse); 2043 mcam->bmap_fcnt--; 2044 } 2045 2046 /* Sets MCAM entry in bitmap as free. Update 2047 * reverse bitmap too. Should be called with 2048 * 'mcam->lock' held. 2049 */ 2050 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2051 { 2052 u16 entry, rentry; 2053 2054 entry = index; 2055 rentry = mcam->bmap_entries - index - 1; 2056 2057 __clear_bit(entry, mcam->bmap); 2058 __clear_bit(rentry, mcam->bmap_reverse); 2059 mcam->bmap_fcnt++; 2060 } 2061 2062 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2063 int blkaddr, u16 pcifunc) 2064 { 2065 u16 index, cntr; 2066 2067 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2068 for (index = 0; index < mcam->bmap_entries; index++) { 2069 if (mcam->entry2pfvf_map[index] == pcifunc) { 2070 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2071 /* Free the entry in bitmap */ 2072 npc_mcam_clear_bit(mcam, index); 2073 /* Disable the entry */ 2074 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2075 2076 /* Update entry2counter mapping */ 2077 cntr = mcam->entry2cntr_map[index]; 2078 if (cntr != NPC_MCAM_INVALID_MAP) 2079 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2080 blkaddr, index, 2081 cntr); 2082 mcam->entry2target_pffunc[index] = 0x0; 2083 } 2084 } 2085 } 2086 2087 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2088 u16 pcifunc) 2089 { 2090 u16 cntr; 2091 2092 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2093 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2094 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2095 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2096 mcam->cntr_refcnt[cntr] = 0; 2097 rvu_free_rsrc(&mcam->counters, cntr); 2098 /* This API is expected to be called after freeing 2099 * MCAM entries, which inturn will remove 2100 * 'entry to counter' mapping. 2101 * No need to do it again. 2102 */ 2103 } 2104 } 2105 } 2106 2107 /* Find area of contiguous free entries of size 'nr'. 2108 * If not found return max contiguous free entries available. 2109 */ 2110 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2111 u16 nr, u16 *max_area) 2112 { 2113 u16 max_area_start = 0; 2114 u16 index, next, end; 2115 2116 *max_area = 0; 2117 2118 again: 2119 index = find_next_zero_bit(map, size, start); 2120 if (index >= size) 2121 return max_area_start; 2122 2123 end = ((index + nr) >= size) ? size : index + nr; 2124 next = find_next_bit(map, end, index); 2125 if (*max_area < (next - index)) { 2126 *max_area = next - index; 2127 max_area_start = index; 2128 } 2129 2130 if (next < end) { 2131 start = next + 1; 2132 goto again; 2133 } 2134 2135 return max_area_start; 2136 } 2137 2138 /* Find number of free MCAM entries available 2139 * within range i.e in between 'start' and 'end'. 2140 */ 2141 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2142 { 2143 u16 index, next; 2144 u16 fcnt = 0; 2145 2146 again: 2147 if (start >= end) 2148 return fcnt; 2149 2150 index = find_next_zero_bit(map, end, start); 2151 if (index >= end) 2152 return fcnt; 2153 2154 next = find_next_bit(map, end, index); 2155 if (next <= end) { 2156 fcnt += next - index; 2157 start = next + 1; 2158 goto again; 2159 } 2160 2161 fcnt += end - index; 2162 return fcnt; 2163 } 2164 2165 static void 2166 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2167 struct npc_mcam_alloc_entry_req *req, 2168 u16 *start, u16 *end, bool *reverse) 2169 { 2170 u16 fcnt; 2171 2172 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2173 goto hprio; 2174 2175 /* For a low priority entry allocation 2176 * - If reference entry is not in hprio zone then 2177 * search range: ref_entry to end. 2178 * - If reference entry is in hprio zone and if 2179 * request can be accomodated in non-hprio zone then 2180 * search range: 'start of middle zone' to 'end' 2181 * - else search in reverse, so that less number of hprio 2182 * zone entries are allocated. 2183 */ 2184 2185 *reverse = false; 2186 *start = req->ref_entry + 1; 2187 *end = mcam->bmap_entries; 2188 2189 if (req->ref_entry >= mcam->hprio_end) 2190 return; 2191 2192 fcnt = npc_mcam_get_free_count(mcam->bmap, 2193 mcam->hprio_end, mcam->bmap_entries); 2194 if (fcnt > req->count) 2195 *start = mcam->hprio_end; 2196 else 2197 *reverse = true; 2198 return; 2199 2200 hprio: 2201 /* For a high priority entry allocation, search is always 2202 * in reverse to preserve hprio zone entries. 2203 * - If reference entry is not in lprio zone then 2204 * search range: 0 to ref_entry. 2205 * - If reference entry is in lprio zone and if 2206 * request can be accomodated in middle zone then 2207 * search range: 'hprio_end' to 'lprio_start' 2208 */ 2209 2210 *reverse = true; 2211 *start = 0; 2212 *end = req->ref_entry; 2213 2214 if (req->ref_entry <= mcam->lprio_start) 2215 return; 2216 2217 fcnt = npc_mcam_get_free_count(mcam->bmap, 2218 mcam->hprio_end, mcam->lprio_start); 2219 if (fcnt < req->count) 2220 return; 2221 *start = mcam->hprio_end; 2222 *end = mcam->lprio_start; 2223 } 2224 2225 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2226 struct npc_mcam_alloc_entry_req *req, 2227 struct npc_mcam_alloc_entry_rsp *rsp) 2228 { 2229 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2230 u16 fcnt, hp_fcnt, lp_fcnt; 2231 u16 start, end, index; 2232 int entry, next_start; 2233 bool reverse = false; 2234 unsigned long *bmap; 2235 u16 max_contig; 2236 2237 mutex_lock(&mcam->lock); 2238 2239 /* Check if there are any free entries */ 2240 if (!mcam->bmap_fcnt) { 2241 mutex_unlock(&mcam->lock); 2242 return NPC_MCAM_ALLOC_FAILED; 2243 } 2244 2245 /* MCAM entries are divided into high priority, middle and 2246 * low priority zones. Idea is to not allocate top and lower 2247 * most entries as much as possible, this is to increase 2248 * probability of honouring priority allocation requests. 2249 * 2250 * Two bitmaps are used for mcam entry management, 2251 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2252 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2253 * 2254 * Reverse bitmap is used to allocate entries 2255 * - when a higher priority entry is requested 2256 * - when available free entries are less. 2257 * Lower priority ones out of avaialble free entries are always 2258 * chosen when 'high vs low' question arises. 2259 */ 2260 2261 /* Get the search range for priority allocation request */ 2262 if (req->priority) { 2263 npc_get_mcam_search_range_priority(mcam, req, 2264 &start, &end, &reverse); 2265 goto alloc; 2266 } 2267 2268 /* Find out the search range for non-priority allocation request 2269 * 2270 * Get MCAM free entry count in middle zone. 2271 */ 2272 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2273 mcam->lprio_start, 2274 mcam->bmap_entries); 2275 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2276 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2277 2278 /* Check if request can be accomodated in the middle zone */ 2279 if (fcnt > req->count) { 2280 start = mcam->hprio_end; 2281 end = mcam->lprio_start; 2282 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2283 /* Expand search zone from half of hprio zone to 2284 * half of lprio zone. 2285 */ 2286 start = mcam->hprio_end / 2; 2287 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2288 reverse = true; 2289 } else { 2290 /* Not enough free entries, search all entries in reverse, 2291 * so that low priority ones will get used up. 2292 */ 2293 reverse = true; 2294 start = 0; 2295 end = mcam->bmap_entries; 2296 } 2297 2298 alloc: 2299 if (reverse) { 2300 bmap = mcam->bmap_reverse; 2301 start = mcam->bmap_entries - start; 2302 end = mcam->bmap_entries - end; 2303 index = start; 2304 start = end; 2305 end = index; 2306 } else { 2307 bmap = mcam->bmap; 2308 } 2309 2310 if (req->contig) { 2311 /* Allocate requested number of contiguous entries, if 2312 * unsuccessful find max contiguous entries available. 2313 */ 2314 index = npc_mcam_find_zero_area(bmap, end, start, 2315 req->count, &max_contig); 2316 rsp->count = max_contig; 2317 if (reverse) 2318 rsp->entry = mcam->bmap_entries - index - max_contig; 2319 else 2320 rsp->entry = index; 2321 } else { 2322 /* Allocate requested number of non-contiguous entries, 2323 * if unsuccessful allocate as many as possible. 2324 */ 2325 rsp->count = 0; 2326 next_start = start; 2327 for (entry = 0; entry < req->count; entry++) { 2328 index = find_next_zero_bit(bmap, end, next_start); 2329 if (index >= end) 2330 break; 2331 2332 next_start = start + (index - start) + 1; 2333 2334 /* Save the entry's index */ 2335 if (reverse) 2336 index = mcam->bmap_entries - index - 1; 2337 entry_list[entry] = index; 2338 rsp->count++; 2339 } 2340 } 2341 2342 /* If allocating requested no of entries is unsucessful, 2343 * expand the search range to full bitmap length and retry. 2344 */ 2345 if (!req->priority && (rsp->count < req->count) && 2346 ((end - start) != mcam->bmap_entries)) { 2347 reverse = true; 2348 start = 0; 2349 end = mcam->bmap_entries; 2350 goto alloc; 2351 } 2352 2353 /* For priority entry allocation requests, if allocation is 2354 * failed then expand search to max possible range and retry. 2355 */ 2356 if (req->priority && rsp->count < req->count) { 2357 if (req->priority == NPC_MCAM_LOWER_PRIO && 2358 (start != (req->ref_entry + 1))) { 2359 start = req->ref_entry + 1; 2360 end = mcam->bmap_entries; 2361 reverse = false; 2362 goto alloc; 2363 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2364 ((end - start) != req->ref_entry)) { 2365 start = 0; 2366 end = req->ref_entry; 2367 reverse = true; 2368 goto alloc; 2369 } 2370 } 2371 2372 /* Copy MCAM entry indices into mbox response entry_list. 2373 * Requester always expects indices in ascending order, so 2374 * so reverse the list if reverse bitmap is used for allocation. 2375 */ 2376 if (!req->contig && rsp->count) { 2377 index = 0; 2378 for (entry = rsp->count - 1; entry >= 0; entry--) { 2379 if (reverse) 2380 rsp->entry_list[index++] = entry_list[entry]; 2381 else 2382 rsp->entry_list[entry] = entry_list[entry]; 2383 } 2384 } 2385 2386 /* Mark the allocated entries as used and set nixlf mapping */ 2387 for (entry = 0; entry < rsp->count; entry++) { 2388 index = req->contig ? 2389 (rsp->entry + entry) : rsp->entry_list[entry]; 2390 npc_mcam_set_bit(mcam, index); 2391 mcam->entry2pfvf_map[index] = pcifunc; 2392 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2393 } 2394 2395 /* Update available free count in mbox response */ 2396 rsp->free_count = mcam->bmap_fcnt; 2397 2398 mutex_unlock(&mcam->lock); 2399 return 0; 2400 } 2401 2402 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2403 struct npc_mcam_alloc_entry_req *req, 2404 struct npc_mcam_alloc_entry_rsp *rsp) 2405 { 2406 struct npc_mcam *mcam = &rvu->hw->mcam; 2407 u16 pcifunc = req->hdr.pcifunc; 2408 int blkaddr; 2409 2410 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2411 if (blkaddr < 0) 2412 return NPC_MCAM_INVALID_REQ; 2413 2414 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2415 rsp->free_count = 0; 2416 2417 /* Check if ref_entry is within range */ 2418 if (req->priority && req->ref_entry >= mcam->bmap_entries) 2419 return NPC_MCAM_INVALID_REQ; 2420 2421 /* ref_entry can't be '0' if requested priority is high. 2422 * Can't be last entry if requested priority is low. 2423 */ 2424 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2425 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2426 req->priority == NPC_MCAM_LOWER_PRIO)) 2427 return NPC_MCAM_INVALID_REQ; 2428 2429 /* Since list of allocated indices needs to be sent to requester, 2430 * max number of non-contiguous entries per mbox msg is limited. 2431 */ 2432 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) 2433 return NPC_MCAM_INVALID_REQ; 2434 2435 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2436 if (!is_nixlf_attached(rvu, pcifunc)) 2437 return NPC_MCAM_ALLOC_DENIED; 2438 2439 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2440 } 2441 2442 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2443 struct npc_mcam_free_entry_req *req, 2444 struct msg_rsp *rsp) 2445 { 2446 struct npc_mcam *mcam = &rvu->hw->mcam; 2447 u16 pcifunc = req->hdr.pcifunc; 2448 int blkaddr, rc = 0; 2449 u16 cntr; 2450 2451 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2452 if (blkaddr < 0) 2453 return NPC_MCAM_INVALID_REQ; 2454 2455 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2456 if (!is_nixlf_attached(rvu, pcifunc)) 2457 return NPC_MCAM_INVALID_REQ; 2458 2459 mutex_lock(&mcam->lock); 2460 2461 if (req->all) 2462 goto free_all; 2463 2464 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2465 if (rc) 2466 goto exit; 2467 2468 mcam->entry2pfvf_map[req->entry] = 0; 2469 mcam->entry2target_pffunc[req->entry] = 0x0; 2470 npc_mcam_clear_bit(mcam, req->entry); 2471 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2472 2473 /* Update entry2counter mapping */ 2474 cntr = mcam->entry2cntr_map[req->entry]; 2475 if (cntr != NPC_MCAM_INVALID_MAP) 2476 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2477 req->entry, cntr); 2478 2479 goto exit; 2480 2481 free_all: 2482 /* Free up all entries allocated to requesting PFFUNC */ 2483 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2484 exit: 2485 mutex_unlock(&mcam->lock); 2486 return rc; 2487 } 2488 2489 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2490 struct npc_mcam_read_entry_req *req, 2491 struct npc_mcam_read_entry_rsp *rsp) 2492 { 2493 struct npc_mcam *mcam = &rvu->hw->mcam; 2494 u16 pcifunc = req->hdr.pcifunc; 2495 int blkaddr, rc; 2496 2497 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2498 if (blkaddr < 0) 2499 return NPC_MCAM_INVALID_REQ; 2500 2501 mutex_lock(&mcam->lock); 2502 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2503 if (!rc) { 2504 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2505 &rsp->entry_data, 2506 &rsp->intf, &rsp->enable); 2507 } 2508 2509 mutex_unlock(&mcam->lock); 2510 return rc; 2511 } 2512 2513 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2514 struct npc_mcam_write_entry_req *req, 2515 struct msg_rsp *rsp) 2516 { 2517 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2518 struct npc_mcam *mcam = &rvu->hw->mcam; 2519 u16 pcifunc = req->hdr.pcifunc; 2520 u16 channel, chan_mask; 2521 int blkaddr, rc; 2522 u8 nix_intf; 2523 2524 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2525 if (blkaddr < 0) 2526 return NPC_MCAM_INVALID_REQ; 2527 2528 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 2529 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 2530 channel &= chan_mask; 2531 2532 mutex_lock(&mcam->lock); 2533 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2534 if (rc) 2535 goto exit; 2536 2537 if (req->set_cntr && 2538 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2539 rc = NPC_MCAM_INVALID_REQ; 2540 goto exit; 2541 } 2542 2543 if (!is_npc_interface_valid(rvu, req->intf)) { 2544 rc = NPC_MCAM_INVALID_REQ; 2545 goto exit; 2546 } 2547 2548 if (is_npc_intf_tx(req->intf)) 2549 nix_intf = pfvf->nix_tx_intf; 2550 else 2551 nix_intf = pfvf->nix_rx_intf; 2552 2553 if (npc_mcam_verify_channel(rvu, pcifunc, req->intf, channel)) { 2554 rc = NPC_MCAM_INVALID_REQ; 2555 goto exit; 2556 } 2557 2558 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 2559 pcifunc)) { 2560 rc = NPC_MCAM_INVALID_REQ; 2561 goto exit; 2562 } 2563 2564 /* For AF installed rules, the nix_intf should be set to target NIX */ 2565 if (is_pffunc_af(req->hdr.pcifunc)) 2566 nix_intf = req->intf; 2567 2568 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2569 &req->entry_data, req->enable_entry); 2570 2571 if (req->set_cntr) 2572 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2573 req->entry, req->cntr); 2574 2575 rc = 0; 2576 exit: 2577 mutex_unlock(&mcam->lock); 2578 return rc; 2579 } 2580 2581 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2582 struct npc_mcam_ena_dis_entry_req *req, 2583 struct msg_rsp *rsp) 2584 { 2585 struct npc_mcam *mcam = &rvu->hw->mcam; 2586 u16 pcifunc = req->hdr.pcifunc; 2587 int blkaddr, rc; 2588 2589 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2590 if (blkaddr < 0) 2591 return NPC_MCAM_INVALID_REQ; 2592 2593 mutex_lock(&mcam->lock); 2594 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2595 mutex_unlock(&mcam->lock); 2596 if (rc) 2597 return rc; 2598 2599 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2600 2601 return 0; 2602 } 2603 2604 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2605 struct npc_mcam_ena_dis_entry_req *req, 2606 struct msg_rsp *rsp) 2607 { 2608 struct npc_mcam *mcam = &rvu->hw->mcam; 2609 u16 pcifunc = req->hdr.pcifunc; 2610 int blkaddr, rc; 2611 2612 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2613 if (blkaddr < 0) 2614 return NPC_MCAM_INVALID_REQ; 2615 2616 mutex_lock(&mcam->lock); 2617 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2618 mutex_unlock(&mcam->lock); 2619 if (rc) 2620 return rc; 2621 2622 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2623 2624 return 0; 2625 } 2626 2627 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2628 struct npc_mcam_shift_entry_req *req, 2629 struct npc_mcam_shift_entry_rsp *rsp) 2630 { 2631 struct npc_mcam *mcam = &rvu->hw->mcam; 2632 u16 pcifunc = req->hdr.pcifunc; 2633 u16 old_entry, new_entry; 2634 u16 index, cntr; 2635 int blkaddr, rc; 2636 2637 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2638 if (blkaddr < 0) 2639 return NPC_MCAM_INVALID_REQ; 2640 2641 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2642 return NPC_MCAM_INVALID_REQ; 2643 2644 mutex_lock(&mcam->lock); 2645 for (index = 0; index < req->shift_count; index++) { 2646 old_entry = req->curr_entry[index]; 2647 new_entry = req->new_entry[index]; 2648 2649 /* Check if both old and new entries are valid and 2650 * does belong to this PFFUNC or not. 2651 */ 2652 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2653 if (rc) 2654 break; 2655 2656 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2657 if (rc) 2658 break; 2659 2660 /* new_entry should not have a counter mapped */ 2661 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2662 rc = NPC_MCAM_PERM_DENIED; 2663 break; 2664 } 2665 2666 /* Disable the new_entry */ 2667 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2668 2669 /* Copy rule from old entry to new entry */ 2670 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2671 2672 /* Copy counter mapping, if any */ 2673 cntr = mcam->entry2cntr_map[old_entry]; 2674 if (cntr != NPC_MCAM_INVALID_MAP) { 2675 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2676 old_entry, cntr); 2677 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2678 new_entry, cntr); 2679 } 2680 2681 /* Enable new_entry and disable old_entry */ 2682 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2683 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2684 } 2685 2686 /* If shift has failed then report the failed index */ 2687 if (index != req->shift_count) { 2688 rc = NPC_MCAM_PERM_DENIED; 2689 rsp->failed_entry_idx = index; 2690 } 2691 2692 mutex_unlock(&mcam->lock); 2693 return rc; 2694 } 2695 2696 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2697 struct npc_mcam_alloc_counter_req *req, 2698 struct npc_mcam_alloc_counter_rsp *rsp) 2699 { 2700 struct npc_mcam *mcam = &rvu->hw->mcam; 2701 u16 pcifunc = req->hdr.pcifunc; 2702 u16 max_contig, cntr; 2703 int blkaddr, index; 2704 2705 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2706 if (blkaddr < 0) 2707 return NPC_MCAM_INVALID_REQ; 2708 2709 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2710 if (!is_nixlf_attached(rvu, pcifunc)) 2711 return NPC_MCAM_INVALID_REQ; 2712 2713 /* Since list of allocated counter IDs needs to be sent to requester, 2714 * max number of non-contiguous counters per mbox msg is limited. 2715 */ 2716 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2717 return NPC_MCAM_INVALID_REQ; 2718 2719 mutex_lock(&mcam->lock); 2720 2721 /* Check if unused counters are available or not */ 2722 if (!rvu_rsrc_free_count(&mcam->counters)) { 2723 mutex_unlock(&mcam->lock); 2724 return NPC_MCAM_ALLOC_FAILED; 2725 } 2726 2727 rsp->count = 0; 2728 2729 if (req->contig) { 2730 /* Allocate requested number of contiguous counters, if 2731 * unsuccessful find max contiguous entries available. 2732 */ 2733 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2734 mcam->counters.max, 0, 2735 req->count, &max_contig); 2736 rsp->count = max_contig; 2737 rsp->cntr = index; 2738 for (cntr = index; cntr < (index + max_contig); cntr++) { 2739 __set_bit(cntr, mcam->counters.bmap); 2740 mcam->cntr2pfvf_map[cntr] = pcifunc; 2741 } 2742 } else { 2743 /* Allocate requested number of non-contiguous counters, 2744 * if unsuccessful allocate as many as possible. 2745 */ 2746 for (cntr = 0; cntr < req->count; cntr++) { 2747 index = rvu_alloc_rsrc(&mcam->counters); 2748 if (index < 0) 2749 break; 2750 rsp->cntr_list[cntr] = index; 2751 rsp->count++; 2752 mcam->cntr2pfvf_map[index] = pcifunc; 2753 } 2754 } 2755 2756 mutex_unlock(&mcam->lock); 2757 return 0; 2758 } 2759 2760 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2761 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2762 { 2763 struct npc_mcam *mcam = &rvu->hw->mcam; 2764 u16 index, entry = 0; 2765 int blkaddr, err; 2766 2767 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2768 if (blkaddr < 0) 2769 return NPC_MCAM_INVALID_REQ; 2770 2771 mutex_lock(&mcam->lock); 2772 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2773 if (err) { 2774 mutex_unlock(&mcam->lock); 2775 return err; 2776 } 2777 2778 /* Mark counter as free/unused */ 2779 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2780 rvu_free_rsrc(&mcam->counters, req->cntr); 2781 2782 /* Disable all MCAM entry's stats which are using this counter */ 2783 while (entry < mcam->bmap_entries) { 2784 if (!mcam->cntr_refcnt[req->cntr]) 2785 break; 2786 2787 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2788 if (index >= mcam->bmap_entries) 2789 break; 2790 entry = index + 1; 2791 if (mcam->entry2cntr_map[index] != req->cntr) 2792 continue; 2793 2794 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2795 index, req->cntr); 2796 } 2797 2798 mutex_unlock(&mcam->lock); 2799 return 0; 2800 } 2801 2802 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2803 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2804 { 2805 struct npc_mcam *mcam = &rvu->hw->mcam; 2806 u16 index, entry = 0; 2807 int blkaddr, rc; 2808 2809 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2810 if (blkaddr < 0) 2811 return NPC_MCAM_INVALID_REQ; 2812 2813 mutex_lock(&mcam->lock); 2814 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2815 if (rc) 2816 goto exit; 2817 2818 /* Unmap the MCAM entry and counter */ 2819 if (!req->all) { 2820 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2821 if (rc) 2822 goto exit; 2823 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2824 req->entry, req->cntr); 2825 goto exit; 2826 } 2827 2828 /* Disable all MCAM entry's stats which are using this counter */ 2829 while (entry < mcam->bmap_entries) { 2830 if (!mcam->cntr_refcnt[req->cntr]) 2831 break; 2832 2833 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2834 if (index >= mcam->bmap_entries) 2835 break; 2836 if (mcam->entry2cntr_map[index] != req->cntr) 2837 continue; 2838 2839 entry = index + 1; 2840 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2841 index, req->cntr); 2842 } 2843 exit: 2844 mutex_unlock(&mcam->lock); 2845 return rc; 2846 } 2847 2848 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2849 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2850 { 2851 struct npc_mcam *mcam = &rvu->hw->mcam; 2852 int blkaddr, err; 2853 2854 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2855 if (blkaddr < 0) 2856 return NPC_MCAM_INVALID_REQ; 2857 2858 mutex_lock(&mcam->lock); 2859 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2860 mutex_unlock(&mcam->lock); 2861 if (err) 2862 return err; 2863 2864 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 2865 2866 return 0; 2867 } 2868 2869 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 2870 struct npc_mcam_oper_counter_req *req, 2871 struct npc_mcam_oper_counter_rsp *rsp) 2872 { 2873 struct npc_mcam *mcam = &rvu->hw->mcam; 2874 int blkaddr, err; 2875 2876 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2877 if (blkaddr < 0) 2878 return NPC_MCAM_INVALID_REQ; 2879 2880 mutex_lock(&mcam->lock); 2881 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2882 mutex_unlock(&mcam->lock); 2883 if (err) 2884 return err; 2885 2886 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 2887 rsp->stat &= BIT_ULL(48) - 1; 2888 2889 return 0; 2890 } 2891 2892 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 2893 struct npc_mcam_alloc_and_write_entry_req *req, 2894 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 2895 { 2896 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2897 struct npc_mcam_alloc_counter_req cntr_req; 2898 struct npc_mcam_alloc_counter_rsp cntr_rsp; 2899 struct npc_mcam_alloc_entry_req entry_req; 2900 struct npc_mcam_alloc_entry_rsp entry_rsp; 2901 struct npc_mcam *mcam = &rvu->hw->mcam; 2902 u16 entry = NPC_MCAM_ENTRY_INVALID; 2903 u16 cntr = NPC_MCAM_ENTRY_INVALID; 2904 u16 channel, chan_mask; 2905 int blkaddr, rc; 2906 u8 nix_intf; 2907 2908 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2909 if (blkaddr < 0) 2910 return NPC_MCAM_INVALID_REQ; 2911 2912 if (!is_npc_interface_valid(rvu, req->intf)) 2913 return NPC_MCAM_INVALID_REQ; 2914 2915 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 2916 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 2917 channel &= chan_mask; 2918 2919 if (npc_mcam_verify_channel(rvu, req->hdr.pcifunc, req->intf, channel)) 2920 return NPC_MCAM_INVALID_REQ; 2921 2922 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 2923 req->hdr.pcifunc)) 2924 return NPC_MCAM_INVALID_REQ; 2925 2926 /* Try to allocate a MCAM entry */ 2927 entry_req.hdr.pcifunc = req->hdr.pcifunc; 2928 entry_req.contig = true; 2929 entry_req.priority = req->priority; 2930 entry_req.ref_entry = req->ref_entry; 2931 entry_req.count = 1; 2932 2933 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 2934 &entry_req, &entry_rsp); 2935 if (rc) 2936 return rc; 2937 2938 if (!entry_rsp.count) 2939 return NPC_MCAM_ALLOC_FAILED; 2940 2941 entry = entry_rsp.entry; 2942 2943 if (!req->alloc_cntr) 2944 goto write_entry; 2945 2946 /* Now allocate counter */ 2947 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 2948 cntr_req.contig = true; 2949 cntr_req.count = 1; 2950 2951 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 2952 if (rc) { 2953 /* Free allocated MCAM entry */ 2954 mutex_lock(&mcam->lock); 2955 mcam->entry2pfvf_map[entry] = 0; 2956 npc_mcam_clear_bit(mcam, entry); 2957 mutex_unlock(&mcam->lock); 2958 return rc; 2959 } 2960 2961 cntr = cntr_rsp.cntr; 2962 2963 write_entry: 2964 mutex_lock(&mcam->lock); 2965 2966 if (is_npc_intf_tx(req->intf)) 2967 nix_intf = pfvf->nix_tx_intf; 2968 else 2969 nix_intf = pfvf->nix_rx_intf; 2970 2971 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 2972 &req->entry_data, req->enable_entry); 2973 2974 if (req->alloc_cntr) 2975 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 2976 mutex_unlock(&mcam->lock); 2977 2978 rsp->entry = entry; 2979 rsp->cntr = cntr; 2980 2981 return 0; 2982 } 2983 2984 #define GET_KEX_CFG(intf) \ 2985 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 2986 2987 #define GET_KEX_FLAGS(ld) \ 2988 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 2989 2990 #define GET_KEX_LD(intf, lid, lt, ld) \ 2991 rvu_read64(rvu, BLKADDR_NPC, \ 2992 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 2993 2994 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 2995 rvu_read64(rvu, BLKADDR_NPC, \ 2996 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 2997 2998 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 2999 struct npc_get_kex_cfg_rsp *rsp) 3000 { 3001 int lid, lt, ld, fl; 3002 3003 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3004 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3005 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3006 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3007 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3008 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3009 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3010 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3011 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3012 } 3013 } 3014 } 3015 for (ld = 0; ld < NPC_MAX_LD; ld++) 3016 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3017 3018 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3019 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3020 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3021 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3022 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3023 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3024 } 3025 } 3026 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3027 return 0; 3028 } 3029 3030 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3031 struct msg_req *req, 3032 struct npc_mcam_read_base_rule_rsp *rsp) 3033 { 3034 struct npc_mcam *mcam = &rvu->hw->mcam; 3035 int index, blkaddr, nixlf, rc = 0; 3036 u16 pcifunc = req->hdr.pcifunc; 3037 struct rvu_pfvf *pfvf; 3038 u8 intf, enable; 3039 3040 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3041 if (blkaddr < 0) 3042 return NPC_MCAM_INVALID_REQ; 3043 3044 /* Return the channel number in case of PF */ 3045 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3046 pfvf = rvu_get_pfvf(rvu, pcifunc); 3047 rsp->entry.kw[0] = pfvf->rx_chan_base; 3048 rsp->entry.kw_mask[0] = 0xFFFULL; 3049 goto out; 3050 } 3051 3052 /* Find the pkt steering rule installed by PF to this VF */ 3053 mutex_lock(&mcam->lock); 3054 for (index = 0; index < mcam->bmap_entries; index++) { 3055 if (mcam->entry2target_pffunc[index] == pcifunc) 3056 goto read_entry; 3057 } 3058 3059 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3060 if (rc < 0) { 3061 mutex_unlock(&mcam->lock); 3062 goto out; 3063 } 3064 /* Read the default ucast entry if there is no pkt steering rule */ 3065 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3066 NIXLF_UCAST_ENTRY); 3067 read_entry: 3068 /* Read the mcam entry */ 3069 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3070 &enable); 3071 mutex_unlock(&mcam->lock); 3072 out: 3073 return rc; 3074 } 3075 3076 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3077 struct npc_mcam_get_stats_req *req, 3078 struct npc_mcam_get_stats_rsp *rsp) 3079 { 3080 struct npc_mcam *mcam = &rvu->hw->mcam; 3081 u16 index, cntr; 3082 int blkaddr; 3083 u64 regval; 3084 u32 bank; 3085 3086 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3087 if (blkaddr < 0) 3088 return NPC_MCAM_INVALID_REQ; 3089 3090 mutex_lock(&mcam->lock); 3091 3092 index = req->entry & (mcam->banksize - 1); 3093 bank = npc_get_bank(mcam, req->entry); 3094 3095 /* read MCAM entry STAT_ACT register */ 3096 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3097 3098 if (!(regval & BIT_ULL(9))) { 3099 rsp->stat_ena = 0; 3100 mutex_unlock(&mcam->lock); 3101 return 0; 3102 } 3103 3104 cntr = regval & 0x1FF; 3105 3106 rsp->stat_ena = 1; 3107 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3108 rsp->stat &= BIT_ULL(48) - 1; 3109 3110 mutex_unlock(&mcam->lock); 3111 3112 return 0; 3113 } 3114