1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/module.h> 9 #include <linux/pci.h> 10 11 #include "rvu_struct.h" 12 #include "rvu_reg.h" 13 #include "rvu.h" 14 #include "npc.h" 15 #include "mcs.h" 16 #include "cgx.h" 17 #include "lmac_common.h" 18 #include "rvu_npc_hash.h" 19 #include "cn20k/npc.h" 20 21 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc); 22 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 23 int type, int chan_id); 24 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 25 int type, bool add); 26 static int nix_setup_ipolicers(struct rvu *rvu, 27 struct nix_hw *nix_hw, int blkaddr); 28 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw); 29 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 30 struct nix_hw *nix_hw, u16 pcifunc); 31 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc); 32 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 33 u32 leaf_prof); 34 static const char *nix_get_ctx_name(int ctype); 35 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc); 36 37 enum mc_tbl_sz { 38 MC_TBL_SZ_256, 39 MC_TBL_SZ_512, 40 MC_TBL_SZ_1K, 41 MC_TBL_SZ_2K, 42 MC_TBL_SZ_4K, 43 MC_TBL_SZ_8K, 44 MC_TBL_SZ_16K, 45 MC_TBL_SZ_32K, 46 MC_TBL_SZ_64K, 47 }; 48 49 enum mc_buf_cnt { 50 MC_BUF_CNT_8, 51 MC_BUF_CNT_16, 52 MC_BUF_CNT_32, 53 MC_BUF_CNT_64, 54 MC_BUF_CNT_128, 55 MC_BUF_CNT_256, 56 MC_BUF_CNT_512, 57 MC_BUF_CNT_1024, 58 MC_BUF_CNT_2048, 59 }; 60 61 enum nix_makr_fmt_indexes { 62 NIX_MARK_CFG_IP_DSCP_RED, 63 NIX_MARK_CFG_IP_DSCP_YELLOW, 64 NIX_MARK_CFG_IP_DSCP_YELLOW_RED, 65 NIX_MARK_CFG_IP_ECN_RED, 66 NIX_MARK_CFG_IP_ECN_YELLOW, 67 NIX_MARK_CFG_IP_ECN_YELLOW_RED, 68 NIX_MARK_CFG_VLAN_DEI_RED, 69 NIX_MARK_CFG_VLAN_DEI_YELLOW, 70 NIX_MARK_CFG_VLAN_DEI_YELLOW_RED, 71 NIX_MARK_CFG_MAX, 72 }; 73 74 /* For now considering MC resources needed for broadcast 75 * pkt replication only. i.e 256 HWVFs + 12 PFs. 76 */ 77 #define MC_TBL_SIZE MC_TBL_SZ_2K 78 #define MC_BUF_CNT MC_BUF_CNT_1024 79 80 #define MC_TX_MAX 2048 81 82 struct mce { 83 struct hlist_node node; 84 u32 rq_rss_index; 85 u16 pcifunc; 86 u16 channel; 87 u8 dest_type; 88 u8 is_active; 89 u8 reserved[2]; 90 }; 91 92 int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr) 93 { 94 int i = 0; 95 96 /*If blkaddr is 0, return the first nix block address*/ 97 if (blkaddr == 0) 98 return rvu->nix_blkaddr[blkaddr]; 99 100 while (i + 1 < MAX_NIX_BLKS) { 101 if (rvu->nix_blkaddr[i] == blkaddr) 102 return rvu->nix_blkaddr[i + 1]; 103 i++; 104 } 105 106 return 0; 107 } 108 109 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc) 110 { 111 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 112 int blkaddr; 113 114 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 115 if (!pfvf->nixlf || blkaddr < 0) 116 return false; 117 return true; 118 } 119 120 int rvu_get_nixlf_count(struct rvu *rvu) 121 { 122 int blkaddr = 0, max = 0; 123 struct rvu_block *block; 124 125 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 126 while (blkaddr) { 127 block = &rvu->hw->block[blkaddr]; 128 max += block->lf.max; 129 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 130 } 131 return max; 132 } 133 134 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr) 135 { 136 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 137 struct rvu_hwinfo *hw = rvu->hw; 138 int blkaddr; 139 140 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 141 if (!pfvf->nixlf || blkaddr < 0) 142 return NIX_AF_ERR_AF_LF_INVALID; 143 144 *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 145 if (*nixlf < 0) 146 return NIX_AF_ERR_AF_LF_INVALID; 147 148 if (nix_blkaddr) 149 *nix_blkaddr = blkaddr; 150 151 return 0; 152 } 153 154 int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc, 155 struct nix_hw **nix_hw, int *blkaddr) 156 { 157 struct rvu_pfvf *pfvf; 158 159 pfvf = rvu_get_pfvf(rvu, pcifunc); 160 *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 161 if (!pfvf->nixlf || *blkaddr < 0) 162 return NIX_AF_ERR_AF_LF_INVALID; 163 164 *nix_hw = get_nix_hw(rvu->hw, *blkaddr); 165 if (!*nix_hw) 166 return NIX_AF_ERR_INVALID_NIXBLK; 167 return 0; 168 } 169 170 static void nix_mce_list_init(struct nix_mce_list *list, int max) 171 { 172 INIT_HLIST_HEAD(&list->head); 173 list->count = 0; 174 list->max = max; 175 } 176 177 static int nix_alloc_mce_list(struct nix_mcast *mcast, int count, u8 dir) 178 { 179 struct rsrc_bmap *mce_counter; 180 int idx; 181 182 if (!mcast) 183 return -EINVAL; 184 185 mce_counter = &mcast->mce_counter[dir]; 186 if (!rvu_rsrc_check_contig(mce_counter, count)) 187 return -ENOSPC; 188 189 idx = rvu_alloc_rsrc_contig(mce_counter, count); 190 return idx; 191 } 192 193 static void nix_free_mce_list(struct nix_mcast *mcast, int count, int start, u8 dir) 194 { 195 struct rsrc_bmap *mce_counter; 196 197 if (!mcast) 198 return; 199 200 mce_counter = &mcast->mce_counter[dir]; 201 rvu_free_rsrc_contig(mce_counter, count, start); 202 } 203 204 struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr) 205 { 206 int nix_blkaddr = 0, i = 0; 207 struct rvu *rvu = hw->rvu; 208 209 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 210 while (nix_blkaddr) { 211 if (blkaddr == nix_blkaddr && hw->nix) 212 return &hw->nix[i]; 213 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 214 i++; 215 } 216 return NULL; 217 } 218 219 int nix_get_dwrr_mtu_reg(struct rvu_hwinfo *hw, int smq_link_type) 220 { 221 if (hw->cap.nix_multiple_dwrr_mtu) 222 return NIX_AF_DWRR_MTUX(smq_link_type); 223 224 if (smq_link_type == SMQ_LINK_TYPE_SDP) 225 return NIX_AF_DWRR_SDP_MTU; 226 227 /* Here it's same reg for RPM and LBK */ 228 return NIX_AF_DWRR_RPM_MTU; 229 } 230 231 u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu) 232 { 233 dwrr_mtu &= 0x1FULL; 234 235 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 236 * Value of 4 is reserved for MTU value of 9728 bytes. 237 * Value of 5 is reserved for MTU value of 10240 bytes. 238 */ 239 switch (dwrr_mtu) { 240 case 4: 241 return 9728; 242 case 5: 243 return 10240; 244 default: 245 return BIT_ULL(dwrr_mtu); 246 } 247 248 return 0; 249 } 250 251 u32 convert_bytes_to_dwrr_mtu(u32 bytes) 252 { 253 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 254 * Value of 4 is reserved for MTU value of 9728 bytes. 255 * Value of 5 is reserved for MTU value of 10240 bytes. 256 */ 257 if (bytes > BIT_ULL(16)) 258 return 0; 259 260 switch (bytes) { 261 case 9728: 262 return 4; 263 case 10240: 264 return 5; 265 default: 266 return ilog2(bytes); 267 } 268 269 return 0; 270 } 271 272 static void nix_rx_sync(struct rvu *rvu, int blkaddr) 273 { 274 int err; 275 276 /* Sync all in flight RX packets to LLC/DRAM */ 277 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 278 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 279 if (err) 280 dev_err(rvu->dev, "SYNC1: NIX RX software sync failed\n"); 281 282 /* SW_SYNC ensures all existing transactions are finished and pkts 283 * are written to LLC/DRAM, queues should be teared down after 284 * successful SW_SYNC. Due to a HW errata, in some rare scenarios 285 * an existing transaction might end after SW_SYNC operation. To 286 * ensure operation is fully done, do the SW_SYNC twice. 287 */ 288 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 289 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 290 if (err) 291 dev_err(rvu->dev, "SYNC2: NIX RX software sync failed\n"); 292 } 293 294 static bool is_valid_txschq(struct rvu *rvu, int blkaddr, 295 int lvl, u16 pcifunc, u16 schq) 296 { 297 struct rvu_hwinfo *hw = rvu->hw; 298 struct nix_txsch *txsch; 299 struct nix_hw *nix_hw; 300 u16 map_func; 301 302 nix_hw = get_nix_hw(rvu->hw, blkaddr); 303 if (!nix_hw) 304 return false; 305 306 txsch = &nix_hw->txsch[lvl]; 307 /* Check out of bounds */ 308 if (schq >= txsch->schq.max) 309 return false; 310 311 mutex_lock(&rvu->rsrc_lock); 312 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 313 mutex_unlock(&rvu->rsrc_lock); 314 315 /* TLs aggegating traffic are shared across PF and VFs */ 316 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 317 if ((nix_get_tx_link(rvu, map_func) != 318 nix_get_tx_link(rvu, pcifunc)) && 319 (rvu_get_pf(rvu->pdev, map_func) != 320 rvu_get_pf(rvu->pdev, pcifunc))) 321 return false; 322 else 323 return true; 324 } 325 326 if (map_func != pcifunc) 327 return false; 328 329 return true; 330 } 331 332 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf, 333 struct nix_lf_alloc_rsp *rsp, bool loop) 334 { 335 struct rvu_pfvf *parent_pf, *pfvf = rvu_get_pfvf(rvu, pcifunc); 336 u16 req_chan_base, req_chan_end, req_chan_cnt; 337 struct rvu_hwinfo *hw = rvu->hw; 338 struct sdp_node_info *sdp_info; 339 int pkind, pf, vf, lbkid, vfid; 340 u8 cgx_id, lmac_id; 341 bool from_vf; 342 int err; 343 344 pf = rvu_get_pf(rvu->pdev, pcifunc); 345 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 346 type != NIX_INTF_TYPE_SDP) 347 return 0; 348 349 switch (type) { 350 case NIX_INTF_TYPE_CGX: 351 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf]; 352 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 353 354 pkind = rvu_npc_get_pkind(rvu, pf); 355 if (pkind < 0) { 356 dev_err(rvu->dev, 357 "PF_Func 0x%x: Invalid pkind\n", pcifunc); 358 return -EINVAL; 359 } 360 pfvf->rx_chan_base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0); 361 pfvf->tx_chan_base = pfvf->rx_chan_base; 362 pfvf->rx_chan_cnt = 1; 363 pfvf->tx_chan_cnt = 1; 364 rsp->tx_link = cgx_id * hw->lmac_per_cgx + lmac_id; 365 366 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind); 367 rvu_npc_set_pkind(rvu, pkind, pfvf); 368 break; 369 case NIX_INTF_TYPE_LBK: 370 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 371 372 /* If NIX1 block is present on the silicon then NIXes are 373 * assigned alternatively for lbk interfaces. NIX0 should 374 * send packets on lbk link 1 channels and NIX1 should send 375 * on lbk link 0 channels for the communication between 376 * NIX0 and NIX1. 377 */ 378 lbkid = 0; 379 if (rvu->hw->lbk_links > 1) 380 lbkid = vf & 0x1 ? 0 : 1; 381 382 /* By default NIX0 is configured to send packet on lbk link 1 383 * (which corresponds to LBK1), same packet will receive on 384 * NIX1 over lbk link 0. If NIX1 sends packet on lbk link 0 385 * (which corresponds to LBK2) packet will receive on NIX0 lbk 386 * link 1. 387 * But if lbk links for NIX0 and NIX1 are negated, i.e NIX0 388 * transmits and receives on lbk link 0, whick corresponds 389 * to LBK1 block, back to back connectivity between NIX and 390 * LBK can be achieved (which is similar to 96xx) 391 * 392 * RX TX 393 * NIX0 lbk link 1 (LBK2) 1 (LBK1) 394 * NIX0 lbk link 0 (LBK0) 0 (LBK0) 395 * NIX1 lbk link 0 (LBK1) 0 (LBK2) 396 * NIX1 lbk link 1 (LBK3) 1 (LBK3) 397 */ 398 if (loop) 399 lbkid = !lbkid; 400 401 /* Note that AF's VFs work in pairs and talk over consecutive 402 * loopback channels.Therefore if odd number of AF VFs are 403 * enabled then the last VF remains with no pair. 404 */ 405 pfvf->rx_chan_base = rvu_nix_chan_lbk(rvu, lbkid, vf); 406 pfvf->tx_chan_base = vf & 0x1 ? 407 rvu_nix_chan_lbk(rvu, lbkid, vf - 1) : 408 rvu_nix_chan_lbk(rvu, lbkid, vf + 1); 409 pfvf->rx_chan_cnt = 1; 410 pfvf->tx_chan_cnt = 1; 411 rsp->tx_link = hw->cgx_links + lbkid; 412 pfvf->lbkid = lbkid; 413 rvu_npc_set_pkind(rvu, NPC_RX_LBK_PKIND, pfvf); 414 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 415 pfvf->rx_chan_base, 416 pfvf->rx_chan_cnt); 417 418 break; 419 case NIX_INTF_TYPE_SDP: 420 from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 421 parent_pf = &rvu->pf[rvu_get_pf(rvu->pdev, pcifunc)]; 422 sdp_info = parent_pf->sdp_info; 423 if (!sdp_info) { 424 dev_err(rvu->dev, "Invalid sdp_info pointer\n"); 425 return -EINVAL; 426 } 427 if (from_vf) { 428 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn + 429 sdp_info->num_pf_rings; 430 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 431 for (vfid = 0; vfid < vf; vfid++) 432 req_chan_base += sdp_info->vf_rings[vfid]; 433 req_chan_cnt = sdp_info->vf_rings[vf]; 434 req_chan_end = req_chan_base + req_chan_cnt - 1; 435 if (req_chan_base < rvu_nix_chan_sdp(rvu, 0) || 436 req_chan_end > rvu_nix_chan_sdp(rvu, 255)) { 437 dev_err(rvu->dev, 438 "PF_Func 0x%x: Invalid channel base and count\n", 439 pcifunc); 440 return -EINVAL; 441 } 442 } else { 443 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn; 444 req_chan_cnt = sdp_info->num_pf_rings; 445 } 446 447 pfvf->rx_chan_base = req_chan_base; 448 pfvf->rx_chan_cnt = req_chan_cnt; 449 pfvf->tx_chan_base = pfvf->rx_chan_base; 450 pfvf->tx_chan_cnt = pfvf->rx_chan_cnt; 451 452 rsp->tx_link = hw->cgx_links + hw->lbk_links; 453 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 454 pfvf->rx_chan_base, 455 pfvf->rx_chan_cnt); 456 break; 457 } 458 459 /* Add a UCAST forwarding rule in MCAM with this NIXLF attached 460 * RVU PF/VF's MAC address. 461 */ 462 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 463 pfvf->rx_chan_base, pfvf->mac_addr); 464 465 /* Add this PF_FUNC to bcast pkt replication list */ 466 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true); 467 if (err) { 468 dev_err(rvu->dev, 469 "Bcast list, failed to enable PF_FUNC 0x%x\n", 470 pcifunc); 471 return err; 472 } 473 /* Install MCAM rule matching Ethernet broadcast mac address */ 474 rvu_npc_install_bcast_match_entry(rvu, pcifunc, 475 nixlf, pfvf->rx_chan_base); 476 477 pfvf->maxlen = NIC_HW_MIN_FRS; 478 pfvf->minlen = NIC_HW_MIN_FRS; 479 480 return 0; 481 } 482 483 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf) 484 { 485 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 486 int err; 487 488 pfvf->maxlen = 0; 489 pfvf->minlen = 0; 490 491 /* Remove this PF_FUNC from bcast pkt replication list */ 492 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false); 493 if (err) { 494 dev_err(rvu->dev, 495 "Bcast list, failed to disable PF_FUNC 0x%x\n", 496 pcifunc); 497 } 498 499 /* Free and disable any MCAM entries used by this NIX LF */ 500 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 501 502 /* Disable DMAC filters used */ 503 rvu_cgx_disable_dmac_entries(rvu, pcifunc); 504 } 505 506 #define NIX_BPIDS_PER_LMAC 8 507 #define NIX_BPIDS_PER_CPT 1 508 static int nix_setup_bpids(struct rvu *rvu, struct nix_hw *hw, int blkaddr) 509 { 510 struct nix_bp *bp = &hw->bp; 511 int err, max_bpids; 512 u64 cfg; 513 514 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 515 max_bpids = FIELD_GET(NIX_CONST_MAX_BPIDS, cfg); 516 517 /* Reserve the BPIds for CGX and SDP */ 518 bp->cgx_bpid_cnt = rvu->hw->cgx_links * NIX_BPIDS_PER_LMAC; 519 bp->sdp_bpid_cnt = rvu->hw->sdp_links * FIELD_GET(NIX_CONST_SDP_CHANS, cfg); 520 bp->free_pool_base = bp->cgx_bpid_cnt + bp->sdp_bpid_cnt + 521 NIX_BPIDS_PER_CPT; 522 bp->bpids.max = max_bpids - bp->free_pool_base; 523 524 err = rvu_alloc_bitmap(&bp->bpids); 525 if (err) 526 return err; 527 528 bp->fn_map = devm_kcalloc(rvu->dev, bp->bpids.max, 529 sizeof(u16), GFP_KERNEL); 530 if (!bp->fn_map) 531 return -ENOMEM; 532 533 bp->intf_map = devm_kcalloc(rvu->dev, bp->bpids.max, 534 sizeof(u8), GFP_KERNEL); 535 if (!bp->intf_map) 536 return -ENOMEM; 537 538 bp->ref_cnt = devm_kcalloc(rvu->dev, bp->bpids.max, 539 sizeof(u8), GFP_KERNEL); 540 if (!bp->ref_cnt) 541 return -ENOMEM; 542 543 return 0; 544 } 545 546 void rvu_nix_flr_free_bpids(struct rvu *rvu, u16 pcifunc) 547 { 548 int blkaddr, bpid, err; 549 struct nix_hw *nix_hw; 550 struct nix_bp *bp; 551 552 if (!is_lbk_vf(rvu, pcifunc)) 553 return; 554 555 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 556 if (err) 557 return; 558 559 bp = &nix_hw->bp; 560 561 mutex_lock(&rvu->rsrc_lock); 562 for (bpid = 0; bpid < bp->bpids.max; bpid++) { 563 if (bp->fn_map[bpid] == pcifunc) { 564 bp->ref_cnt[bpid]--; 565 if (bp->ref_cnt[bpid]) 566 continue; 567 rvu_free_rsrc(&bp->bpids, bpid); 568 bp->fn_map[bpid] = 0; 569 } 570 } 571 mutex_unlock(&rvu->rsrc_lock); 572 } 573 574 static u16 nix_get_channel(u16 chan, bool cpt_link) 575 { 576 /* CPT channel for a given link channel is always 577 * assumed to be BIT(11) set in link channel. 578 */ 579 return cpt_link ? chan | BIT(11) : chan; 580 } 581 582 static int nix_bp_disable(struct rvu *rvu, 583 struct nix_bp_cfg_req *req, 584 struct msg_rsp *rsp, bool cpt_link) 585 { 586 u16 pcifunc = req->hdr.pcifunc; 587 int blkaddr, pf, type, err; 588 u16 chan_base, chan, bpid; 589 struct rvu_pfvf *pfvf; 590 struct nix_hw *nix_hw; 591 struct nix_bp *bp; 592 u16 chan_v; 593 u64 cfg; 594 595 pf = rvu_get_pf(rvu->pdev, pcifunc); 596 type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 597 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK) 598 return 0; 599 600 if (is_sdp_pfvf(rvu, pcifunc)) 601 type = NIX_INTF_TYPE_SDP; 602 603 if (cpt_link && !rvu->hw->cpt_links) 604 return 0; 605 606 pfvf = rvu_get_pfvf(rvu, pcifunc); 607 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 608 if (err) 609 return err; 610 611 bp = &nix_hw->bp; 612 chan_base = pfvf->rx_chan_base + req->chan_base; 613 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 614 chan_v = nix_get_channel(chan, cpt_link); 615 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v)); 616 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v), 617 cfg & ~BIT_ULL(16)); 618 619 if (type == NIX_INTF_TYPE_LBK) { 620 bpid = cfg & GENMASK(8, 0); 621 mutex_lock(&rvu->rsrc_lock); 622 rvu_free_rsrc(&bp->bpids, bpid - bp->free_pool_base); 623 for (bpid = 0; bpid < bp->bpids.max; bpid++) { 624 if (bp->fn_map[bpid] == pcifunc) { 625 bp->fn_map[bpid] = 0; 626 bp->ref_cnt[bpid] = 0; 627 } 628 } 629 mutex_unlock(&rvu->rsrc_lock); 630 } 631 } 632 return 0; 633 } 634 635 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu, 636 struct nix_bp_cfg_req *req, 637 struct msg_rsp *rsp) 638 { 639 return nix_bp_disable(rvu, req, rsp, false); 640 } 641 642 int rvu_mbox_handler_nix_cpt_bp_disable(struct rvu *rvu, 643 struct nix_bp_cfg_req *req, 644 struct msg_rsp *rsp) 645 { 646 return nix_bp_disable(rvu, req, rsp, true); 647 } 648 649 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 650 int type, int chan_id) 651 { 652 int bpid, blkaddr, sdp_chan_base, err; 653 struct rvu_hwinfo *hw = rvu->hw; 654 struct rvu_pfvf *pfvf; 655 struct nix_hw *nix_hw; 656 u8 cgx_id, lmac_id; 657 struct nix_bp *bp; 658 659 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 660 661 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 662 if (err) 663 return err; 664 665 bp = &nix_hw->bp; 666 667 /* Backpressure IDs range division 668 * CGX channles are mapped to (0 - 191) BPIDs 669 * LBK channles are mapped to (192 - 255) BPIDs 670 * SDP channles are mapped to (256 - 511) BPIDs 671 * 672 * Lmac channles and bpids mapped as follows 673 * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15) 674 * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) .... 675 * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) .... 676 */ 677 switch (type) { 678 case NIX_INTF_TYPE_CGX: 679 if ((req->chan_base + req->chan_cnt) > NIX_BPIDS_PER_LMAC) 680 return NIX_AF_ERR_INVALID_BPID_REQ; 681 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 682 /* Assign bpid based on cgx, lmac and chan id */ 683 bpid = (cgx_id * hw->lmac_per_cgx * NIX_BPIDS_PER_LMAC) + 684 (lmac_id * NIX_BPIDS_PER_LMAC) + req->chan_base; 685 686 if (req->bpid_per_chan) 687 bpid += chan_id; 688 if (bpid > bp->cgx_bpid_cnt) 689 return NIX_AF_ERR_INVALID_BPID; 690 break; 691 692 case NIX_INTF_TYPE_LBK: 693 /* Alloc bpid from the free pool */ 694 mutex_lock(&rvu->rsrc_lock); 695 bpid = rvu_alloc_rsrc(&bp->bpids); 696 if (bpid < 0) { 697 mutex_unlock(&rvu->rsrc_lock); 698 return NIX_AF_ERR_INVALID_BPID; 699 } 700 bp->fn_map[bpid] = req->hdr.pcifunc; 701 bp->ref_cnt[bpid]++; 702 bpid += bp->free_pool_base; 703 mutex_unlock(&rvu->rsrc_lock); 704 break; 705 case NIX_INTF_TYPE_SDP: 706 if ((req->chan_base + req->chan_cnt) > bp->sdp_bpid_cnt) 707 return NIX_AF_ERR_INVALID_BPID_REQ; 708 709 /* Handle usecase of 2 SDP blocks */ 710 if (!hw->cap.programmable_chans) 711 sdp_chan_base = pfvf->rx_chan_base - NIX_CHAN_SDP_CH_START; 712 else 713 sdp_chan_base = pfvf->rx_chan_base - hw->sdp_chan_base; 714 715 bpid = bp->cgx_bpid_cnt + req->chan_base + sdp_chan_base; 716 if (req->bpid_per_chan) 717 bpid += chan_id; 718 719 if (bpid > (bp->cgx_bpid_cnt + bp->sdp_bpid_cnt)) 720 return NIX_AF_ERR_INVALID_BPID; 721 break; 722 default: 723 return -EINVAL; 724 } 725 return bpid; 726 } 727 728 static int nix_bp_enable(struct rvu *rvu, 729 struct nix_bp_cfg_req *req, 730 struct nix_bp_cfg_rsp *rsp, 731 bool cpt_link) 732 { 733 int blkaddr, pf, type, chan_id = 0; 734 u16 pcifunc = req->hdr.pcifunc; 735 struct rvu_pfvf *pfvf; 736 u16 chan_base, chan; 737 s16 bpid, bpid_base; 738 u16 chan_v; 739 u64 cfg; 740 741 pf = rvu_get_pf(rvu->pdev, pcifunc); 742 type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 743 if (is_sdp_pfvf(rvu, pcifunc)) 744 type = NIX_INTF_TYPE_SDP; 745 746 /* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */ 747 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 748 type != NIX_INTF_TYPE_SDP) 749 return 0; 750 751 if (cpt_link && !rvu->hw->cpt_links) 752 return 0; 753 754 pfvf = rvu_get_pfvf(rvu, pcifunc); 755 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 756 757 bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id); 758 chan_base = pfvf->rx_chan_base + req->chan_base; 759 bpid = bpid_base; 760 761 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 762 if (bpid < 0) { 763 dev_warn(rvu->dev, "Fail to enable backpressure\n"); 764 return -EINVAL; 765 } 766 767 chan_v = nix_get_channel(chan, cpt_link); 768 769 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v)); 770 cfg &= ~GENMASK_ULL(8, 0); 771 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v), 772 cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16)); 773 chan_id++; 774 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id); 775 } 776 777 for (chan = 0; chan < req->chan_cnt; chan++) { 778 /* Map channel and bpid assign to it */ 779 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 | 780 (bpid_base & 0x3FF); 781 if (req->bpid_per_chan) 782 bpid_base++; 783 } 784 rsp->chan_cnt = req->chan_cnt; 785 786 return 0; 787 } 788 789 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu, 790 struct nix_bp_cfg_req *req, 791 struct nix_bp_cfg_rsp *rsp) 792 { 793 return nix_bp_enable(rvu, req, rsp, false); 794 } 795 796 int rvu_mbox_handler_nix_cpt_bp_enable(struct rvu *rvu, 797 struct nix_bp_cfg_req *req, 798 struct nix_bp_cfg_rsp *rsp) 799 { 800 return nix_bp_enable(rvu, req, rsp, true); 801 } 802 803 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr, 804 u64 format, bool v4, u64 *fidx) 805 { 806 struct nix_lso_format field = {0}; 807 808 /* IP's Length field */ 809 field.layer = NIX_TXLAYER_OL3; 810 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ 811 field.offset = v4 ? 2 : 4; 812 field.sizem1 = 1; /* i.e 2 bytes */ 813 field.alg = NIX_LSOALG_ADD_PAYLEN; 814 rvu_write64(rvu, blkaddr, 815 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 816 *(u64 *)&field); 817 818 /* No ID field in IPv6 header */ 819 if (!v4) 820 return; 821 822 /* IP's ID field */ 823 field.layer = NIX_TXLAYER_OL3; 824 field.offset = 4; 825 field.sizem1 = 1; /* i.e 2 bytes */ 826 field.alg = NIX_LSOALG_ADD_SEGNUM; 827 rvu_write64(rvu, blkaddr, 828 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 829 *(u64 *)&field); 830 } 831 832 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr, 833 u64 format, u64 *fidx) 834 { 835 struct nix_lso_format field = {0}; 836 837 /* TCP's sequence number field */ 838 field.layer = NIX_TXLAYER_OL4; 839 field.offset = 4; 840 field.sizem1 = 3; /* i.e 4 bytes */ 841 field.alg = NIX_LSOALG_ADD_OFFSET; 842 rvu_write64(rvu, blkaddr, 843 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 844 *(u64 *)&field); 845 846 /* TCP's flags field */ 847 field.layer = NIX_TXLAYER_OL4; 848 field.offset = 12; 849 field.sizem1 = 1; /* 2 bytes */ 850 field.alg = NIX_LSOALG_TCP_FLAGS; 851 rvu_write64(rvu, blkaddr, 852 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 853 *(u64 *)&field); 854 } 855 856 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 857 { 858 u64 cfg, idx, fidx = 0; 859 860 /* Get max HW supported format indices */ 861 cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF; 862 nix_hw->lso.total = cfg; 863 864 /* Enable LSO */ 865 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG); 866 /* For TSO, set first and middle segment flags to 867 * mask out PSH, RST & FIN flags in TCP packet 868 */ 869 cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16)); 870 cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16); 871 rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63)); 872 873 /* Setup default static LSO formats 874 * 875 * Configure format fields for TCPv4 segmentation offload 876 */ 877 idx = NIX_LSO_FORMAT_IDX_TSOV4; 878 nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx); 879 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 880 881 /* Set rest of the fields to NOP */ 882 for (; fidx < 8; fidx++) { 883 rvu_write64(rvu, blkaddr, 884 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 885 } 886 nix_hw->lso.in_use++; 887 888 /* Configure format fields for TCPv6 segmentation offload */ 889 idx = NIX_LSO_FORMAT_IDX_TSOV6; 890 fidx = 0; 891 nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx); 892 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 893 894 /* Set rest of the fields to NOP */ 895 for (; fidx < 8; fidx++) { 896 rvu_write64(rvu, blkaddr, 897 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 898 } 899 nix_hw->lso.in_use++; 900 } 901 902 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) 903 { 904 kfree(pfvf->rq_bmap); 905 kfree(pfvf->sq_bmap); 906 kfree(pfvf->cq_bmap); 907 if (pfvf->rq_ctx) 908 qmem_free(rvu->dev, pfvf->rq_ctx); 909 if (pfvf->sq_ctx) 910 qmem_free(rvu->dev, pfvf->sq_ctx); 911 if (pfvf->cq_ctx) 912 qmem_free(rvu->dev, pfvf->cq_ctx); 913 if (pfvf->rss_ctx) 914 qmem_free(rvu->dev, pfvf->rss_ctx); 915 if (pfvf->nix_qints_ctx) 916 qmem_free(rvu->dev, pfvf->nix_qints_ctx); 917 if (pfvf->cq_ints_ctx) 918 qmem_free(rvu->dev, pfvf->cq_ints_ctx); 919 920 pfvf->rq_bmap = NULL; 921 pfvf->cq_bmap = NULL; 922 pfvf->sq_bmap = NULL; 923 pfvf->rq_ctx = NULL; 924 pfvf->sq_ctx = NULL; 925 pfvf->cq_ctx = NULL; 926 pfvf->rss_ctx = NULL; 927 pfvf->nix_qints_ctx = NULL; 928 pfvf->cq_ints_ctx = NULL; 929 } 930 931 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr, 932 struct rvu_pfvf *pfvf, int nixlf, 933 int rss_sz, int rss_grps, int hwctx_size, 934 u64 way_mask, bool tag_lsb_as_adder) 935 { 936 int err, grp, num_indices; 937 u64 val; 938 939 /* RSS is not requested for this NIXLF */ 940 if (!rss_sz) 941 return 0; 942 num_indices = rss_sz * rss_grps; 943 944 /* Alloc NIX RSS HW context memory and config the base */ 945 err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size); 946 if (err) 947 return err; 948 949 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf), 950 (u64)pfvf->rss_ctx->iova); 951 952 /* Config full RSS table size, enable RSS and caching */ 953 val = BIT_ULL(36) | BIT_ULL(4) | way_mask << 20 | 954 ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE); 955 956 if (tag_lsb_as_adder) 957 val |= BIT_ULL(5); 958 959 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf), val); 960 /* Config RSS group offset and sizes */ 961 for (grp = 0; grp < rss_grps; grp++) 962 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp), 963 ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp)); 964 return 0; 965 } 966 967 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, 968 struct nix_aq_inst_s *inst) 969 { 970 struct admin_queue *aq = block->aq; 971 struct nix_aq_res_s *result; 972 int timeout = 1000; 973 u64 reg, head; 974 int ret; 975 976 result = (struct nix_aq_res_s *)aq->res->base; 977 978 /* Get current head pointer where to append this instruction */ 979 reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS); 980 head = (reg >> 4) & AQ_PTR_MASK; 981 982 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), 983 (void *)inst, aq->inst->entry_sz); 984 memset(result, 0, sizeof(*result)); 985 /* sync into memory */ 986 wmb(); 987 988 /* Ring the doorbell and wait for result */ 989 rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1); 990 while (result->compcode == NIX_AQ_COMP_NOTDONE) { 991 cpu_relax(); 992 udelay(1); 993 timeout--; 994 if (!timeout) 995 return -EBUSY; 996 } 997 998 if (result->compcode != NIX_AQ_COMP_GOOD) { 999 /* TODO: Replace this with some error code */ 1000 if (result->compcode == NIX_AQ_COMP_CTX_FAULT || 1001 result->compcode == NIX_AQ_COMP_LOCKERR || 1002 result->compcode == NIX_AQ_COMP_CTX_POISON) { 1003 ret = rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_RX); 1004 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_TX); 1005 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_RX); 1006 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_TX); 1007 if (ret) 1008 dev_err(rvu->dev, 1009 "%s: Not able to unlock cachelines\n", __func__); 1010 } 1011 1012 return -EBUSY; 1013 } 1014 1015 return 0; 1016 } 1017 1018 static void nix_get_aq_req_smq(struct rvu *rvu, struct nix_aq_enq_req *req, 1019 u16 *smq, u16 *smq_mask) 1020 { 1021 struct nix_cn10k_aq_enq_req *aq_req; 1022 1023 if (is_cn20k(rvu->pdev)) { 1024 *smq = ((struct nix_cn20k_aq_enq_req *)req)->sq.smq; 1025 *smq_mask = ((struct nix_cn20k_aq_enq_req *)req)->sq_mask.smq; 1026 return; 1027 } 1028 1029 if (!is_rvu_otx2(rvu)) { 1030 aq_req = (struct nix_cn10k_aq_enq_req *)req; 1031 *smq = aq_req->sq.smq; 1032 *smq_mask = aq_req->sq_mask.smq; 1033 } else { 1034 *smq = req->sq.smq; 1035 *smq_mask = req->sq_mask.smq; 1036 } 1037 } 1038 1039 static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, 1040 struct nix_aq_enq_req *req, 1041 struct nix_aq_enq_rsp *rsp) 1042 { 1043 struct rvu_hwinfo *hw = rvu->hw; 1044 u16 pcifunc = req->hdr.pcifunc; 1045 int nixlf, blkaddr, rc = 0; 1046 struct nix_aq_inst_s inst; 1047 struct rvu_block *block; 1048 struct admin_queue *aq; 1049 struct rvu_pfvf *pfvf; 1050 u16 smq, smq_mask; 1051 void *ctx, *mask; 1052 bool ena; 1053 u64 cfg; 1054 1055 blkaddr = nix_hw->blkaddr; 1056 block = &hw->block[blkaddr]; 1057 aq = block->aq; 1058 if (!aq) { 1059 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__); 1060 return NIX_AF_ERR_AQ_ENQUEUE; 1061 } 1062 1063 pfvf = rvu_get_pfvf(rvu, pcifunc); 1064 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1065 1066 /* Skip NIXLF check for broadcast MCE entry and bandwidth profile 1067 * operations done by AF itself. 1068 */ 1069 if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) || 1070 (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) { 1071 if (!pfvf->nixlf || nixlf < 0) 1072 return NIX_AF_ERR_AF_LF_INVALID; 1073 } 1074 1075 switch (req->ctype) { 1076 case NIX_AQ_CTYPE_RQ: 1077 /* Check if index exceeds max no of queues */ 1078 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize) 1079 rc = NIX_AF_ERR_AQ_ENQUEUE; 1080 break; 1081 case NIX_AQ_CTYPE_SQ: 1082 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize) 1083 rc = NIX_AF_ERR_AQ_ENQUEUE; 1084 break; 1085 case NIX_AQ_CTYPE_CQ: 1086 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize) 1087 rc = NIX_AF_ERR_AQ_ENQUEUE; 1088 break; 1089 case NIX_AQ_CTYPE_RSS: 1090 /* Check if RSS is enabled and qidx is within range */ 1091 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf)); 1092 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx || 1093 (req->qidx >= (256UL << (cfg & 0xF)))) 1094 rc = NIX_AF_ERR_AQ_ENQUEUE; 1095 break; 1096 case NIX_AQ_CTYPE_MCE: 1097 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG); 1098 1099 /* Check if index exceeds MCE list length */ 1100 if (!nix_hw->mcast.mce_ctx || 1101 (req->qidx >= (256UL << (cfg & 0xF)))) 1102 rc = NIX_AF_ERR_AQ_ENQUEUE; 1103 1104 /* Adding multicast lists for requests from PF/VFs is not 1105 * yet supported, so ignore this. 1106 */ 1107 if (rsp) 1108 rc = NIX_AF_ERR_AQ_ENQUEUE; 1109 break; 1110 case NIX_AQ_CTYPE_BANDPROF: 1111 if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req, 1112 nix_hw, pcifunc)) 1113 rc = NIX_AF_ERR_INVALID_BANDPROF; 1114 break; 1115 default: 1116 rc = NIX_AF_ERR_AQ_ENQUEUE; 1117 } 1118 1119 if (rc) 1120 return rc; 1121 1122 nix_get_aq_req_smq(rvu, req, &smq, &smq_mask); 1123 /* Check if SQ pointed SMQ belongs to this PF/VF or not */ 1124 if (req->ctype == NIX_AQ_CTYPE_SQ && 1125 ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) || 1126 (req->op == NIX_AQ_INSTOP_WRITE && 1127 req->sq_mask.ena && req->sq.ena && smq_mask))) { 1128 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ, 1129 pcifunc, smq)) 1130 return NIX_AF_ERR_AQ_ENQUEUE; 1131 } 1132 1133 memset(&inst, 0, sizeof(struct nix_aq_inst_s)); 1134 inst.lf = nixlf; 1135 inst.cindex = req->qidx; 1136 inst.ctype = req->ctype; 1137 inst.op = req->op; 1138 /* Currently we are not supporting enqueuing multiple instructions, 1139 * so always choose first entry in result memory. 1140 */ 1141 inst.res_addr = (u64)aq->res->iova; 1142 1143 /* Hardware uses same aq->res->base for updating result of 1144 * previous instruction hence wait here till it is done. 1145 */ 1146 spin_lock(&aq->lock); 1147 1148 /* Clean result + context memory */ 1149 memset(aq->res->base, 0, aq->res->entry_sz); 1150 /* Context needs to be written at RES_ADDR + 128 */ 1151 ctx = aq->res->base + 128; 1152 /* Mask needs to be written at RES_ADDR + 256 */ 1153 mask = aq->res->base + 256; 1154 1155 switch (req->op) { 1156 case NIX_AQ_INSTOP_WRITE: 1157 if (req->ctype == NIX_AQ_CTYPE_RQ) 1158 memcpy(mask, &req->rq_mask, 1159 NIX_MAX_CTX_SIZE); 1160 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1161 memcpy(mask, &req->sq_mask, 1162 NIX_MAX_CTX_SIZE); 1163 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1164 memcpy(mask, &req->cq_mask, 1165 NIX_MAX_CTX_SIZE); 1166 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1167 memcpy(mask, &req->rss_mask, 1168 NIX_MAX_CTX_SIZE); 1169 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1170 memcpy(mask, &req->mce_mask, 1171 NIX_MAX_CTX_SIZE); 1172 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1173 memcpy(mask, &req->prof_mask, 1174 NIX_MAX_CTX_SIZE); 1175 fallthrough; 1176 case NIX_AQ_INSTOP_INIT: 1177 if (req->ctype == NIX_AQ_CTYPE_RQ) 1178 memcpy(ctx, &req->rq, NIX_MAX_CTX_SIZE); 1179 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1180 memcpy(ctx, &req->sq, NIX_MAX_CTX_SIZE); 1181 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1182 memcpy(ctx, &req->cq, NIX_MAX_CTX_SIZE); 1183 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1184 memcpy(ctx, &req->rss, NIX_MAX_CTX_SIZE); 1185 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1186 memcpy(ctx, &req->mce, NIX_MAX_CTX_SIZE); 1187 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1188 memcpy(ctx, &req->prof, NIX_MAX_CTX_SIZE); 1189 break; 1190 case NIX_AQ_INSTOP_NOP: 1191 case NIX_AQ_INSTOP_READ: 1192 case NIX_AQ_INSTOP_LOCK: 1193 case NIX_AQ_INSTOP_UNLOCK: 1194 break; 1195 default: 1196 rc = NIX_AF_ERR_AQ_ENQUEUE; 1197 spin_unlock(&aq->lock); 1198 return rc; 1199 } 1200 1201 /* Submit the instruction to AQ */ 1202 rc = nix_aq_enqueue_wait(rvu, block, &inst); 1203 if (rc) { 1204 spin_unlock(&aq->lock); 1205 return rc; 1206 } 1207 1208 /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */ 1209 if (req->op == NIX_AQ_INSTOP_INIT) { 1210 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena) 1211 __set_bit(req->qidx, pfvf->rq_bmap); 1212 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena) 1213 __set_bit(req->qidx, pfvf->sq_bmap); 1214 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena) 1215 __set_bit(req->qidx, pfvf->cq_bmap); 1216 } 1217 1218 if (req->op == NIX_AQ_INSTOP_WRITE) { 1219 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1220 ena = (req->rq.ena & req->rq_mask.ena) | 1221 (test_bit(req->qidx, pfvf->rq_bmap) & 1222 ~req->rq_mask.ena); 1223 if (ena) 1224 __set_bit(req->qidx, pfvf->rq_bmap); 1225 else 1226 __clear_bit(req->qidx, pfvf->rq_bmap); 1227 } 1228 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1229 ena = (req->rq.ena & req->sq_mask.ena) | 1230 (test_bit(req->qidx, pfvf->sq_bmap) & 1231 ~req->sq_mask.ena); 1232 if (ena) 1233 __set_bit(req->qidx, pfvf->sq_bmap); 1234 else 1235 __clear_bit(req->qidx, pfvf->sq_bmap); 1236 } 1237 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1238 ena = (req->rq.ena & req->cq_mask.ena) | 1239 (test_bit(req->qidx, pfvf->cq_bmap) & 1240 ~req->cq_mask.ena); 1241 if (ena) 1242 __set_bit(req->qidx, pfvf->cq_bmap); 1243 else 1244 __clear_bit(req->qidx, pfvf->cq_bmap); 1245 } 1246 } 1247 1248 if (rsp) { 1249 /* Copy read context into mailbox */ 1250 if (req->op == NIX_AQ_INSTOP_READ) { 1251 if (req->ctype == NIX_AQ_CTYPE_RQ) 1252 memcpy(&rsp->rq, ctx, 1253 NIX_MAX_CTX_SIZE); 1254 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1255 memcpy(&rsp->sq, ctx, 1256 NIX_MAX_CTX_SIZE); 1257 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1258 memcpy(&rsp->cq, ctx, 1259 NIX_MAX_CTX_SIZE); 1260 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1261 memcpy(&rsp->rss, ctx, 1262 NIX_MAX_CTX_SIZE); 1263 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1264 memcpy(&rsp->mce, ctx, 1265 NIX_MAX_CTX_SIZE); 1266 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1267 memcpy(&rsp->prof, ctx, 1268 NIX_MAX_CTX_SIZE); 1269 } 1270 } 1271 1272 spin_unlock(&aq->lock); 1273 return 0; 1274 } 1275 1276 static int rvu_nix_verify_aq_ctx(struct rvu *rvu, struct nix_hw *nix_hw, 1277 struct nix_aq_enq_req *req, u8 ctype) 1278 { 1279 struct nix_cn10k_aq_enq_req aq_req; 1280 struct nix_cn10k_aq_enq_rsp aq_rsp; 1281 int rc, word; 1282 1283 if (req->ctype != NIX_AQ_CTYPE_CQ) 1284 return 0; 1285 1286 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 1287 req->hdr.pcifunc, ctype, req->qidx); 1288 if (rc) { 1289 dev_err(rvu->dev, 1290 "%s: Failed to fetch %s%d context of PFFUNC 0x%x\n", 1291 __func__, nix_get_ctx_name(ctype), req->qidx, 1292 req->hdr.pcifunc); 1293 return rc; 1294 } 1295 1296 /* Make copy of original context & mask which are required 1297 * for resubmission 1298 */ 1299 memcpy(&aq_req.cq_mask, &req->cq_mask, NIX_MAX_CTX_SIZE); 1300 memcpy(&aq_req.cq, &req->cq, NIX_MAX_CTX_SIZE); 1301 1302 /* exclude fields which HW can update */ 1303 aq_req.cq_mask.cq_err = 0; 1304 aq_req.cq_mask.wrptr = 0; 1305 aq_req.cq_mask.tail = 0; 1306 aq_req.cq_mask.head = 0; 1307 aq_req.cq_mask.avg_level = 0; 1308 aq_req.cq_mask.update_time = 0; 1309 aq_req.cq_mask.substream = 0; 1310 1311 /* Context mask (cq_mask) holds mask value of fields which 1312 * are changed in AQ WRITE operation. 1313 * for example cq.drop = 0xa; 1314 * cq_mask.drop = 0xff; 1315 * Below logic performs '&' between cq and cq_mask so that non 1316 * updated fields are masked out for request and response 1317 * comparison 1318 */ 1319 for (word = 0; word < NIX_MAX_CTX_SIZE / sizeof(u64); 1320 word++) { 1321 *(u64 *)((u8 *)&aq_rsp.cq + word * 8) &= 1322 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1323 *(u64 *)((u8 *)&aq_req.cq + word * 8) &= 1324 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1325 } 1326 1327 if (memcmp(&aq_req.cq, &aq_rsp.cq, NIX_MAX_CTX_SIZE)) 1328 return NIX_AF_ERR_AQ_CTX_RETRY_WRITE; 1329 1330 return 0; 1331 } 1332 1333 int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req, 1334 struct nix_aq_enq_rsp *rsp) 1335 { 1336 struct nix_hw *nix_hw; 1337 int err, retries = 5; 1338 int blkaddr; 1339 1340 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc); 1341 if (blkaddr < 0) 1342 return NIX_AF_ERR_AF_LF_INVALID; 1343 1344 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1345 if (!nix_hw) 1346 return NIX_AF_ERR_INVALID_NIXBLK; 1347 1348 retry: 1349 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp); 1350 1351 /* HW errata 'AQ Modification to CQ could be discarded on heavy traffic' 1352 * As a work around perfrom CQ context read after each AQ write. If AQ 1353 * read shows AQ write is not updated perform AQ write again. 1354 */ 1355 if (!err && req->op == NIX_AQ_INSTOP_WRITE) { 1356 err = rvu_nix_verify_aq_ctx(rvu, nix_hw, req, NIX_AQ_CTYPE_CQ); 1357 if (err == NIX_AF_ERR_AQ_CTX_RETRY_WRITE) { 1358 if (retries--) 1359 goto retry; 1360 else 1361 return NIX_AF_ERR_CQ_CTX_WRITE_ERR; 1362 } 1363 } 1364 1365 return err; 1366 } 1367 1368 static const char *nix_get_ctx_name(int ctype) 1369 { 1370 switch (ctype) { 1371 case NIX_AQ_CTYPE_CQ: 1372 return "CQ"; 1373 case NIX_AQ_CTYPE_SQ: 1374 return "SQ"; 1375 case NIX_AQ_CTYPE_RQ: 1376 return "RQ"; 1377 case NIX_AQ_CTYPE_RSS: 1378 return "RSS"; 1379 } 1380 return ""; 1381 } 1382 1383 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) 1384 { 1385 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1386 struct nix_aq_enq_req aq_req; 1387 unsigned long *bmap; 1388 int qidx, q_cnt = 0; 1389 int err = 0, rc; 1390 1391 if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx) 1392 return NIX_AF_ERR_AQ_ENQUEUE; 1393 1394 memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); 1395 aq_req.hdr.pcifunc = req->hdr.pcifunc; 1396 1397 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1398 aq_req.cq.ena = 0; 1399 aq_req.cq_mask.ena = 1; 1400 aq_req.cq.bp_ena = 0; 1401 aq_req.cq_mask.bp_ena = 1; 1402 q_cnt = pfvf->cq_ctx->qsize; 1403 bmap = pfvf->cq_bmap; 1404 } 1405 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1406 aq_req.sq.ena = 0; 1407 aq_req.sq_mask.ena = 1; 1408 q_cnt = pfvf->sq_ctx->qsize; 1409 bmap = pfvf->sq_bmap; 1410 } 1411 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1412 aq_req.rq.ena = 0; 1413 aq_req.rq_mask.ena = 1; 1414 q_cnt = pfvf->rq_ctx->qsize; 1415 bmap = pfvf->rq_bmap; 1416 } 1417 1418 aq_req.ctype = req->ctype; 1419 aq_req.op = NIX_AQ_INSTOP_WRITE; 1420 1421 for (qidx = 0; qidx < q_cnt; qidx++) { 1422 if (!test_bit(qidx, bmap)) 1423 continue; 1424 aq_req.qidx = qidx; 1425 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL); 1426 if (rc) { 1427 err = rc; 1428 dev_err(rvu->dev, "Failed to disable %s:%d context\n", 1429 nix_get_ctx_name(req->ctype), qidx); 1430 } 1431 } 1432 1433 return err; 1434 } 1435 1436 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 1437 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req) 1438 { 1439 struct nix_aq_enq_req lock_ctx_req; 1440 int err; 1441 1442 if (req->op != NIX_AQ_INSTOP_INIT) 1443 return 0; 1444 1445 if (req->ctype == NIX_AQ_CTYPE_MCE || 1446 req->ctype == NIX_AQ_CTYPE_DYNO) 1447 return 0; 1448 1449 memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req)); 1450 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; 1451 lock_ctx_req.ctype = req->ctype; 1452 lock_ctx_req.op = NIX_AQ_INSTOP_LOCK; 1453 lock_ctx_req.qidx = req->qidx; 1454 err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL); 1455 if (err) 1456 dev_err(rvu->dev, 1457 "PFUNC 0x%x: Failed to lock NIX %s:%d context\n", 1458 req->hdr.pcifunc, 1459 nix_get_ctx_name(req->ctype), req->qidx); 1460 return err; 1461 } 1462 1463 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1464 struct nix_aq_enq_req *req, 1465 struct nix_aq_enq_rsp *rsp) 1466 { 1467 int err; 1468 1469 err = rvu_nix_aq_enq_inst(rvu, req, rsp); 1470 if (!err) 1471 err = nix_lf_hwctx_lockdown(rvu, req); 1472 return err; 1473 } 1474 #else 1475 1476 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1477 struct nix_aq_enq_req *req, 1478 struct nix_aq_enq_rsp *rsp) 1479 { 1480 return rvu_nix_aq_enq_inst(rvu, req, rsp); 1481 } 1482 #endif 1483 /* CN10K mbox handler */ 1484 int rvu_mbox_handler_nix_cn10k_aq_enq(struct rvu *rvu, 1485 struct nix_cn10k_aq_enq_req *req, 1486 struct nix_cn10k_aq_enq_rsp *rsp) 1487 { 1488 return rvu_nix_aq_enq_inst(rvu, (struct nix_aq_enq_req *)req, 1489 (struct nix_aq_enq_rsp *)rsp); 1490 } 1491 1492 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu, 1493 struct hwctx_disable_req *req, 1494 struct msg_rsp *rsp) 1495 { 1496 return nix_lf_hwctx_disable(rvu, req); 1497 } 1498 1499 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu, 1500 struct nix_lf_alloc_req *req, 1501 struct nix_lf_alloc_rsp *rsp) 1502 { 1503 int nixlf, qints, hwctx_size, intf, rc = 0; 1504 u16 bcast, mcast, promisc, ucast; 1505 struct rvu_hwinfo *hw = rvu->hw; 1506 u16 pcifunc = req->hdr.pcifunc; 1507 bool rules_created = false; 1508 struct rvu_block *block; 1509 struct rvu_pfvf *pfvf; 1510 u64 cfg, ctx_cfg; 1511 int blkaddr; 1512 1513 if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt) 1514 return NIX_AF_ERR_PARAM; 1515 1516 if (req->way_mask) 1517 req->way_mask &= 0xFFFF; 1518 1519 pfvf = rvu_get_pfvf(rvu, pcifunc); 1520 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1521 if (!pfvf->nixlf || blkaddr < 0) 1522 return NIX_AF_ERR_AF_LF_INVALID; 1523 1524 block = &hw->block[blkaddr]; 1525 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1526 if (nixlf < 0) 1527 return NIX_AF_ERR_AF_LF_INVALID; 1528 1529 /* Check if requested 'NIXLF <=> NPALF' mapping is valid */ 1530 if (req->npa_func) { 1531 /* If default, use 'this' NIXLF's PFFUNC */ 1532 if (req->npa_func == RVU_DEFAULT_PF_FUNC) 1533 req->npa_func = pcifunc; 1534 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA)) 1535 return NIX_AF_INVAL_NPA_PF_FUNC; 1536 } 1537 1538 /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */ 1539 if (req->sso_func) { 1540 /* If default, use 'this' NIXLF's PFFUNC */ 1541 if (req->sso_func == RVU_DEFAULT_PF_FUNC) 1542 req->sso_func = pcifunc; 1543 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO)) 1544 return NIX_AF_INVAL_SSO_PF_FUNC; 1545 } 1546 1547 /* If RSS is being enabled, check if requested config is valid. 1548 * RSS table size should be power of two, otherwise 1549 * RSS_GRP::OFFSET + adder might go beyond that group or 1550 * won't be able to use entire table. 1551 */ 1552 if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE || 1553 !is_power_of_2(req->rss_sz))) 1554 return NIX_AF_ERR_RSS_SIZE_INVALID; 1555 1556 if (req->rss_sz && 1557 (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS)) 1558 return NIX_AF_ERR_RSS_GRPS_INVALID; 1559 1560 /* Reset this NIX LF */ 1561 rc = rvu_lf_reset(rvu, block, nixlf); 1562 if (rc) { 1563 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1564 block->addr - BLKADDR_NIX0, nixlf); 1565 return NIX_AF_ERR_LF_RESET; 1566 } 1567 1568 ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3); 1569 1570 /* Alloc NIX RQ HW context memory and config the base */ 1571 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); 1572 rc = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size); 1573 if (rc) 1574 goto free_mem; 1575 1576 pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL); 1577 if (!pfvf->rq_bmap) { 1578 rc = -ENOMEM; 1579 goto free_mem; 1580 } 1581 1582 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf), 1583 (u64)pfvf->rq_ctx->iova); 1584 1585 /* Set caching and queue count in HW */ 1586 cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20; 1587 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg); 1588 1589 /* Alloc NIX SQ HW context memory and config the base */ 1590 hwctx_size = 1UL << (ctx_cfg & 0xF); 1591 rc = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size); 1592 if (rc) 1593 goto free_mem; 1594 1595 pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL); 1596 if (!pfvf->sq_bmap) { 1597 rc = -ENOMEM; 1598 goto free_mem; 1599 } 1600 1601 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf), 1602 (u64)pfvf->sq_ctx->iova); 1603 1604 cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20; 1605 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg); 1606 1607 /* Alloc NIX CQ HW context memory and config the base */ 1608 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); 1609 rc = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size); 1610 if (rc) 1611 goto free_mem; 1612 1613 pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL); 1614 if (!pfvf->cq_bmap) { 1615 rc = -ENOMEM; 1616 goto free_mem; 1617 } 1618 1619 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf), 1620 (u64)pfvf->cq_ctx->iova); 1621 1622 cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20; 1623 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg); 1624 1625 /* Initialize receive side scaling (RSS) */ 1626 hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF); 1627 rc = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz, 1628 req->rss_grps, hwctx_size, req->way_mask, 1629 !!(req->flags & NIX_LF_RSS_TAG_LSB_AS_ADDER)); 1630 if (rc) 1631 goto free_mem; 1632 1633 /* Alloc memory for CQINT's HW contexts */ 1634 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1635 qints = (cfg >> 24) & 0xFFF; 1636 hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF); 1637 rc = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size); 1638 if (rc) 1639 goto free_mem; 1640 1641 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf), 1642 (u64)pfvf->cq_ints_ctx->iova); 1643 1644 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf), 1645 BIT_ULL(36) | req->way_mask << 20); 1646 1647 /* Alloc memory for QINT's HW contexts */ 1648 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1649 qints = (cfg >> 12) & 0xFFF; 1650 hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF); 1651 rc = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size); 1652 if (rc) 1653 goto free_mem; 1654 1655 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf), 1656 (u64)pfvf->nix_qints_ctx->iova); 1657 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf), 1658 BIT_ULL(36) | req->way_mask << 20); 1659 1660 /* Setup VLANX TPID's. 1661 * Use VLAN1 for 802.1Q 1662 * and VLAN0 for 802.1AD. 1663 */ 1664 cfg = (0x8100ULL << 16) | 0x88A8ULL; 1665 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 1666 1667 /* Enable LMTST for this NIX LF */ 1668 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0)); 1669 1670 /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */ 1671 if (req->npa_func) 1672 cfg = req->npa_func; 1673 if (req->sso_func) 1674 cfg |= (u64)req->sso_func << 16; 1675 1676 cfg |= (u64)req->xqe_sz << 33; 1677 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg); 1678 1679 /* Config Rx pkt length, csum checks and apad enable / disable */ 1680 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg); 1681 1682 /* Configure pkind for TX parse config */ 1683 cfg = NPC_TX_DEF_PKIND; 1684 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg); 1685 1686 if (is_rep_dev(rvu, pcifunc)) { 1687 pfvf->tx_chan_base = RVU_SWITCH_LBK_CHAN; 1688 pfvf->tx_chan_cnt = 1; 1689 goto exit; 1690 } 1691 1692 intf = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 1693 if (is_sdp_pfvf(rvu, pcifunc)) 1694 intf = NIX_INTF_TYPE_SDP; 1695 1696 if (is_cn20k(rvu->pdev)) { 1697 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 1698 &promisc, &ucast); 1699 if (rc) { 1700 rc = npc_cn20k_dft_rules_alloc(rvu, pcifunc); 1701 if (rc) 1702 goto free_mem; 1703 1704 rules_created = true; 1705 } 1706 } 1707 1708 rc = nix_interface_init(rvu, pcifunc, intf, nixlf, rsp, 1709 !!(req->flags & NIX_LF_LBK_BLK_SEL)); 1710 if (rc) 1711 goto free_dft; 1712 1713 /* Disable NPC entries as NIXLF's contexts are not initialized yet */ 1714 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1715 1716 /* Configure RX VTAG Type 7 (strip) for vf vlan */ 1717 rvu_write64(rvu, blkaddr, 1718 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7), 1719 VTAGSIZE_T4 | VTAG_STRIP); 1720 1721 goto exit; 1722 1723 free_dft: 1724 if (is_cn20k(rvu->pdev) && rules_created) 1725 npc_cn20k_dft_rules_free(rvu, pcifunc); 1726 1727 free_mem: 1728 nix_ctx_free(rvu, pfvf); 1729 1730 exit: 1731 /* Set macaddr of this PF/VF */ 1732 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 1733 1734 /* set SQB size info */ 1735 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST); 1736 rsp->sqb_size = (cfg >> 34) & 0xFFFF; 1737 rsp->rx_chan_base = pfvf->rx_chan_base; 1738 rsp->tx_chan_base = pfvf->tx_chan_base; 1739 rsp->rx_chan_cnt = pfvf->rx_chan_cnt; 1740 rsp->tx_chan_cnt = pfvf->tx_chan_cnt; 1741 rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4; 1742 rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6; 1743 /* Get HW supported stat count */ 1744 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 1745 rsp->lf_rx_stats = ((cfg >> 32) & 0xFF); 1746 rsp->lf_tx_stats = ((cfg >> 24) & 0xFF); 1747 /* Get count of CQ IRQs and error IRQs supported per LF */ 1748 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1749 rsp->qints = ((cfg >> 12) & 0xFFF); 1750 rsp->cints = ((cfg >> 24) & 0xFFF); 1751 rsp->cgx_links = hw->cgx_links; 1752 rsp->lbk_links = hw->lbk_links; 1753 rsp->sdp_links = hw->sdp_links; 1754 1755 return rc; 1756 } 1757 1758 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req, 1759 struct msg_rsp *rsp) 1760 { 1761 struct rvu_hwinfo *hw = rvu->hw; 1762 u16 pcifunc = req->hdr.pcifunc; 1763 struct rvu_block *block; 1764 int blkaddr, nixlf, err; 1765 struct rvu_pfvf *pfvf; 1766 1767 pfvf = rvu_get_pfvf(rvu, pcifunc); 1768 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1769 if (!pfvf->nixlf || blkaddr < 0) 1770 return NIX_AF_ERR_AF_LF_INVALID; 1771 1772 block = &hw->block[blkaddr]; 1773 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1774 if (nixlf < 0) 1775 return NIX_AF_ERR_AF_LF_INVALID; 1776 1777 if (is_rep_dev(rvu, pcifunc)) 1778 goto free_lf; 1779 1780 if (req->flags & NIX_LF_DISABLE_FLOWS) 1781 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 1782 else 1783 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 1784 1785 /* Free any tx vtag def entries used by this NIX LF */ 1786 if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG)) 1787 nix_free_tx_vtag_entries(rvu, pcifunc); 1788 1789 nix_interface_deinit(rvu, pcifunc, nixlf); 1790 1791 free_lf: 1792 /* Reset this NIX LF */ 1793 err = rvu_lf_reset(rvu, block, nixlf); 1794 if (err) { 1795 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1796 block->addr - BLKADDR_NIX0, nixlf); 1797 return NIX_AF_ERR_LF_RESET; 1798 } 1799 1800 nix_ctx_free(rvu, pfvf); 1801 1802 if (is_cn20k(rvu->pdev) && !(req->flags & NIX_LF_DONT_FREE_DFT_IDXS)) 1803 npc_cn20k_dft_rules_free(rvu, pcifunc); 1804 1805 return 0; 1806 } 1807 1808 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu, 1809 struct nix_mark_format_cfg *req, 1810 struct nix_mark_format_cfg_rsp *rsp) 1811 { 1812 u16 pcifunc = req->hdr.pcifunc; 1813 struct nix_hw *nix_hw; 1814 struct rvu_pfvf *pfvf; 1815 int blkaddr, rc; 1816 u32 cfg; 1817 1818 pfvf = rvu_get_pfvf(rvu, pcifunc); 1819 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1820 if (!pfvf->nixlf || blkaddr < 0) 1821 return NIX_AF_ERR_AF_LF_INVALID; 1822 1823 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1824 if (!nix_hw) 1825 return NIX_AF_ERR_INVALID_NIXBLK; 1826 1827 cfg = (((u32)req->offset & 0x7) << 16) | 1828 (((u32)req->y_mask & 0xF) << 12) | 1829 (((u32)req->y_val & 0xF) << 8) | 1830 (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF); 1831 1832 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg); 1833 if (rc < 0) { 1834 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)", 1835 rvu_get_pf(rvu->pdev, pcifunc), 1836 pcifunc & RVU_PFVF_FUNC_MASK); 1837 return NIX_AF_ERR_MARK_CFG_FAIL; 1838 } 1839 1840 rsp->mark_format_idx = rc; 1841 return 0; 1842 } 1843 1844 /* Handle shaper update specially for few revisions */ 1845 static bool 1846 handle_txschq_shaper_update(struct rvu *rvu, int blkaddr, int nixlf, 1847 int lvl, u64 reg, u64 regval) 1848 { 1849 u64 regbase, oldval, sw_xoff = 0; 1850 u64 dbgval, md_debug0 = 0; 1851 unsigned long poll_tmo; 1852 bool rate_reg = 0; 1853 u32 schq; 1854 1855 regbase = reg & 0xFFFF; 1856 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 1857 1858 /* Check for rate register */ 1859 switch (lvl) { 1860 case NIX_TXSCH_LVL_TL1: 1861 md_debug0 = NIX_AF_TL1X_MD_DEBUG0(schq); 1862 sw_xoff = NIX_AF_TL1X_SW_XOFF(schq); 1863 1864 rate_reg = !!(regbase == NIX_AF_TL1X_CIR(0)); 1865 break; 1866 case NIX_TXSCH_LVL_TL2: 1867 md_debug0 = NIX_AF_TL2X_MD_DEBUG0(schq); 1868 sw_xoff = NIX_AF_TL2X_SW_XOFF(schq); 1869 1870 rate_reg = (regbase == NIX_AF_TL2X_CIR(0) || 1871 regbase == NIX_AF_TL2X_PIR(0)); 1872 break; 1873 case NIX_TXSCH_LVL_TL3: 1874 md_debug0 = NIX_AF_TL3X_MD_DEBUG0(schq); 1875 sw_xoff = NIX_AF_TL3X_SW_XOFF(schq); 1876 1877 rate_reg = (regbase == NIX_AF_TL3X_CIR(0) || 1878 regbase == NIX_AF_TL3X_PIR(0)); 1879 break; 1880 case NIX_TXSCH_LVL_TL4: 1881 md_debug0 = NIX_AF_TL4X_MD_DEBUG0(schq); 1882 sw_xoff = NIX_AF_TL4X_SW_XOFF(schq); 1883 1884 rate_reg = (regbase == NIX_AF_TL4X_CIR(0) || 1885 regbase == NIX_AF_TL4X_PIR(0)); 1886 break; 1887 case NIX_TXSCH_LVL_MDQ: 1888 sw_xoff = NIX_AF_MDQX_SW_XOFF(schq); 1889 rate_reg = (regbase == NIX_AF_MDQX_CIR(0) || 1890 regbase == NIX_AF_MDQX_PIR(0)); 1891 break; 1892 } 1893 1894 if (!rate_reg) 1895 return false; 1896 1897 /* Nothing special to do when state is not toggled */ 1898 oldval = rvu_read64(rvu, blkaddr, reg); 1899 if ((oldval & 0x1) == (regval & 0x1)) { 1900 rvu_write64(rvu, blkaddr, reg, regval); 1901 return true; 1902 } 1903 1904 /* PIR/CIR disable */ 1905 if (!(regval & 0x1)) { 1906 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1907 rvu_write64(rvu, blkaddr, reg, 0); 1908 udelay(4); 1909 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1910 return true; 1911 } 1912 1913 /* PIR/CIR enable */ 1914 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1915 if (md_debug0) { 1916 poll_tmo = jiffies + usecs_to_jiffies(10000); 1917 /* Wait until VLD(bit32) == 1 or C_CON(bit48) == 0 */ 1918 do { 1919 if (time_after(jiffies, poll_tmo)) { 1920 dev_err(rvu->dev, 1921 "NIXLF%d: TLX%u(lvl %u) CIR/PIR enable failed\n", 1922 nixlf, schq, lvl); 1923 goto exit; 1924 } 1925 usleep_range(1, 5); 1926 dbgval = rvu_read64(rvu, blkaddr, md_debug0); 1927 } while (!(dbgval & BIT_ULL(32)) && (dbgval & BIT_ULL(48))); 1928 } 1929 rvu_write64(rvu, blkaddr, reg, regval); 1930 exit: 1931 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1932 return true; 1933 } 1934 1935 static void nix_reset_tx_schedule(struct rvu *rvu, int blkaddr, 1936 int lvl, int schq) 1937 { 1938 u64 tlx_parent = 0, tlx_schedule = 0; 1939 1940 switch (lvl) { 1941 case NIX_TXSCH_LVL_TL2: 1942 tlx_parent = NIX_AF_TL2X_PARENT(schq); 1943 tlx_schedule = NIX_AF_TL2X_SCHEDULE(schq); 1944 break; 1945 case NIX_TXSCH_LVL_TL3: 1946 tlx_parent = NIX_AF_TL3X_PARENT(schq); 1947 tlx_schedule = NIX_AF_TL3X_SCHEDULE(schq); 1948 break; 1949 case NIX_TXSCH_LVL_TL4: 1950 tlx_parent = NIX_AF_TL4X_PARENT(schq); 1951 tlx_schedule = NIX_AF_TL4X_SCHEDULE(schq); 1952 break; 1953 case NIX_TXSCH_LVL_MDQ: 1954 /* no need to reset SMQ_CFG as HW clears this CSR 1955 * on SMQ flush 1956 */ 1957 tlx_parent = NIX_AF_MDQX_PARENT(schq); 1958 tlx_schedule = NIX_AF_MDQX_SCHEDULE(schq); 1959 break; 1960 default: 1961 return; 1962 } 1963 1964 if (tlx_parent) 1965 rvu_write64(rvu, blkaddr, tlx_parent, 0x0); 1966 1967 if (tlx_schedule) 1968 rvu_write64(rvu, blkaddr, tlx_schedule, 0x0); 1969 } 1970 1971 /* Disable shaping of pkts by a scheduler queue 1972 * at a given scheduler level. 1973 */ 1974 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr, 1975 int nixlf, int lvl, int schq) 1976 { 1977 struct rvu_hwinfo *hw = rvu->hw; 1978 u64 cir_reg = 0, pir_reg = 0; 1979 u64 cfg; 1980 1981 switch (lvl) { 1982 case NIX_TXSCH_LVL_TL1: 1983 cir_reg = NIX_AF_TL1X_CIR(schq); 1984 pir_reg = 0; /* PIR not available at TL1 */ 1985 break; 1986 case NIX_TXSCH_LVL_TL2: 1987 cir_reg = NIX_AF_TL2X_CIR(schq); 1988 pir_reg = NIX_AF_TL2X_PIR(schq); 1989 break; 1990 case NIX_TXSCH_LVL_TL3: 1991 cir_reg = NIX_AF_TL3X_CIR(schq); 1992 pir_reg = NIX_AF_TL3X_PIR(schq); 1993 break; 1994 case NIX_TXSCH_LVL_TL4: 1995 cir_reg = NIX_AF_TL4X_CIR(schq); 1996 pir_reg = NIX_AF_TL4X_PIR(schq); 1997 break; 1998 case NIX_TXSCH_LVL_MDQ: 1999 cir_reg = NIX_AF_MDQX_CIR(schq); 2000 pir_reg = NIX_AF_MDQX_PIR(schq); 2001 break; 2002 } 2003 2004 /* Shaper state toggle needs wait/poll */ 2005 if (hw->cap.nix_shaper_toggle_wait) { 2006 if (cir_reg) 2007 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2008 lvl, cir_reg, 0); 2009 if (pir_reg) 2010 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2011 lvl, pir_reg, 0); 2012 return; 2013 } 2014 2015 if (!cir_reg) 2016 return; 2017 cfg = rvu_read64(rvu, blkaddr, cir_reg); 2018 rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0)); 2019 2020 if (!pir_reg) 2021 return; 2022 cfg = rvu_read64(rvu, blkaddr, pir_reg); 2023 rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0)); 2024 } 2025 2026 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr, 2027 int lvl, int schq) 2028 { 2029 struct rvu_hwinfo *hw = rvu->hw; 2030 int link_level; 2031 int link; 2032 2033 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2034 return; 2035 2036 /* Reset TL4's SDP link config */ 2037 if (lvl == NIX_TXSCH_LVL_TL4) 2038 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00); 2039 2040 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2041 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2042 if (lvl != link_level) 2043 return; 2044 2045 /* Reset TL2's CGX or LBK link config */ 2046 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++) 2047 rvu_write64(rvu, blkaddr, 2048 NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00); 2049 } 2050 2051 static void nix_clear_tx_xoff(struct rvu *rvu, int blkaddr, 2052 int lvl, int schq) 2053 { 2054 struct rvu_hwinfo *hw = rvu->hw; 2055 u64 reg; 2056 2057 /* Skip this if shaping is not supported */ 2058 if (!hw->cap.nix_shaping) 2059 return; 2060 2061 /* Clear level specific SW_XOFF */ 2062 switch (lvl) { 2063 case NIX_TXSCH_LVL_TL1: 2064 reg = NIX_AF_TL1X_SW_XOFF(schq); 2065 break; 2066 case NIX_TXSCH_LVL_TL2: 2067 reg = NIX_AF_TL2X_SW_XOFF(schq); 2068 break; 2069 case NIX_TXSCH_LVL_TL3: 2070 reg = NIX_AF_TL3X_SW_XOFF(schq); 2071 break; 2072 case NIX_TXSCH_LVL_TL4: 2073 reg = NIX_AF_TL4X_SW_XOFF(schq); 2074 break; 2075 case NIX_TXSCH_LVL_MDQ: 2076 reg = NIX_AF_MDQX_SW_XOFF(schq); 2077 break; 2078 default: 2079 return; 2080 } 2081 2082 rvu_write64(rvu, blkaddr, reg, 0x0); 2083 } 2084 2085 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc) 2086 { 2087 struct rvu_hwinfo *hw = rvu->hw; 2088 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2089 u8 cgx_id = 0, lmac_id = 0; 2090 2091 if (is_lbk_vf(rvu, pcifunc)) {/* LBK links */ 2092 return hw->cgx_links; 2093 } else if (is_pf_cgxmapped(rvu, pf)) { 2094 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 2095 return (cgx_id * hw->lmac_per_cgx) + lmac_id; 2096 } 2097 2098 /* SDP link */ 2099 return hw->cgx_links + hw->lbk_links; 2100 } 2101 2102 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc, 2103 int link, int *start, int *end) 2104 { 2105 struct rvu_hwinfo *hw = rvu->hw; 2106 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2107 2108 /* LBK links */ 2109 if (is_lbk_vf(rvu, pcifunc) || is_rep_dev(rvu, pcifunc)) { 2110 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 2111 *end = *start + hw->cap.nix_txsch_per_lbk_lmac; 2112 } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */ 2113 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 2114 *end = *start + hw->cap.nix_txsch_per_cgx_lmac; 2115 } else { /* SDP link */ 2116 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) + 2117 (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links); 2118 *end = *start + hw->cap.nix_txsch_per_sdp_lmac; 2119 } 2120 } 2121 2122 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc, 2123 struct nix_hw *nix_hw, 2124 struct nix_txsch_alloc_req *req) 2125 { 2126 struct rvu_hwinfo *hw = rvu->hw; 2127 int schq, req_schq, free_cnt; 2128 struct nix_txsch *txsch; 2129 int link, start, end; 2130 2131 txsch = &nix_hw->txsch[lvl]; 2132 req_schq = req->schq_contig[lvl] + req->schq[lvl]; 2133 2134 if (!req_schq) 2135 return 0; 2136 2137 link = nix_get_tx_link(rvu, pcifunc); 2138 2139 /* For traffic aggregating scheduler level, one queue is enough */ 2140 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2141 if (req_schq != 1) 2142 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2143 return 0; 2144 } 2145 2146 /* Get free SCHQ count and check if request can be accomodated */ 2147 if (hw->cap.nix_fixed_txschq_mapping) { 2148 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 2149 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK); 2150 if (end <= txsch->schq.max && schq < end && 2151 !test_bit(schq, txsch->schq.bmap)) 2152 free_cnt = 1; 2153 else 2154 free_cnt = 0; 2155 } else { 2156 free_cnt = rvu_rsrc_free_count(&txsch->schq); 2157 } 2158 2159 if (free_cnt < req_schq || req->schq[lvl] > MAX_TXSCHQ_PER_FUNC || 2160 req->schq_contig[lvl] > MAX_TXSCHQ_PER_FUNC) 2161 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2162 2163 /* If contiguous queues are needed, check for availability */ 2164 if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] && 2165 !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl])) 2166 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2167 2168 return 0; 2169 } 2170 2171 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch, 2172 struct nix_txsch_alloc_rsp *rsp, 2173 int lvl, int start, int end) 2174 { 2175 struct rvu_hwinfo *hw = rvu->hw; 2176 u16 pcifunc = rsp->hdr.pcifunc; 2177 int idx, schq; 2178 2179 /* For traffic aggregating levels, queue alloc is based 2180 * on transmit link to which PF_FUNC is mapped to. 2181 */ 2182 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2183 /* A single TL queue is allocated */ 2184 if (rsp->schq_contig[lvl]) { 2185 rsp->schq_contig[lvl] = 1; 2186 rsp->schq_contig_list[lvl][0] = start; 2187 } 2188 2189 /* Both contig and non-contig reqs doesn't make sense here */ 2190 if (rsp->schq_contig[lvl]) 2191 rsp->schq[lvl] = 0; 2192 2193 if (rsp->schq[lvl]) { 2194 rsp->schq[lvl] = 1; 2195 rsp->schq_list[lvl][0] = start; 2196 } 2197 return; 2198 } 2199 2200 /* Adjust the queue request count if HW supports 2201 * only one queue per level configuration. 2202 */ 2203 if (hw->cap.nix_fixed_txschq_mapping) { 2204 idx = pcifunc & RVU_PFVF_FUNC_MASK; 2205 schq = start + idx; 2206 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) { 2207 rsp->schq_contig[lvl] = 0; 2208 rsp->schq[lvl] = 0; 2209 return; 2210 } 2211 2212 if (rsp->schq_contig[lvl]) { 2213 rsp->schq_contig[lvl] = 1; 2214 set_bit(schq, txsch->schq.bmap); 2215 rsp->schq_contig_list[lvl][0] = schq; 2216 rsp->schq[lvl] = 0; 2217 } else if (rsp->schq[lvl]) { 2218 rsp->schq[lvl] = 1; 2219 set_bit(schq, txsch->schq.bmap); 2220 rsp->schq_list[lvl][0] = schq; 2221 } 2222 return; 2223 } 2224 2225 /* Allocate contiguous queue indices requesty first */ 2226 if (rsp->schq_contig[lvl]) { 2227 schq = bitmap_find_next_zero_area(txsch->schq.bmap, 2228 txsch->schq.max, start, 2229 rsp->schq_contig[lvl], 0); 2230 if (schq >= end) 2231 rsp->schq_contig[lvl] = 0; 2232 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) { 2233 set_bit(schq, txsch->schq.bmap); 2234 rsp->schq_contig_list[lvl][idx] = schq; 2235 schq++; 2236 } 2237 } 2238 2239 /* Allocate non-contiguous queue indices */ 2240 if (rsp->schq[lvl]) { 2241 idx = 0; 2242 for (schq = start; schq < end; schq++) { 2243 if (!test_bit(schq, txsch->schq.bmap)) { 2244 set_bit(schq, txsch->schq.bmap); 2245 rsp->schq_list[lvl][idx++] = schq; 2246 } 2247 if (idx == rsp->schq[lvl]) 2248 break; 2249 } 2250 /* Update how many were allocated */ 2251 rsp->schq[lvl] = idx; 2252 } 2253 } 2254 2255 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu, 2256 struct nix_txsch_alloc_req *req, 2257 struct nix_txsch_alloc_rsp *rsp) 2258 { 2259 struct rvu_hwinfo *hw = rvu->hw; 2260 u16 pcifunc = req->hdr.pcifunc; 2261 int link, blkaddr, rc = 0; 2262 int lvl, idx, start, end; 2263 struct nix_txsch *txsch; 2264 struct nix_hw *nix_hw; 2265 u32 *pfvf_map; 2266 int nixlf; 2267 u16 schq; 2268 2269 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2270 if (rc) 2271 return rc; 2272 2273 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2274 if (!nix_hw) 2275 return NIX_AF_ERR_INVALID_NIXBLK; 2276 2277 mutex_lock(&rvu->rsrc_lock); 2278 2279 /* Check if request is valid as per HW capabilities 2280 * and can be accomodated. 2281 */ 2282 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2283 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req); 2284 if (rc) 2285 goto err; 2286 } 2287 2288 /* Allocate requested Tx scheduler queues */ 2289 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2290 txsch = &nix_hw->txsch[lvl]; 2291 pfvf_map = txsch->pfvf_map; 2292 2293 if (!req->schq[lvl] && !req->schq_contig[lvl]) 2294 continue; 2295 2296 rsp->schq[lvl] = req->schq[lvl]; 2297 rsp->schq_contig[lvl] = req->schq_contig[lvl]; 2298 2299 link = nix_get_tx_link(rvu, pcifunc); 2300 2301 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2302 start = link; 2303 end = link; 2304 } else if (hw->cap.nix_fixed_txschq_mapping) { 2305 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 2306 } else { 2307 start = 0; 2308 end = txsch->schq.max; 2309 } 2310 2311 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end); 2312 2313 /* Reset queue config */ 2314 for (idx = 0; idx < req->schq_contig[lvl]; idx++) { 2315 schq = rsp->schq_contig_list[lvl][idx]; 2316 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2317 NIX_TXSCHQ_CFG_DONE)) 2318 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2319 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2320 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2321 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2322 } 2323 2324 for (idx = 0; idx < req->schq[lvl]; idx++) { 2325 schq = rsp->schq_list[lvl][idx]; 2326 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2327 NIX_TXSCHQ_CFG_DONE)) 2328 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2329 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2330 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2331 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2332 } 2333 } 2334 2335 rsp->aggr_level = hw->cap.nix_tx_aggr_lvl; 2336 rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO; 2337 rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr, 2338 NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2339 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2340 goto exit; 2341 err: 2342 rc = NIX_AF_ERR_TLX_ALLOC_FAIL; 2343 exit: 2344 mutex_unlock(&rvu->rsrc_lock); 2345 return rc; 2346 } 2347 2348 static void nix_smq_flush_fill_ctx(struct rvu *rvu, int blkaddr, int smq, 2349 struct nix_smq_flush_ctx *smq_flush_ctx) 2350 { 2351 struct nix_smq_tree_ctx *smq_tree_ctx; 2352 u64 parent_off, regval; 2353 u16 schq; 2354 int lvl; 2355 2356 smq_flush_ctx->smq = smq; 2357 2358 schq = smq; 2359 for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) { 2360 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl]; 2361 smq_tree_ctx->schq = schq; 2362 if (lvl == NIX_TXSCH_LVL_TL1) { 2363 smq_tree_ctx->cir_off = NIX_AF_TL1X_CIR(schq); 2364 smq_tree_ctx->pir_off = 0; 2365 smq_tree_ctx->pir_val = 0; 2366 parent_off = 0; 2367 } else if (lvl == NIX_TXSCH_LVL_TL2) { 2368 smq_tree_ctx->cir_off = NIX_AF_TL2X_CIR(schq); 2369 smq_tree_ctx->pir_off = NIX_AF_TL2X_PIR(schq); 2370 parent_off = NIX_AF_TL2X_PARENT(schq); 2371 } else if (lvl == NIX_TXSCH_LVL_TL3) { 2372 smq_tree_ctx->cir_off = NIX_AF_TL3X_CIR(schq); 2373 smq_tree_ctx->pir_off = NIX_AF_TL3X_PIR(schq); 2374 parent_off = NIX_AF_TL3X_PARENT(schq); 2375 } else if (lvl == NIX_TXSCH_LVL_TL4) { 2376 smq_tree_ctx->cir_off = NIX_AF_TL4X_CIR(schq); 2377 smq_tree_ctx->pir_off = NIX_AF_TL4X_PIR(schq); 2378 parent_off = NIX_AF_TL4X_PARENT(schq); 2379 } else if (lvl == NIX_TXSCH_LVL_MDQ) { 2380 smq_tree_ctx->cir_off = NIX_AF_MDQX_CIR(schq); 2381 smq_tree_ctx->pir_off = NIX_AF_MDQX_PIR(schq); 2382 parent_off = NIX_AF_MDQX_PARENT(schq); 2383 } 2384 /* save cir/pir register values */ 2385 smq_tree_ctx->cir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->cir_off); 2386 if (smq_tree_ctx->pir_off) 2387 smq_tree_ctx->pir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->pir_off); 2388 2389 /* get parent txsch node */ 2390 if (parent_off) { 2391 regval = rvu_read64(rvu, blkaddr, parent_off); 2392 schq = (regval >> 16) & 0x1FF; 2393 } 2394 } 2395 } 2396 2397 static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr, 2398 struct nix_smq_flush_ctx *smq_flush_ctx, bool enable) 2399 { 2400 struct nix_txsch *txsch; 2401 struct nix_hw *nix_hw; 2402 int tl2, tl2_schq; 2403 u64 regoff; 2404 2405 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2406 if (!nix_hw) 2407 return; 2408 2409 /* loop through all TL2s with matching PF_FUNC */ 2410 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2]; 2411 tl2_schq = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL2].schq; 2412 for (tl2 = 0; tl2 < txsch->schq.max; tl2++) { 2413 /* skip the smq(flush) TL2 */ 2414 if (tl2 == tl2_schq) 2415 continue; 2416 /* skip unused TL2s */ 2417 if (TXSCH_MAP_FLAGS(txsch->pfvf_map[tl2]) & NIX_TXSCHQ_FREE) 2418 continue; 2419 /* skip if PF_FUNC doesn't match */ 2420 if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) != 2421 (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq]) & 2422 ~RVU_PFVF_FUNC_MASK)) 2423 continue; 2424 /* enable/disable XOFF */ 2425 regoff = NIX_AF_TL2X_SW_XOFF(tl2); 2426 if (enable) 2427 rvu_write64(rvu, blkaddr, regoff, 0x1); 2428 else 2429 rvu_write64(rvu, blkaddr, regoff, 0x0); 2430 } 2431 } 2432 2433 static void nix_smq_flush_enadis_rate(struct rvu *rvu, int blkaddr, 2434 struct nix_smq_flush_ctx *smq_flush_ctx, bool enable) 2435 { 2436 u64 cir_off, pir_off, cir_val, pir_val; 2437 struct nix_smq_tree_ctx *smq_tree_ctx; 2438 int lvl; 2439 2440 for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) { 2441 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl]; 2442 cir_off = smq_tree_ctx->cir_off; 2443 cir_val = smq_tree_ctx->cir_val; 2444 pir_off = smq_tree_ctx->pir_off; 2445 pir_val = smq_tree_ctx->pir_val; 2446 2447 if (enable) { 2448 rvu_write64(rvu, blkaddr, cir_off, cir_val); 2449 if (lvl != NIX_TXSCH_LVL_TL1) 2450 rvu_write64(rvu, blkaddr, pir_off, pir_val); 2451 } else { 2452 rvu_write64(rvu, blkaddr, cir_off, 0x0); 2453 if (lvl != NIX_TXSCH_LVL_TL1) 2454 rvu_write64(rvu, blkaddr, pir_off, 0x0); 2455 } 2456 } 2457 } 2458 2459 static int nix_smq_flush(struct rvu *rvu, int blkaddr, 2460 int smq, u16 pcifunc, int nixlf) 2461 { 2462 struct nix_smq_flush_ctx *smq_flush_ctx; 2463 int err, restore_tx_en = 0, i; 2464 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2465 u8 cgx_id = 0, lmac_id = 0; 2466 u16 tl2_tl3_link_schq; 2467 u8 link, link_level; 2468 u64 cfg, bmap = 0; 2469 2470 if (!is_rvu_otx2(rvu)) { 2471 /* Skip SMQ flush if pkt count is zero */ 2472 cfg = rvu_read64(rvu, blkaddr, NIX_AF_MDQX_IN_MD_COUNT(smq)); 2473 if (!cfg) 2474 return 0; 2475 } 2476 2477 /* enable cgx tx if disabled */ 2478 if (is_pf_cgxmapped(rvu, pf)) { 2479 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 2480 restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), 2481 lmac_id, true); 2482 } 2483 2484 /* XOFF all TL2s whose parent TL1 matches SMQ tree TL1 */ 2485 smq_flush_ctx = kzalloc_obj(*smq_flush_ctx); 2486 if (!smq_flush_ctx) 2487 return -ENOMEM; 2488 nix_smq_flush_fill_ctx(rvu, blkaddr, smq, smq_flush_ctx); 2489 nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, true); 2490 nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, false); 2491 2492 /* Disable backpressure from physical link, 2493 * otherwise SMQ flush may stall. 2494 */ 2495 rvu_cgx_enadis_rx_bp(rvu, pf, false); 2496 2497 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2498 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2499 tl2_tl3_link_schq = smq_flush_ctx->smq_tree_ctx[link_level].schq; 2500 link = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL1].schq; 2501 2502 /* SMQ set enqueue xoff */ 2503 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); 2504 cfg |= BIT_ULL(50); 2505 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg); 2506 2507 /* Clear all NIX_AF_TL3_TL2_LINK_CFG[ENA] for the TL3/TL2 queue */ 2508 for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { 2509 cfg = rvu_read64(rvu, blkaddr, 2510 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); 2511 if (!(cfg & BIT_ULL(12))) 2512 continue; 2513 bmap |= BIT_ULL(i); 2514 cfg &= ~BIT_ULL(12); 2515 rvu_write64(rvu, blkaddr, 2516 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); 2517 } 2518 2519 /* Do SMQ flush and set enqueue xoff */ 2520 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); 2521 cfg |= BIT_ULL(50) | BIT_ULL(49); 2522 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg); 2523 2524 /* Wait for flush to complete */ 2525 err = rvu_poll_reg(rvu, blkaddr, 2526 NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true); 2527 if (err) 2528 dev_info(rvu->dev, 2529 "NIXLF%d: SMQ%d flush failed, txlink might be busy\n", 2530 nixlf, smq); 2531 2532 /* Set NIX_AF_TL3_TL2_LINKX_CFG[ENA] for the TL3/TL2 queue */ 2533 for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { 2534 if (!(bmap & BIT_ULL(i))) 2535 continue; 2536 cfg = rvu_read64(rvu, blkaddr, 2537 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); 2538 cfg |= BIT_ULL(12); 2539 rvu_write64(rvu, blkaddr, 2540 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); 2541 } 2542 2543 /* clear XOFF on TL2s */ 2544 nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, true); 2545 nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, false); 2546 kfree(smq_flush_ctx); 2547 2548 rvu_cgx_enadis_rx_bp(rvu, pf, true); 2549 /* restore cgx tx state */ 2550 if (restore_tx_en) 2551 rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); 2552 return err; 2553 } 2554 2555 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc) 2556 { 2557 int blkaddr, nixlf, lvl, schq, err; 2558 struct rvu_hwinfo *hw = rvu->hw; 2559 struct nix_txsch *txsch; 2560 struct nix_hw *nix_hw; 2561 u16 map_func; 2562 2563 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2564 if (blkaddr < 0) 2565 return NIX_AF_ERR_AF_LF_INVALID; 2566 2567 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2568 if (!nix_hw) 2569 return NIX_AF_ERR_INVALID_NIXBLK; 2570 2571 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2572 if (nixlf < 0) 2573 return NIX_AF_ERR_AF_LF_INVALID; 2574 2575 /* Disable TL2/3 queue links and all XOFF's before SMQ flush*/ 2576 mutex_lock(&rvu->rsrc_lock); 2577 for (lvl = NIX_TXSCH_LVL_MDQ; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2578 txsch = &nix_hw->txsch[lvl]; 2579 2580 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2581 continue; 2582 2583 for (schq = 0; schq < txsch->schq.max; schq++) { 2584 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2585 continue; 2586 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2587 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2588 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2589 } 2590 } 2591 nix_clear_tx_xoff(rvu, blkaddr, NIX_TXSCH_LVL_TL1, 2592 nix_get_tx_link(rvu, pcifunc)); 2593 2594 /* On PF cleanup, clear cfg done flag as 2595 * PF would have changed default config. 2596 */ 2597 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 2598 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL1]; 2599 schq = nix_get_tx_link(rvu, pcifunc); 2600 /* Do not clear pcifunc in txsch->pfvf_map[schq] because 2601 * VF might be using this TL1 queue 2602 */ 2603 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 2604 txsch->pfvf_map[schq] = TXSCH_SET_FLAG(map_func, 0x0); 2605 } 2606 2607 /* Flush SMQs */ 2608 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ]; 2609 for (schq = 0; schq < txsch->schq.max; schq++) { 2610 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2611 continue; 2612 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2613 } 2614 2615 /* Now free scheduler queues to free pool */ 2616 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2617 /* TLs above aggregation level are shared across all PF 2618 * and it's VFs, hence skip freeing them. 2619 */ 2620 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2621 continue; 2622 2623 txsch = &nix_hw->txsch[lvl]; 2624 for (schq = 0; schq < txsch->schq.max; schq++) { 2625 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2626 continue; 2627 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2628 rvu_free_rsrc(&txsch->schq, schq); 2629 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2630 } 2631 } 2632 mutex_unlock(&rvu->rsrc_lock); 2633 2634 err = rvu_ndc_sync(rvu, blkaddr, nixlf, NIX_AF_NDC_TX_SYNC); 2635 if (err) 2636 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf); 2637 2638 return 0; 2639 } 2640 2641 static int nix_txschq_free_one(struct rvu *rvu, 2642 struct nix_txsch_free_req *req) 2643 { 2644 struct rvu_hwinfo *hw = rvu->hw; 2645 u16 pcifunc = req->hdr.pcifunc; 2646 int lvl, schq, nixlf, blkaddr; 2647 struct nix_txsch *txsch; 2648 struct nix_hw *nix_hw; 2649 u32 *pfvf_map; 2650 int rc; 2651 2652 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2653 if (blkaddr < 0) 2654 return NIX_AF_ERR_AF_LF_INVALID; 2655 2656 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2657 if (!nix_hw) 2658 return NIX_AF_ERR_INVALID_NIXBLK; 2659 2660 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2661 if (nixlf < 0) 2662 return NIX_AF_ERR_AF_LF_INVALID; 2663 2664 lvl = req->schq_lvl; 2665 schq = req->schq; 2666 txsch = &nix_hw->txsch[lvl]; 2667 2668 if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max) 2669 return 0; 2670 2671 pfvf_map = txsch->pfvf_map; 2672 mutex_lock(&rvu->rsrc_lock); 2673 2674 if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) { 2675 rc = NIX_AF_ERR_TLX_INVALID; 2676 goto err; 2677 } 2678 2679 /* Clear SW_XOFF of this resource only. 2680 * For SMQ level, all path XOFF's 2681 * need to be made clear by user 2682 */ 2683 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2684 2685 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2686 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2687 2688 /* Flush if it is a SMQ. Onus of disabling 2689 * TL2/3 queue links before SMQ flush is on user 2690 */ 2691 if (lvl == NIX_TXSCH_LVL_SMQ && 2692 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf)) { 2693 rc = NIX_AF_SMQ_FLUSH_FAILED; 2694 goto err; 2695 } 2696 2697 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2698 2699 /* Free the resource */ 2700 rvu_free_rsrc(&txsch->schq, schq); 2701 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2702 mutex_unlock(&rvu->rsrc_lock); 2703 return 0; 2704 err: 2705 mutex_unlock(&rvu->rsrc_lock); 2706 return rc; 2707 } 2708 2709 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu, 2710 struct nix_txsch_free_req *req, 2711 struct msg_rsp *rsp) 2712 { 2713 if (req->flags & TXSCHQ_FREE_ALL) 2714 return nix_txschq_free(rvu, req->hdr.pcifunc); 2715 else 2716 return nix_txschq_free_one(rvu, req); 2717 } 2718 2719 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr, 2720 int lvl, u64 reg, u64 regval) 2721 { 2722 u64 regbase = reg & 0xFFFF; 2723 u16 schq, parent; 2724 2725 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg)) 2726 return false; 2727 2728 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2729 /* Check if this schq belongs to this PF/VF or not */ 2730 if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq)) 2731 return false; 2732 2733 parent = (regval >> 16) & 0x1FF; 2734 /* Validate MDQ's TL4 parent */ 2735 if (regbase == NIX_AF_MDQX_PARENT(0) && 2736 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent)) 2737 return false; 2738 2739 /* Validate TL4's TL3 parent */ 2740 if (regbase == NIX_AF_TL4X_PARENT(0) && 2741 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent)) 2742 return false; 2743 2744 /* Validate TL3's TL2 parent */ 2745 if (regbase == NIX_AF_TL3X_PARENT(0) && 2746 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent)) 2747 return false; 2748 2749 /* Validate TL2's TL1 parent */ 2750 if (regbase == NIX_AF_TL2X_PARENT(0) && 2751 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent)) 2752 return false; 2753 2754 return true; 2755 } 2756 2757 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg) 2758 { 2759 u64 regbase; 2760 2761 if (hw->cap.nix_shaping) 2762 return true; 2763 2764 /* If shaping and coloring is not supported, then 2765 * *_CIR and *_PIR registers should not be configured. 2766 */ 2767 regbase = reg & 0xFFFF; 2768 2769 switch (lvl) { 2770 case NIX_TXSCH_LVL_TL1: 2771 if (regbase == NIX_AF_TL1X_CIR(0)) 2772 return false; 2773 break; 2774 case NIX_TXSCH_LVL_TL2: 2775 if (regbase == NIX_AF_TL2X_CIR(0) || 2776 regbase == NIX_AF_TL2X_PIR(0)) 2777 return false; 2778 break; 2779 case NIX_TXSCH_LVL_TL3: 2780 if (regbase == NIX_AF_TL3X_CIR(0) || 2781 regbase == NIX_AF_TL3X_PIR(0)) 2782 return false; 2783 break; 2784 case NIX_TXSCH_LVL_TL4: 2785 if (regbase == NIX_AF_TL4X_CIR(0) || 2786 regbase == NIX_AF_TL4X_PIR(0)) 2787 return false; 2788 break; 2789 case NIX_TXSCH_LVL_MDQ: 2790 if (regbase == NIX_AF_MDQX_CIR(0) || 2791 regbase == NIX_AF_MDQX_PIR(0)) 2792 return false; 2793 break; 2794 } 2795 return true; 2796 } 2797 2798 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw, 2799 u16 pcifunc, int blkaddr) 2800 { 2801 u32 *pfvf_map; 2802 int schq; 2803 2804 schq = nix_get_tx_link(rvu, pcifunc); 2805 pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map; 2806 /* Skip if PF has already done the config */ 2807 if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE) 2808 return; 2809 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq), 2810 (TXSCH_TL1_DFLT_RR_PRIO << 1)); 2811 2812 /* On OcteonTx2 the config was in bytes and newer silcons 2813 * it's changed to weight. 2814 */ 2815 if (!rvu->hw->cap.nix_common_dwrr_mtu) 2816 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2817 TXSCH_TL1_DFLT_RR_QTM); 2818 else 2819 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2820 CN10K_MAX_DWRR_WEIGHT); 2821 2822 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00); 2823 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE); 2824 } 2825 2826 /* Register offset - [15:0] 2827 * Scheduler Queue number - [25:16] 2828 */ 2829 #define NIX_TX_SCHQ_MASK GENMASK_ULL(25, 0) 2830 2831 static int nix_txschq_cfg_read(struct rvu *rvu, struct nix_hw *nix_hw, 2832 int blkaddr, struct nix_txschq_config *req, 2833 struct nix_txschq_config *rsp) 2834 { 2835 u16 pcifunc = req->hdr.pcifunc; 2836 int idx, schq; 2837 u64 reg; 2838 2839 for (idx = 0; idx < req->num_regs; idx++) { 2840 reg = req->reg[idx]; 2841 reg &= NIX_TX_SCHQ_MASK; 2842 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2843 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, req->lvl, reg) || 2844 !is_valid_txschq(rvu, blkaddr, req->lvl, pcifunc, schq)) 2845 return NIX_AF_INVAL_TXSCHQ_CFG; 2846 rsp->regval[idx] = rvu_read64(rvu, blkaddr, reg); 2847 } 2848 rsp->lvl = req->lvl; 2849 rsp->num_regs = req->num_regs; 2850 return 0; 2851 } 2852 2853 void rvu_nix_tx_tl2_cfg(struct rvu *rvu, int blkaddr, u16 pcifunc, 2854 struct nix_txsch *txsch, bool enable) 2855 { 2856 struct rvu_hwinfo *hw = rvu->hw; 2857 int lbk_link_start, lbk_links; 2858 u8 pf = rvu_get_pf(rvu->pdev, pcifunc); 2859 int schq; 2860 u64 cfg; 2861 2862 if (!is_pf_cgxmapped(rvu, pf) && !is_rep_dev(rvu, pcifunc)) 2863 return; 2864 2865 cfg = enable ? (BIT_ULL(12) | RVU_SWITCH_LBK_CHAN) : 0; 2866 lbk_link_start = hw->cgx_links; 2867 2868 for (schq = 0; schq < txsch->schq.max; schq++) { 2869 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2870 continue; 2871 /* Enable all LBK links with channel 63 by default so that 2872 * packets can be sent to LBK with a NPC TX MCAM rule 2873 */ 2874 lbk_links = hw->lbk_links; 2875 while (lbk_links--) 2876 rvu_write64(rvu, blkaddr, 2877 NIX_AF_TL3_TL2X_LINKX_CFG(schq, 2878 lbk_link_start + 2879 lbk_links), cfg); 2880 } 2881 } 2882 2883 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu, 2884 struct nix_txschq_config *req, 2885 struct nix_txschq_config *rsp) 2886 { 2887 u64 reg, val, regval, schq_regbase, val_mask; 2888 struct rvu_hwinfo *hw = rvu->hw; 2889 u16 pcifunc = req->hdr.pcifunc; 2890 struct nix_txsch *txsch; 2891 struct nix_hw *nix_hw; 2892 int blkaddr, idx, err; 2893 int nixlf, schq; 2894 u32 *pfvf_map; 2895 2896 if (req->lvl >= NIX_TXSCH_LVL_CNT || 2897 req->num_regs > MAX_REGS_PER_MBOX_MSG) 2898 return NIX_AF_INVAL_TXSCHQ_CFG; 2899 2900 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2901 if (err) 2902 return err; 2903 2904 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2905 if (!nix_hw) 2906 return NIX_AF_ERR_INVALID_NIXBLK; 2907 2908 if (req->read) 2909 return nix_txschq_cfg_read(rvu, nix_hw, blkaddr, req, rsp); 2910 2911 txsch = &nix_hw->txsch[req->lvl]; 2912 pfvf_map = txsch->pfvf_map; 2913 2914 if (req->lvl >= hw->cap.nix_tx_aggr_lvl && 2915 pcifunc & RVU_PFVF_FUNC_MASK) { 2916 mutex_lock(&rvu->rsrc_lock); 2917 if (req->lvl == NIX_TXSCH_LVL_TL1) 2918 nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr); 2919 mutex_unlock(&rvu->rsrc_lock); 2920 return 0; 2921 } 2922 2923 for (idx = 0; idx < req->num_regs; idx++) { 2924 reg = req->reg[idx]; 2925 reg &= NIX_TX_SCHQ_MASK; 2926 regval = req->regval[idx]; 2927 schq_regbase = reg & 0xFFFF; 2928 val_mask = req->regval_mask[idx]; 2929 2930 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr, 2931 txsch->lvl, reg, regval)) 2932 return NIX_AF_INVAL_TXSCHQ_CFG; 2933 2934 /* Check if shaping and coloring is supported */ 2935 if (!is_txschq_shaping_valid(hw, req->lvl, reg)) 2936 continue; 2937 2938 val = rvu_read64(rvu, blkaddr, reg); 2939 regval = (val & val_mask) | (regval & ~val_mask); 2940 2941 /* Handle shaping state toggle specially */ 2942 if (hw->cap.nix_shaper_toggle_wait && 2943 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2944 req->lvl, reg, regval)) 2945 continue; 2946 2947 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */ 2948 if (schq_regbase == NIX_AF_SMQX_CFG(0)) { 2949 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], 2950 pcifunc, 0); 2951 regval &= ~(0x7FULL << 24); 2952 regval |= ((u64)nixlf << 24); 2953 } 2954 2955 /* Clear 'BP_ENA' config, if it's not allowed */ 2956 if (!hw->cap.nix_tx_link_bp) { 2957 if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) || 2958 (schq_regbase & 0xFF00) == 2959 NIX_AF_TL3_TL2X_LINKX_CFG(0, 0)) 2960 regval &= ~BIT_ULL(13); 2961 } 2962 2963 /* Mark config as done for TL1 by PF */ 2964 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) && 2965 schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) { 2966 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2967 mutex_lock(&rvu->rsrc_lock); 2968 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], 2969 NIX_TXSCHQ_CFG_DONE); 2970 mutex_unlock(&rvu->rsrc_lock); 2971 } 2972 2973 /* SMQ flush is special hence split register writes such 2974 * that flush first and write rest of the bits later. 2975 */ 2976 if (schq_regbase == NIX_AF_SMQX_CFG(0) && 2977 (regval & BIT_ULL(49))) { 2978 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2979 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2980 regval &= ~BIT_ULL(49); 2981 } 2982 rvu_write64(rvu, blkaddr, reg, regval); 2983 } 2984 2985 return 0; 2986 } 2987 2988 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr, 2989 struct nix_vtag_config *req) 2990 { 2991 u64 regval = req->vtag_size; 2992 2993 if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 || 2994 req->vtag_size > VTAGSIZE_T8) 2995 return -EINVAL; 2996 2997 /* RX VTAG Type 7 reserved for vf vlan */ 2998 if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7) 2999 return NIX_AF_ERR_RX_VTAG_INUSE; 3000 3001 if (req->rx.capture_vtag) 3002 regval |= BIT_ULL(5); 3003 if (req->rx.strip_vtag) 3004 regval |= BIT_ULL(4); 3005 3006 rvu_write64(rvu, blkaddr, 3007 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval); 3008 return 0; 3009 } 3010 3011 static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr, 3012 u16 pcifunc, int index) 3013 { 3014 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3015 struct nix_txvlan *vlan; 3016 3017 if (!nix_hw) 3018 return NIX_AF_ERR_INVALID_NIXBLK; 3019 3020 vlan = &nix_hw->txvlan; 3021 if (vlan->entry2pfvf_map[index] != pcifunc) 3022 return NIX_AF_ERR_PARAM; 3023 3024 rvu_write64(rvu, blkaddr, 3025 NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull); 3026 rvu_write64(rvu, blkaddr, 3027 NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull); 3028 3029 vlan->entry2pfvf_map[index] = 0; 3030 rvu_free_rsrc(&vlan->rsrc, index); 3031 3032 return 0; 3033 } 3034 3035 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc) 3036 { 3037 struct nix_txvlan *vlan; 3038 struct nix_hw *nix_hw; 3039 int index, blkaddr; 3040 3041 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3042 if (blkaddr < 0) 3043 return; 3044 3045 nix_hw = get_nix_hw(rvu->hw, blkaddr); 3046 if (!nix_hw) 3047 return; 3048 3049 vlan = &nix_hw->txvlan; 3050 3051 mutex_lock(&vlan->rsrc_lock); 3052 /* Scan all the entries and free the ones mapped to 'pcifunc' */ 3053 for (index = 0; index < vlan->rsrc.max; index++) { 3054 if (vlan->entry2pfvf_map[index] == pcifunc) 3055 nix_tx_vtag_free(rvu, blkaddr, pcifunc, index); 3056 } 3057 mutex_unlock(&vlan->rsrc_lock); 3058 } 3059 3060 static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr, 3061 u64 vtag, u8 size) 3062 { 3063 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3064 struct nix_txvlan *vlan; 3065 u64 regval; 3066 int index; 3067 3068 if (!nix_hw) 3069 return NIX_AF_ERR_INVALID_NIXBLK; 3070 3071 vlan = &nix_hw->txvlan; 3072 3073 mutex_lock(&vlan->rsrc_lock); 3074 3075 index = rvu_alloc_rsrc(&vlan->rsrc); 3076 if (index < 0) { 3077 mutex_unlock(&vlan->rsrc_lock); 3078 return index; 3079 } 3080 3081 mutex_unlock(&vlan->rsrc_lock); 3082 3083 regval = size ? vtag : vtag << 32; 3084 3085 rvu_write64(rvu, blkaddr, 3086 NIX_AF_TX_VTAG_DEFX_DATA(index), regval); 3087 rvu_write64(rvu, blkaddr, 3088 NIX_AF_TX_VTAG_DEFX_CTL(index), size); 3089 3090 return index; 3091 } 3092 3093 static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr, 3094 struct nix_vtag_config *req) 3095 { 3096 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3097 u16 pcifunc = req->hdr.pcifunc; 3098 int idx0 = req->tx.vtag0_idx; 3099 int idx1 = req->tx.vtag1_idx; 3100 struct nix_txvlan *vlan; 3101 int err = 0; 3102 3103 if (!nix_hw) 3104 return NIX_AF_ERR_INVALID_NIXBLK; 3105 3106 vlan = &nix_hw->txvlan; 3107 if (req->tx.free_vtag0 && req->tx.free_vtag1) 3108 if (vlan->entry2pfvf_map[idx0] != pcifunc || 3109 vlan->entry2pfvf_map[idx1] != pcifunc) 3110 return NIX_AF_ERR_PARAM; 3111 3112 mutex_lock(&vlan->rsrc_lock); 3113 3114 if (req->tx.free_vtag0) { 3115 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0); 3116 if (err) 3117 goto exit; 3118 } 3119 3120 if (req->tx.free_vtag1) 3121 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1); 3122 3123 exit: 3124 mutex_unlock(&vlan->rsrc_lock); 3125 return err; 3126 } 3127 3128 static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr, 3129 struct nix_vtag_config *req, 3130 struct nix_vtag_config_rsp *rsp) 3131 { 3132 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3133 struct nix_txvlan *vlan; 3134 u16 pcifunc = req->hdr.pcifunc; 3135 3136 if (!nix_hw) 3137 return NIX_AF_ERR_INVALID_NIXBLK; 3138 3139 vlan = &nix_hw->txvlan; 3140 if (req->tx.cfg_vtag0) { 3141 rsp->vtag0_idx = 3142 nix_tx_vtag_alloc(rvu, blkaddr, 3143 req->tx.vtag0, req->vtag_size); 3144 3145 if (rsp->vtag0_idx < 0) 3146 return NIX_AF_ERR_TX_VTAG_NOSPC; 3147 3148 vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc; 3149 } 3150 3151 if (req->tx.cfg_vtag1) { 3152 rsp->vtag1_idx = 3153 nix_tx_vtag_alloc(rvu, blkaddr, 3154 req->tx.vtag1, req->vtag_size); 3155 3156 if (rsp->vtag1_idx < 0) 3157 goto err_free; 3158 3159 vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc; 3160 } 3161 3162 return 0; 3163 3164 err_free: 3165 if (req->tx.cfg_vtag0) 3166 nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx); 3167 3168 return NIX_AF_ERR_TX_VTAG_NOSPC; 3169 } 3170 3171 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu, 3172 struct nix_vtag_config *req, 3173 struct nix_vtag_config_rsp *rsp) 3174 { 3175 u16 pcifunc = req->hdr.pcifunc; 3176 int blkaddr, nixlf, err; 3177 3178 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3179 if (err) 3180 return err; 3181 3182 if (req->cfg_type) { 3183 /* rx vtag configuration */ 3184 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req); 3185 if (err) 3186 return NIX_AF_ERR_PARAM; 3187 } else { 3188 /* tx vtag configuration */ 3189 if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) && 3190 (req->tx.free_vtag0 || req->tx.free_vtag1)) 3191 return NIX_AF_ERR_PARAM; 3192 3193 if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1) 3194 return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp); 3195 3196 if (req->tx.free_vtag0 || req->tx.free_vtag1) 3197 return nix_tx_vtag_decfg(rvu, blkaddr, req); 3198 } 3199 3200 return 0; 3201 } 3202 3203 static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw, 3204 int mce, u8 op, u16 pcifunc, int next, 3205 int index, u8 mce_op, bool eol) 3206 { 3207 struct nix_aq_enq_req aq_req; 3208 int err; 3209 3210 aq_req.hdr.pcifunc = 0; 3211 aq_req.ctype = NIX_AQ_CTYPE_MCE; 3212 aq_req.op = op; 3213 aq_req.qidx = mce; 3214 3215 /* Use RSS with RSS index 0 */ 3216 aq_req.mce.op = mce_op; 3217 aq_req.mce.index = index; 3218 aq_req.mce.eol = eol; 3219 aq_req.mce.pf_func = pcifunc; 3220 aq_req.mce.next = next; 3221 3222 /* All fields valid */ 3223 *(u64 *)(&aq_req.mce_mask) = ~0ULL; 3224 3225 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL); 3226 if (err) { 3227 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n", 3228 rvu_get_pf(rvu->pdev, pcifunc), 3229 pcifunc & RVU_PFVF_FUNC_MASK); 3230 return err; 3231 } 3232 return 0; 3233 } 3234 3235 static void nix_delete_mcast_mce_list(struct nix_mce_list *mce_list) 3236 { 3237 struct hlist_node *tmp; 3238 struct mce *mce; 3239 3240 /* Scan through the current list */ 3241 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 3242 hlist_del(&mce->node); 3243 kfree(mce); 3244 } 3245 3246 mce_list->count = 0; 3247 mce_list->max = 0; 3248 } 3249 3250 static int nix_get_last_mce_list_index(struct nix_mcast_grp_elem *elem) 3251 { 3252 return elem->mce_start_index + elem->mcast_mce_list.count - 1; 3253 } 3254 3255 static int nix_update_ingress_mce_list_hw(struct rvu *rvu, 3256 struct nix_hw *nix_hw, 3257 struct nix_mcast_grp_elem *elem) 3258 { 3259 int idx, last_idx, next_idx, err; 3260 struct nix_mce_list *mce_list; 3261 struct mce *mce, *prev_mce; 3262 3263 mce_list = &elem->mcast_mce_list; 3264 idx = elem->mce_start_index; 3265 last_idx = nix_get_last_mce_list_index(elem); 3266 hlist_for_each_entry(mce, &mce_list->head, node) { 3267 if (idx > last_idx) 3268 break; 3269 3270 if (!mce->is_active) { 3271 if (idx == elem->mce_start_index) { 3272 idx++; 3273 prev_mce = mce; 3274 elem->mce_start_index = idx; 3275 continue; 3276 } else if (idx == last_idx) { 3277 err = nix_blk_setup_mce(rvu, nix_hw, idx - 1, NIX_AQ_INSTOP_WRITE, 3278 prev_mce->pcifunc, next_idx, 3279 prev_mce->rq_rss_index, 3280 prev_mce->dest_type, 3281 false); 3282 if (err) 3283 return err; 3284 3285 break; 3286 } 3287 } 3288 3289 next_idx = idx + 1; 3290 /* EOL should be set in last MCE */ 3291 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE, 3292 mce->pcifunc, next_idx, 3293 mce->rq_rss_index, mce->dest_type, 3294 (next_idx > last_idx) ? true : false); 3295 if (err) 3296 return err; 3297 3298 idx++; 3299 prev_mce = mce; 3300 } 3301 3302 return 0; 3303 } 3304 3305 static void nix_update_egress_mce_list_hw(struct rvu *rvu, 3306 struct nix_hw *nix_hw, 3307 struct nix_mcast_grp_elem *elem) 3308 { 3309 struct nix_mce_list *mce_list; 3310 int idx, last_idx, next_idx; 3311 struct mce *mce, *prev_mce; 3312 u64 regval; 3313 u8 eol; 3314 3315 mce_list = &elem->mcast_mce_list; 3316 idx = elem->mce_start_index; 3317 last_idx = nix_get_last_mce_list_index(elem); 3318 hlist_for_each_entry(mce, &mce_list->head, node) { 3319 if (idx > last_idx) 3320 break; 3321 3322 if (!mce->is_active) { 3323 if (idx == elem->mce_start_index) { 3324 idx++; 3325 prev_mce = mce; 3326 elem->mce_start_index = idx; 3327 continue; 3328 } else if (idx == last_idx) { 3329 regval = (next_idx << 16) | (1 << 12) | prev_mce->channel; 3330 rvu_write64(rvu, nix_hw->blkaddr, 3331 NIX_AF_TX_MCASTX(idx - 1), 3332 regval); 3333 break; 3334 } 3335 } 3336 3337 eol = 0; 3338 next_idx = idx + 1; 3339 /* EOL should be set in last MCE */ 3340 if (next_idx > last_idx) 3341 eol = 1; 3342 3343 regval = (next_idx << 16) | (eol << 12) | mce->channel; 3344 rvu_write64(rvu, nix_hw->blkaddr, 3345 NIX_AF_TX_MCASTX(idx), 3346 regval); 3347 idx++; 3348 prev_mce = mce; 3349 } 3350 } 3351 3352 static int nix_del_mce_list_entry(struct rvu *rvu, 3353 struct nix_hw *nix_hw, 3354 struct nix_mcast_grp_elem *elem, 3355 struct nix_mcast_grp_update_req *req) 3356 { 3357 u32 num_entry = req->num_mce_entry; 3358 struct nix_mce_list *mce_list; 3359 struct mce *mce; 3360 bool is_found; 3361 int i; 3362 3363 mce_list = &elem->mcast_mce_list; 3364 for (i = 0; i < num_entry; i++) { 3365 is_found = false; 3366 hlist_for_each_entry(mce, &mce_list->head, node) { 3367 /* If already exists, then delete */ 3368 if (mce->pcifunc == req->pcifunc[i]) { 3369 hlist_del(&mce->node); 3370 kfree(mce); 3371 mce_list->count--; 3372 is_found = true; 3373 break; 3374 } 3375 } 3376 3377 if (!is_found) 3378 return NIX_AF_ERR_INVALID_MCAST_DEL_REQ; 3379 } 3380 3381 mce_list->max = mce_list->count; 3382 /* Dump the updated list to HW */ 3383 if (elem->dir == NIX_MCAST_INGRESS) 3384 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 3385 3386 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 3387 return 0; 3388 } 3389 3390 static int nix_add_mce_list_entry(struct rvu *rvu, 3391 struct nix_hw *nix_hw, 3392 struct nix_mcast_grp_elem *elem, 3393 struct nix_mcast_grp_update_req *req) 3394 { 3395 u32 num_entry = req->num_mce_entry; 3396 struct nix_mce_list *mce_list; 3397 struct hlist_node *tmp; 3398 struct mce *mce; 3399 int i; 3400 3401 mce_list = &elem->mcast_mce_list; 3402 for (i = 0; i < num_entry; i++) { 3403 mce = kzalloc_obj(*mce); 3404 if (!mce) 3405 goto free_mce; 3406 3407 mce->pcifunc = req->pcifunc[i]; 3408 mce->channel = req->channel[i]; 3409 mce->rq_rss_index = req->rq_rss_index[i]; 3410 mce->dest_type = req->dest_type[i]; 3411 mce->is_active = 1; 3412 hlist_add_head(&mce->node, &mce_list->head); 3413 mce_list->count++; 3414 } 3415 3416 mce_list->max += num_entry; 3417 3418 /* Dump the updated list to HW */ 3419 if (elem->dir == NIX_MCAST_INGRESS) 3420 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 3421 3422 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 3423 return 0; 3424 3425 free_mce: 3426 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 3427 hlist_del(&mce->node); 3428 kfree(mce); 3429 mce_list->count--; 3430 } 3431 3432 return -ENOMEM; 3433 } 3434 3435 static int nix_update_mce_list_entry(struct nix_mce_list *mce_list, 3436 u16 pcifunc, bool add) 3437 { 3438 struct mce *mce, *tail = NULL; 3439 bool delete = false; 3440 3441 /* Scan through the current list */ 3442 hlist_for_each_entry(mce, &mce_list->head, node) { 3443 /* If already exists, then delete */ 3444 if (mce->pcifunc == pcifunc && !add) { 3445 delete = true; 3446 break; 3447 } else if (mce->pcifunc == pcifunc && add) { 3448 /* entry already exists */ 3449 return 0; 3450 } 3451 tail = mce; 3452 } 3453 3454 if (delete) { 3455 hlist_del(&mce->node); 3456 kfree(mce); 3457 mce_list->count--; 3458 return 0; 3459 } 3460 3461 if (!add) 3462 return 0; 3463 3464 /* Add a new one to the list, at the tail */ 3465 mce = kzalloc_obj(*mce); 3466 if (!mce) 3467 return -ENOMEM; 3468 mce->pcifunc = pcifunc; 3469 if (!tail) 3470 hlist_add_head(&mce->node, &mce_list->head); 3471 else 3472 hlist_add_behind(&mce->node, &tail->node); 3473 mce_list->count++; 3474 return 0; 3475 } 3476 3477 int nix_update_mce_list(struct rvu *rvu, u16 pcifunc, 3478 struct nix_mce_list *mce_list, 3479 int mce_idx, int mcam_index, bool add) 3480 { 3481 int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr; 3482 struct npc_mcam *mcam = &rvu->hw->mcam; 3483 struct nix_mcast *mcast; 3484 struct nix_hw *nix_hw; 3485 struct mce *mce; 3486 3487 if (!mce_list) 3488 return -EINVAL; 3489 3490 /* Get this PF/VF func's MCE index */ 3491 idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK); 3492 3493 if (idx > (mce_idx + mce_list->max)) { 3494 dev_err(rvu->dev, 3495 "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n", 3496 __func__, idx, mce_list->max, 3497 rvu_get_pf(rvu->pdev, pcifunc)); 3498 return -EINVAL; 3499 } 3500 3501 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 3502 if (err) 3503 return err; 3504 3505 mcast = &nix_hw->mcast; 3506 mutex_lock(&mcast->mce_lock); 3507 3508 err = nix_update_mce_list_entry(mce_list, pcifunc, add); 3509 if (err) 3510 goto end; 3511 3512 /* Disable MCAM entry in NPC */ 3513 if (!mce_list->count) { 3514 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3515 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false); 3516 goto end; 3517 } 3518 3519 /* Dump the updated list to HW */ 3520 idx = mce_idx; 3521 last_idx = idx + mce_list->count - 1; 3522 hlist_for_each_entry(mce, &mce_list->head, node) { 3523 if (idx > last_idx) 3524 break; 3525 3526 next_idx = idx + 1; 3527 /* EOL should be set in last MCE */ 3528 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE, 3529 mce->pcifunc, next_idx, 3530 0, 1, 3531 (next_idx > last_idx) ? true : false); 3532 if (err) 3533 goto end; 3534 idx++; 3535 } 3536 3537 end: 3538 mutex_unlock(&mcast->mce_lock); 3539 return err; 3540 } 3541 3542 void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type, 3543 struct nix_mce_list **mce_list, int *mce_idx) 3544 { 3545 struct rvu_hwinfo *hw = rvu->hw; 3546 struct rvu_pfvf *pfvf; 3547 3548 if (!hw->cap.nix_rx_multicast || 3549 !is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, 3550 pcifunc & ~RVU_PFVF_FUNC_MASK))) { 3551 *mce_list = NULL; 3552 *mce_idx = 0; 3553 return; 3554 } 3555 3556 /* Get this PF/VF func's MCE index */ 3557 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 3558 3559 if (type == NIXLF_BCAST_ENTRY) { 3560 *mce_list = &pfvf->bcast_mce_list; 3561 *mce_idx = pfvf->bcast_mce_idx; 3562 } else if (type == NIXLF_ALLMULTI_ENTRY) { 3563 *mce_list = &pfvf->mcast_mce_list; 3564 *mce_idx = pfvf->mcast_mce_idx; 3565 } else if (type == NIXLF_PROMISC_ENTRY) { 3566 *mce_list = &pfvf->promisc_mce_list; 3567 *mce_idx = pfvf->promisc_mce_idx; 3568 } else { 3569 *mce_list = NULL; 3570 *mce_idx = 0; 3571 } 3572 } 3573 3574 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 3575 int type, bool add) 3576 { 3577 int err = 0, nixlf, blkaddr, mcam_index, mce_idx; 3578 struct npc_mcam *mcam = &rvu->hw->mcam; 3579 struct rvu_hwinfo *hw = rvu->hw; 3580 struct nix_mce_list *mce_list; 3581 int pf; 3582 3583 /* skip multicast pkt replication for AF's VFs & SDP links */ 3584 if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(rvu, pcifunc)) 3585 return 0; 3586 3587 if (!hw->cap.nix_rx_multicast) 3588 return 0; 3589 3590 pf = rvu_get_pf(rvu->pdev, pcifunc); 3591 if (!is_pf_cgxmapped(rvu, pf)) 3592 return 0; 3593 3594 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3595 if (blkaddr < 0) 3596 return -EINVAL; 3597 3598 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 3599 if (nixlf < 0) 3600 return -EINVAL; 3601 3602 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 3603 3604 mcam_index = npc_get_nixlf_mcam_index(mcam, 3605 pcifunc & ~RVU_PFVF_FUNC_MASK, 3606 nixlf, type); 3607 if (mcam_index < 0) 3608 return -EINVAL; 3609 3610 err = nix_update_mce_list(rvu, pcifunc, mce_list, 3611 mce_idx, mcam_index, add); 3612 return err; 3613 } 3614 3615 static void nix_setup_mcast_grp(struct nix_hw *nix_hw) 3616 { 3617 struct nix_mcast_grp *mcast_grp = &nix_hw->mcast_grp; 3618 3619 INIT_LIST_HEAD(&mcast_grp->mcast_grp_head); 3620 mutex_init(&mcast_grp->mcast_grp_lock); 3621 mcast_grp->next_grp_index = 1; 3622 mcast_grp->count = 0; 3623 } 3624 3625 static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw) 3626 { 3627 struct nix_mcast *mcast = &nix_hw->mcast; 3628 int err, pf, numvfs, idx; 3629 struct rvu_pfvf *pfvf; 3630 u16 pcifunc; 3631 u64 cfg; 3632 3633 /* Skip PF0 (i.e AF) */ 3634 for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) { 3635 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3636 /* If PF is not enabled, nothing to do */ 3637 if (!((cfg >> 20) & 0x01)) 3638 continue; 3639 /* Get numVFs attached to this PF */ 3640 numvfs = (cfg >> 12) & 0xFF; 3641 3642 pfvf = &rvu->pf[pf]; 3643 3644 /* This NIX0/1 block mapped to PF ? */ 3645 if (pfvf->nix_blkaddr != nix_hw->blkaddr) 3646 continue; 3647 3648 /* save start idx of broadcast mce list */ 3649 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3650 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1); 3651 3652 /* save start idx of multicast mce list */ 3653 pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3654 nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1); 3655 3656 /* save the start idx of promisc mce list */ 3657 pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3658 nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1); 3659 3660 for (idx = 0; idx < (numvfs + 1); idx++) { 3661 /* idx-0 is for PF, followed by VFs */ 3662 pcifunc = rvu_make_pcifunc(rvu->pdev, pf, 0); 3663 pcifunc |= idx; 3664 /* Add dummy entries now, so that we don't have to check 3665 * for whether AQ_OP should be INIT/WRITE later on. 3666 * Will be updated when a NIXLF is attached/detached to 3667 * these PF/VFs. 3668 */ 3669 err = nix_blk_setup_mce(rvu, nix_hw, 3670 pfvf->bcast_mce_idx + idx, 3671 NIX_AQ_INSTOP_INIT, 3672 pcifunc, 0, 0, 1, true); 3673 if (err) 3674 return err; 3675 3676 /* add dummy entries to multicast mce list */ 3677 err = nix_blk_setup_mce(rvu, nix_hw, 3678 pfvf->mcast_mce_idx + idx, 3679 NIX_AQ_INSTOP_INIT, 3680 pcifunc, 0, 0, 1, true); 3681 if (err) 3682 return err; 3683 3684 /* add dummy entries to promisc mce list */ 3685 err = nix_blk_setup_mce(rvu, nix_hw, 3686 pfvf->promisc_mce_idx + idx, 3687 NIX_AQ_INSTOP_INIT, 3688 pcifunc, 0, 0, 1, true); 3689 if (err) 3690 return err; 3691 } 3692 } 3693 return 0; 3694 } 3695 3696 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3697 { 3698 struct nix_mcast *mcast = &nix_hw->mcast; 3699 struct rvu_hwinfo *hw = rvu->hw; 3700 int err, size; 3701 3702 size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F; 3703 size = BIT_ULL(size); 3704 3705 /* Allocate bitmap for rx mce entries */ 3706 mcast->mce_counter[NIX_MCAST_INGRESS].max = 256UL << MC_TBL_SIZE; 3707 err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3708 if (err) 3709 return -ENOMEM; 3710 3711 /* Allocate bitmap for tx mce entries */ 3712 mcast->mce_counter[NIX_MCAST_EGRESS].max = MC_TX_MAX; 3713 err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3714 if (err) { 3715 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3716 return -ENOMEM; 3717 } 3718 3719 /* Alloc memory for multicast/mirror replication entries */ 3720 err = qmem_alloc(rvu->dev, &mcast->mce_ctx, 3721 mcast->mce_counter[NIX_MCAST_INGRESS].max, size); 3722 if (err) { 3723 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3724 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3725 return -ENOMEM; 3726 } 3727 3728 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE, 3729 (u64)mcast->mce_ctx->iova); 3730 3731 /* Set max list length equal to max no of VFs per PF + PF itself */ 3732 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG, 3733 BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE); 3734 3735 /* Alloc memory for multicast replication buffers */ 3736 size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF; 3737 err = qmem_alloc(rvu->dev, &mcast->mcast_buf, 3738 (8UL << MC_BUF_CNT), size); 3739 if (err) { 3740 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3741 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3742 return -ENOMEM; 3743 } 3744 3745 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE, 3746 (u64)mcast->mcast_buf->iova); 3747 3748 /* Alloc pkind for NIX internal RX multicast/mirror replay */ 3749 mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc); 3750 3751 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG, 3752 BIT_ULL(63) | (mcast->replay_pkind << 24) | 3753 BIT_ULL(20) | MC_BUF_CNT); 3754 3755 mutex_init(&mcast->mce_lock); 3756 3757 nix_setup_mcast_grp(nix_hw); 3758 3759 return nix_setup_mce_tables(rvu, nix_hw); 3760 } 3761 3762 static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw) 3763 { 3764 struct nix_txvlan *vlan = &nix_hw->txvlan; 3765 int err; 3766 3767 /* Allocate resource bimap for tx vtag def registers*/ 3768 vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX; 3769 err = rvu_alloc_bitmap(&vlan->rsrc); 3770 if (err) 3771 return -ENOMEM; 3772 3773 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 3774 vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max, 3775 sizeof(u16), GFP_KERNEL); 3776 if (!vlan->entry2pfvf_map) 3777 goto free_mem; 3778 3779 mutex_init(&vlan->rsrc_lock); 3780 return 0; 3781 3782 free_mem: 3783 kfree(vlan->rsrc.bmap); 3784 return -ENOMEM; 3785 } 3786 3787 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3788 { 3789 struct nix_txsch *txsch; 3790 int err, lvl, schq; 3791 u64 cfg, reg; 3792 3793 /* Get scheduler queue count of each type and alloc 3794 * bitmap for each for alloc/free/attach operations. 3795 */ 3796 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 3797 txsch = &nix_hw->txsch[lvl]; 3798 txsch->lvl = lvl; 3799 switch (lvl) { 3800 case NIX_TXSCH_LVL_SMQ: 3801 reg = NIX_AF_MDQ_CONST; 3802 break; 3803 case NIX_TXSCH_LVL_TL4: 3804 reg = NIX_AF_TL4_CONST; 3805 break; 3806 case NIX_TXSCH_LVL_TL3: 3807 reg = NIX_AF_TL3_CONST; 3808 break; 3809 case NIX_TXSCH_LVL_TL2: 3810 reg = NIX_AF_TL2_CONST; 3811 break; 3812 case NIX_TXSCH_LVL_TL1: 3813 reg = NIX_AF_TL1_CONST; 3814 break; 3815 } 3816 cfg = rvu_read64(rvu, blkaddr, reg); 3817 txsch->schq.max = cfg & 0xFFFF; 3818 err = rvu_alloc_bitmap(&txsch->schq); 3819 if (err) 3820 return err; 3821 3822 /* Allocate memory for scheduler queues to 3823 * PF/VF pcifunc mapping info. 3824 */ 3825 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max, 3826 sizeof(u32), GFP_KERNEL); 3827 if (!txsch->pfvf_map) 3828 return -ENOMEM; 3829 for (schq = 0; schq < txsch->schq.max; schq++) 3830 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 3831 } 3832 3833 /* Setup a default value of 8192 as DWRR MTU */ 3834 if (rvu->hw->cap.nix_common_dwrr_mtu || 3835 rvu->hw->cap.nix_multiple_dwrr_mtu) { 3836 rvu_write64(rvu, blkaddr, 3837 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM), 3838 convert_bytes_to_dwrr_mtu(8192)); 3839 rvu_write64(rvu, blkaddr, 3840 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK), 3841 convert_bytes_to_dwrr_mtu(8192)); 3842 rvu_write64(rvu, blkaddr, 3843 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP), 3844 convert_bytes_to_dwrr_mtu(8192)); 3845 } 3846 3847 return 0; 3848 } 3849 3850 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw, 3851 int blkaddr, u32 cfg) 3852 { 3853 int fmt_idx; 3854 3855 for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) { 3856 if (nix_hw->mark_format.cfg[fmt_idx] == cfg) 3857 return fmt_idx; 3858 } 3859 if (fmt_idx >= nix_hw->mark_format.total) 3860 return -ERANGE; 3861 3862 rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg); 3863 nix_hw->mark_format.cfg[fmt_idx] = cfg; 3864 nix_hw->mark_format.in_use++; 3865 return fmt_idx; 3866 } 3867 3868 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw, 3869 int blkaddr) 3870 { 3871 u64 cfgs[] = { 3872 [NIX_MARK_CFG_IP_DSCP_RED] = 0x10003, 3873 [NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200, 3874 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203, 3875 [NIX_MARK_CFG_IP_ECN_RED] = 0x6000c, 3876 [NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00, 3877 [NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c, 3878 [NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008, 3879 [NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800, 3880 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808, 3881 }; 3882 int i, rc; 3883 u64 total; 3884 3885 total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8; 3886 nix_hw->mark_format.total = (u8)total; 3887 nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32), 3888 GFP_KERNEL); 3889 if (!nix_hw->mark_format.cfg) 3890 return -ENOMEM; 3891 for (i = 0; i < NIX_MARK_CFG_MAX; i++) { 3892 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]); 3893 if (rc < 0) 3894 dev_err(rvu->dev, "Err %d in setup mark format %d\n", 3895 i, rc); 3896 } 3897 3898 return 0; 3899 } 3900 3901 static void rvu_get_lbk_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3902 { 3903 /* CN10K supports LBK FIFO size 72 KB */ 3904 if (rvu->hw->lbk_bufsize == 0x12000) 3905 *max_mtu = CN10K_LBK_LINK_MAX_FRS; 3906 else 3907 *max_mtu = NIC_HW_MAX_FRS; 3908 } 3909 3910 static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3911 { 3912 int fifo_size = rvu_cgx_get_fifolen(rvu); 3913 3914 /* RPM supports FIFO len 128 KB and RPM2 supports double the 3915 * FIFO len to accommodate 8 LMACS 3916 */ 3917 if (fifo_size == 0x20000 || fifo_size == 0x40000) 3918 *max_mtu = CN10K_LMAC_LINK_MAX_FRS; 3919 else 3920 *max_mtu = NIC_HW_MAX_FRS; 3921 } 3922 3923 int rvu_mbox_handler_nix_get_hw_info(struct rvu *rvu, struct msg_req *req, 3924 struct nix_hw_info *rsp) 3925 { 3926 u16 pcifunc = req->hdr.pcifunc; 3927 u64 dwrr_mtu; 3928 int blkaddr; 3929 3930 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3931 if (blkaddr < 0) 3932 return NIX_AF_ERR_AF_LF_INVALID; 3933 3934 if (is_lbk_vf(rvu, pcifunc)) 3935 rvu_get_lbk_link_max_frs(rvu, &rsp->max_mtu); 3936 else 3937 rvu_get_lmac_link_max_frs(rvu, &rsp->max_mtu); 3938 3939 rsp->min_mtu = NIC_HW_MIN_FRS; 3940 3941 if (!rvu->hw->cap.nix_common_dwrr_mtu && 3942 !rvu->hw->cap.nix_multiple_dwrr_mtu) { 3943 /* Return '1' on OTx2 */ 3944 rsp->rpm_dwrr_mtu = 1; 3945 rsp->sdp_dwrr_mtu = 1; 3946 rsp->lbk_dwrr_mtu = 1; 3947 return 0; 3948 } 3949 3950 /* Return DWRR_MTU for TLx_SCHEDULE[RR_WEIGHT] config */ 3951 dwrr_mtu = rvu_read64(rvu, blkaddr, 3952 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM)); 3953 rsp->rpm_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3954 3955 dwrr_mtu = rvu_read64(rvu, blkaddr, 3956 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP)); 3957 rsp->sdp_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3958 3959 dwrr_mtu = rvu_read64(rvu, blkaddr, 3960 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK)); 3961 rsp->lbk_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3962 3963 return 0; 3964 } 3965 3966 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req, 3967 struct msg_rsp *rsp) 3968 { 3969 u16 pcifunc = req->hdr.pcifunc; 3970 int i, nixlf, blkaddr, err; 3971 u64 stats; 3972 3973 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3974 if (err) 3975 return err; 3976 3977 /* Get stats count supported by HW */ 3978 stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 3979 3980 /* Reset tx stats */ 3981 for (i = 0; i < ((stats >> 24) & 0xFF); i++) 3982 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0); 3983 3984 /* Reset rx stats */ 3985 for (i = 0; i < ((stats >> 32) & 0xFF); i++) 3986 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0); 3987 3988 return 0; 3989 } 3990 3991 /* Returns the ALG index to be set into NPC_RX_ACTION */ 3992 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg) 3993 { 3994 int i; 3995 3996 /* Scan over exiting algo entries to find a match */ 3997 for (i = 0; i < nix_hw->flowkey.in_use; i++) 3998 if (nix_hw->flowkey.flowkey[i] == flow_cfg) 3999 return i; 4000 4001 return -ERANGE; 4002 } 4003 4004 /* Mask to match ipv6(NPC_LT_LC_IP6) and ipv6 ext(NPC_LT_LC_IP6_EXT) */ 4005 #define NPC_LT_LC_IP6_MATCH_MSK ((~(NPC_LT_LC_IP6 ^ NPC_LT_LC_IP6_EXT)) & 0xf) 4006 /* Mask to match both ipv4(NPC_LT_LC_IP) and ipv4 ext(NPC_LT_LC_IP_OPT) */ 4007 #define NPC_LT_LC_IP_MATCH_MSK ((~(NPC_LT_LC_IP ^ NPC_LT_LC_IP_OPT)) & 0xf) 4008 4009 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg) 4010 { 4011 int idx, nr_field, key_off, field_marker, keyoff_marker; 4012 int max_key_off, max_bit_pos, group_member; 4013 struct nix_rx_flowkey_alg *field; 4014 struct nix_rx_flowkey_alg tmp; 4015 u32 key_type, valid_key; 4016 u32 l3_l4_src_dst; 4017 int l4_key_offset = 0; 4018 4019 if (!alg) 4020 return -EINVAL; 4021 4022 #define FIELDS_PER_ALG 5 4023 #define MAX_KEY_OFF 40 4024 /* Clear all fields */ 4025 memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG); 4026 4027 /* Each of the 32 possible flow key algorithm definitions should 4028 * fall into above incremental config (except ALG0). Otherwise a 4029 * single NPC MCAM entry is not sufficient for supporting RSS. 4030 * 4031 * If a different definition or combination needed then NPC MCAM 4032 * has to be programmed to filter such pkts and it's action should 4033 * point to this definition to calculate flowtag or hash. 4034 * 4035 * The `for loop` goes over _all_ protocol field and the following 4036 * variables depicts the state machine forward progress logic. 4037 * 4038 * keyoff_marker - Enabled when hash byte length needs to be accounted 4039 * in field->key_offset update. 4040 * field_marker - Enabled when a new field needs to be selected. 4041 * group_member - Enabled when protocol is part of a group. 4042 */ 4043 4044 /* Last 4 bits (31:28) are reserved to specify SRC, DST 4045 * selection for L3, L4 i.e IPV[4,6]_SRC, IPV[4,6]_DST, 4046 * [TCP,UDP,SCTP]_SRC, [TCP,UDP,SCTP]_DST 4047 * 31 => L3_SRC, 30 => L3_DST, 29 => L4_SRC, 28 => L4_DST 4048 */ 4049 l3_l4_src_dst = flow_cfg; 4050 /* Reset these 4 bits, so that these won't be part of key */ 4051 flow_cfg &= NIX_FLOW_KEY_TYPE_L3_L4_MASK; 4052 4053 keyoff_marker = 0; max_key_off = 0; group_member = 0; 4054 nr_field = 0; key_off = 0; field_marker = 1; 4055 field = &tmp; max_bit_pos = fls(flow_cfg); 4056 for (idx = 0; 4057 idx < max_bit_pos && nr_field < FIELDS_PER_ALG && 4058 key_off < MAX_KEY_OFF; idx++) { 4059 key_type = BIT(idx); 4060 valid_key = flow_cfg & key_type; 4061 /* Found a field marker, reset the field values */ 4062 if (field_marker) 4063 memset(&tmp, 0, sizeof(tmp)); 4064 4065 field_marker = true; 4066 keyoff_marker = true; 4067 switch (key_type) { 4068 case NIX_FLOW_KEY_TYPE_PORT: 4069 field->sel_chan = true; 4070 /* This should be set to 1, when SEL_CHAN is set */ 4071 field->bytesm1 = 1; 4072 break; 4073 case NIX_FLOW_KEY_TYPE_IPV4_PROTO: 4074 field->lid = NPC_LID_LC; 4075 field->hdr_offset = 9; /* offset */ 4076 field->bytesm1 = 0; /* 1 byte */ 4077 field->ltype_match = NPC_LT_LC_IP; 4078 field->ltype_mask = NPC_LT_LC_IP_MATCH_MSK; 4079 break; 4080 case NIX_FLOW_KEY_TYPE_IPV4: 4081 case NIX_FLOW_KEY_TYPE_INNR_IPV4: 4082 field->lid = NPC_LID_LC; 4083 field->ltype_match = NPC_LT_LC_IP; 4084 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) { 4085 field->lid = NPC_LID_LG; 4086 field->ltype_match = NPC_LT_LG_TU_IP; 4087 } 4088 field->hdr_offset = 12; /* SIP offset */ 4089 field->bytesm1 = 7; /* SIP + DIP, 8 bytes */ 4090 4091 /* Only SIP */ 4092 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY) 4093 field->bytesm1 = 3; /* SIP, 4 bytes */ 4094 4095 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) { 4096 /* Both SIP + DIP */ 4097 if (field->bytesm1 == 3) { 4098 field->bytesm1 = 7; /* SIP + DIP, 8B */ 4099 } else { 4100 /* Only DIP */ 4101 field->hdr_offset = 16; /* DIP off */ 4102 field->bytesm1 = 3; /* DIP, 4 bytes */ 4103 } 4104 } 4105 field->ltype_mask = NPC_LT_LC_IP_MATCH_MSK; 4106 keyoff_marker = false; 4107 break; 4108 case NIX_FLOW_KEY_TYPE_IPV6: 4109 case NIX_FLOW_KEY_TYPE_INNR_IPV6: 4110 field->lid = NPC_LID_LC; 4111 field->ltype_match = NPC_LT_LC_IP6; 4112 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) { 4113 field->lid = NPC_LID_LG; 4114 field->ltype_match = NPC_LT_LG_TU_IP6; 4115 } 4116 field->hdr_offset = 8; /* SIP offset */ 4117 field->bytesm1 = 31; /* SIP + DIP, 32 bytes */ 4118 4119 /* Only SIP */ 4120 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY) 4121 field->bytesm1 = 15; /* SIP, 16 bytes */ 4122 4123 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) { 4124 /* Both SIP + DIP */ 4125 if (field->bytesm1 == 15) { 4126 /* SIP + DIP, 32 bytes */ 4127 field->bytesm1 = 31; 4128 } else { 4129 /* Only DIP */ 4130 field->hdr_offset = 24; /* DIP off */ 4131 field->bytesm1 = 15; /* DIP,16 bytes */ 4132 } 4133 } 4134 field->ltype_mask = NPC_LT_LC_IP6_MATCH_MSK; 4135 break; 4136 case NIX_FLOW_KEY_TYPE_TCP: 4137 case NIX_FLOW_KEY_TYPE_UDP: 4138 case NIX_FLOW_KEY_TYPE_SCTP: 4139 case NIX_FLOW_KEY_TYPE_INNR_TCP: 4140 case NIX_FLOW_KEY_TYPE_INNR_UDP: 4141 case NIX_FLOW_KEY_TYPE_INNR_SCTP: 4142 field->lid = NPC_LID_LD; 4143 if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP || 4144 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP || 4145 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) 4146 field->lid = NPC_LID_LH; 4147 field->bytesm1 = 3; /* Sport + Dport, 4 bytes */ 4148 4149 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_SRC_ONLY) 4150 field->bytesm1 = 1; /* SRC, 2 bytes */ 4151 4152 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_DST_ONLY) { 4153 /* Both SRC + DST */ 4154 if (field->bytesm1 == 1) { 4155 /* SRC + DST, 4 bytes */ 4156 field->bytesm1 = 3; 4157 } else { 4158 /* Only DIP */ 4159 field->hdr_offset = 2; /* DST off */ 4160 field->bytesm1 = 1; /* DST, 2 bytes */ 4161 } 4162 } 4163 4164 /* Enum values for NPC_LID_LD and NPC_LID_LG are same, 4165 * so no need to change the ltype_match, just change 4166 * the lid for inner protocols 4167 */ 4168 BUILD_BUG_ON((int)NPC_LT_LD_TCP != 4169 (int)NPC_LT_LH_TU_TCP); 4170 BUILD_BUG_ON((int)NPC_LT_LD_UDP != 4171 (int)NPC_LT_LH_TU_UDP); 4172 BUILD_BUG_ON((int)NPC_LT_LD_SCTP != 4173 (int)NPC_LT_LH_TU_SCTP); 4174 4175 if ((key_type == NIX_FLOW_KEY_TYPE_TCP || 4176 key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) && 4177 valid_key) { 4178 field->ltype_match |= NPC_LT_LD_TCP; 4179 group_member = true; 4180 } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP || 4181 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) && 4182 valid_key) { 4183 field->ltype_match |= NPC_LT_LD_UDP; 4184 group_member = true; 4185 } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP || 4186 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) && 4187 valid_key) { 4188 field->ltype_match |= NPC_LT_LD_SCTP; 4189 group_member = true; 4190 } 4191 field->ltype_mask = ~field->ltype_match; 4192 if (key_type == NIX_FLOW_KEY_TYPE_SCTP || 4193 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) { 4194 /* Handle the case where any of the group item 4195 * is enabled in the group but not the final one 4196 */ 4197 if (group_member) { 4198 valid_key = true; 4199 group_member = false; 4200 } 4201 } else { 4202 field_marker = false; 4203 keyoff_marker = false; 4204 } 4205 4206 /* TCP/UDP/SCTP and ESP/AH falls at same offset so 4207 * remember the TCP key offset of 40 byte hash key. 4208 */ 4209 if (key_type == NIX_FLOW_KEY_TYPE_TCP) 4210 l4_key_offset = key_off; 4211 break; 4212 case NIX_FLOW_KEY_TYPE_NVGRE: 4213 field->lid = NPC_LID_LD; 4214 field->hdr_offset = 4; /* VSID offset */ 4215 field->bytesm1 = 2; 4216 field->ltype_match = NPC_LT_LD_NVGRE; 4217 field->ltype_mask = 0xF; 4218 break; 4219 case NIX_FLOW_KEY_TYPE_VXLAN: 4220 case NIX_FLOW_KEY_TYPE_GENEVE: 4221 field->lid = NPC_LID_LE; 4222 field->bytesm1 = 2; 4223 field->hdr_offset = 4; 4224 field->ltype_mask = 0xF; 4225 field_marker = false; 4226 keyoff_marker = false; 4227 4228 if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) { 4229 field->ltype_match |= NPC_LT_LE_VXLAN; 4230 group_member = true; 4231 } 4232 4233 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) { 4234 field->ltype_match |= NPC_LT_LE_GENEVE; 4235 group_member = true; 4236 } 4237 4238 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) { 4239 if (group_member) { 4240 field->ltype_mask = ~field->ltype_match; 4241 field_marker = true; 4242 keyoff_marker = true; 4243 valid_key = true; 4244 group_member = false; 4245 } 4246 } 4247 break; 4248 case NIX_FLOW_KEY_TYPE_ETH_DMAC: 4249 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC: 4250 field->lid = NPC_LID_LA; 4251 field->ltype_match = NPC_LT_LA_ETHER; 4252 if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) { 4253 field->lid = NPC_LID_LF; 4254 field->ltype_match = NPC_LT_LF_TU_ETHER; 4255 } 4256 field->hdr_offset = 0; 4257 field->bytesm1 = 5; /* DMAC 6 Byte */ 4258 field->ltype_mask = 0xF; 4259 break; 4260 case NIX_FLOW_KEY_TYPE_IPV6_EXT: 4261 field->lid = NPC_LID_LC; 4262 field->hdr_offset = 40; /* IPV6 hdr */ 4263 field->bytesm1 = 0; /* 1 Byte ext hdr*/ 4264 field->ltype_match = NPC_LT_LC_IP6_EXT; 4265 field->ltype_mask = 0xF; 4266 break; 4267 case NIX_FLOW_KEY_TYPE_GTPU: 4268 field->lid = NPC_LID_LE; 4269 field->hdr_offset = 4; 4270 field->bytesm1 = 3; /* 4 bytes TID*/ 4271 field->ltype_match = NPC_LT_LE_GTPU; 4272 field->ltype_mask = 0xF; 4273 break; 4274 case NIX_FLOW_KEY_TYPE_CUSTOM0: 4275 field->lid = NPC_LID_LC; 4276 field->hdr_offset = 6; 4277 field->bytesm1 = 1; /* 2 Bytes*/ 4278 field->ltype_match = NPC_LT_LC_CUSTOM0; 4279 field->ltype_mask = 0xF; 4280 break; 4281 case NIX_FLOW_KEY_TYPE_VLAN: 4282 field->lid = NPC_LID_LB; 4283 field->hdr_offset = 2; /* Skip TPID (2-bytes) */ 4284 field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */ 4285 field->ltype_match = NPC_LT_LB_CTAG; 4286 field->ltype_mask = 0xF; 4287 field->fn_mask = 1; /* Mask out the first nibble */ 4288 break; 4289 case NIX_FLOW_KEY_TYPE_AH: 4290 case NIX_FLOW_KEY_TYPE_ESP: 4291 field->hdr_offset = 0; 4292 field->bytesm1 = 7; /* SPI + sequence number */ 4293 field->ltype_mask = 0xF; 4294 field->lid = NPC_LID_LE; 4295 field->ltype_match = NPC_LT_LE_ESP; 4296 if (key_type == NIX_FLOW_KEY_TYPE_AH) { 4297 field->lid = NPC_LID_LD; 4298 field->ltype_match = NPC_LT_LD_AH; 4299 field->hdr_offset = 4; 4300 keyoff_marker = false; 4301 } 4302 break; 4303 } 4304 field->ena = 1; 4305 4306 /* Found a valid flow key type */ 4307 if (valid_key) { 4308 /* Use the key offset of TCP/UDP/SCTP fields 4309 * for ESP/AH fields. 4310 */ 4311 if (key_type == NIX_FLOW_KEY_TYPE_ESP || 4312 key_type == NIX_FLOW_KEY_TYPE_AH) 4313 key_off = l4_key_offset; 4314 field->key_offset = key_off; 4315 memcpy(&alg[nr_field], field, sizeof(*field)); 4316 max_key_off = max(max_key_off, field->bytesm1 + 1); 4317 4318 /* Found a field marker, get the next field */ 4319 if (field_marker) 4320 nr_field++; 4321 } 4322 4323 /* Found a keyoff marker, update the new key_off */ 4324 if (keyoff_marker) { 4325 key_off += max_key_off; 4326 max_key_off = 0; 4327 } 4328 } 4329 /* Processed all the flow key types */ 4330 if (idx == max_bit_pos && key_off <= MAX_KEY_OFF) 4331 return 0; 4332 else 4333 return NIX_AF_ERR_RSS_NOSPC_FIELD; 4334 } 4335 4336 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg) 4337 { 4338 u64 field[FIELDS_PER_ALG]; 4339 struct nix_hw *hw; 4340 int fid, rc; 4341 4342 hw = get_nix_hw(rvu->hw, blkaddr); 4343 if (!hw) 4344 return NIX_AF_ERR_INVALID_NIXBLK; 4345 4346 /* No room to add new flow hash algoritham */ 4347 if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX) 4348 return NIX_AF_ERR_RSS_NOSPC_ALGO; 4349 4350 /* Generate algo fields for the given flow_cfg */ 4351 rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg); 4352 if (rc) 4353 return rc; 4354 4355 /* Update ALGX_FIELDX register with generated fields */ 4356 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 4357 rvu_write64(rvu, blkaddr, 4358 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use, 4359 fid), field[fid]); 4360 4361 /* Store the flow_cfg for futher lookup */ 4362 rc = hw->flowkey.in_use; 4363 hw->flowkey.flowkey[rc] = flow_cfg; 4364 hw->flowkey.in_use++; 4365 4366 return rc; 4367 } 4368 4369 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu, 4370 struct nix_rss_flowkey_cfg *req, 4371 struct nix_rss_flowkey_cfg_rsp *rsp) 4372 { 4373 u16 pcifunc = req->hdr.pcifunc; 4374 int alg_idx, nixlf, blkaddr; 4375 struct nix_hw *nix_hw; 4376 int err; 4377 4378 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4379 if (err) 4380 return err; 4381 4382 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4383 if (!nix_hw) 4384 return NIX_AF_ERR_INVALID_NIXBLK; 4385 4386 alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg); 4387 /* Failed to get algo index from the exiting list, reserve new */ 4388 if (alg_idx < 0) { 4389 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr, 4390 req->flowkey_cfg); 4391 if (alg_idx < 0) 4392 return alg_idx; 4393 } 4394 rsp->alg_idx = alg_idx; 4395 rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group, 4396 alg_idx, req->mcam_index); 4397 return 0; 4398 } 4399 4400 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr) 4401 { 4402 u32 flowkey_cfg, minkey_cfg; 4403 int alg, fid, rc; 4404 4405 /* Disable all flow key algx fieldx */ 4406 for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) { 4407 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 4408 rvu_write64(rvu, blkaddr, 4409 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid), 4410 0); 4411 } 4412 4413 /* IPv4/IPv6 SIP/DIPs */ 4414 flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6; 4415 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4416 if (rc < 0) 4417 return rc; 4418 4419 /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4420 minkey_cfg = flowkey_cfg; 4421 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP; 4422 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4423 if (rc < 0) 4424 return rc; 4425 4426 /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4427 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP; 4428 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4429 if (rc < 0) 4430 return rc; 4431 4432 /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4433 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP; 4434 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4435 if (rc < 0) 4436 return rc; 4437 4438 /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4439 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4440 NIX_FLOW_KEY_TYPE_UDP; 4441 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4442 if (rc < 0) 4443 return rc; 4444 4445 /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4446 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4447 NIX_FLOW_KEY_TYPE_SCTP; 4448 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4449 if (rc < 0) 4450 return rc; 4451 4452 /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4453 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP | 4454 NIX_FLOW_KEY_TYPE_SCTP; 4455 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4456 if (rc < 0) 4457 return rc; 4458 4459 /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4460 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4461 NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP; 4462 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4463 if (rc < 0) 4464 return rc; 4465 4466 return 0; 4467 } 4468 4469 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu, 4470 struct nix_set_mac_addr *req, 4471 struct msg_rsp *rsp) 4472 { 4473 bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK; 4474 u16 pcifunc = req->hdr.pcifunc; 4475 int blkaddr, nixlf, err; 4476 struct rvu_pfvf *pfvf; 4477 4478 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4479 if (err) 4480 return err; 4481 4482 pfvf = rvu_get_pfvf(rvu, pcifunc); 4483 4484 /* untrusted VF can't overwrite admin(PF) changes */ 4485 if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 4486 (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) { 4487 dev_warn(rvu->dev, 4488 "MAC address set by admin(PF) cannot be overwritten by untrusted VF"); 4489 return -EPERM; 4490 } 4491 4492 ether_addr_copy(pfvf->mac_addr, req->mac_addr); 4493 4494 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 4495 pfvf->rx_chan_base, req->mac_addr); 4496 4497 if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf) 4498 ether_addr_copy(pfvf->default_mac, req->mac_addr); 4499 4500 return 0; 4501 } 4502 4503 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu, 4504 struct msg_req *req, 4505 struct nix_get_mac_addr_rsp *rsp) 4506 { 4507 u16 pcifunc = req->hdr.pcifunc; 4508 struct rvu_pfvf *pfvf; 4509 4510 if (!is_nixlf_attached(rvu, pcifunc)) 4511 return NIX_AF_ERR_AF_LF_INVALID; 4512 4513 pfvf = rvu_get_pfvf(rvu, pcifunc); 4514 4515 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 4516 4517 return 0; 4518 } 4519 4520 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, 4521 struct msg_rsp *rsp) 4522 { 4523 bool allmulti, promisc, nix_rx_multicast; 4524 u16 pcifunc = req->hdr.pcifunc; 4525 struct rvu_pfvf *pfvf; 4526 int nixlf, err; 4527 4528 pfvf = rvu_get_pfvf(rvu, pcifunc); 4529 promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false; 4530 allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false; 4531 pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false; 4532 4533 nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list; 4534 4535 if (is_vf(pcifunc) && !nix_rx_multicast && 4536 (promisc || allmulti)) { 4537 dev_warn_ratelimited(rvu->dev, 4538 "VF promisc/multicast not supported\n"); 4539 return 0; 4540 } 4541 4542 /* untrusted VF can't configure promisc/allmulti */ 4543 if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 4544 (promisc || allmulti)) 4545 return 0; 4546 4547 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4548 if (err) 4549 return err; 4550 4551 if (nix_rx_multicast) { 4552 /* add/del this PF_FUNC to/from mcast pkt replication list */ 4553 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY, 4554 allmulti); 4555 if (err) { 4556 dev_err(rvu->dev, 4557 "Failed to update pcifunc 0x%x to multicast list\n", 4558 pcifunc); 4559 return err; 4560 } 4561 4562 /* add/del this PF_FUNC to/from promisc pkt replication list */ 4563 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY, 4564 promisc); 4565 if (err) { 4566 dev_err(rvu->dev, 4567 "Failed to update pcifunc 0x%x to promisc list\n", 4568 pcifunc); 4569 return err; 4570 } 4571 } 4572 4573 /* install/uninstall allmulti entry */ 4574 if (allmulti) { 4575 rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf, 4576 pfvf->rx_chan_base); 4577 } else { 4578 if (!nix_rx_multicast) 4579 rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false); 4580 } 4581 4582 /* install/uninstall promisc entry */ 4583 if (promisc) 4584 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 4585 pfvf->rx_chan_base, 4586 pfvf->rx_chan_cnt); 4587 else 4588 if (!nix_rx_multicast) 4589 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); 4590 4591 return 0; 4592 } 4593 4594 static void nix_find_link_frs(struct rvu *rvu, 4595 struct nix_frs_cfg *req, u16 pcifunc) 4596 { 4597 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4598 struct rvu_pfvf *pfvf; 4599 int maxlen, minlen; 4600 int numvfs, hwvf; 4601 int vf; 4602 4603 /* Update with requester's min/max lengths */ 4604 pfvf = rvu_get_pfvf(rvu, pcifunc); 4605 pfvf->maxlen = req->maxlen; 4606 if (req->update_minlen) 4607 pfvf->minlen = req->minlen; 4608 4609 maxlen = req->maxlen; 4610 minlen = req->update_minlen ? req->minlen : 0; 4611 4612 /* Get this PF's numVFs and starting hwvf */ 4613 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf); 4614 4615 /* For each VF, compare requested max/minlen */ 4616 for (vf = 0; vf < numvfs; vf++) { 4617 pfvf = &rvu->hwvf[hwvf + vf]; 4618 if (pfvf->maxlen > maxlen) 4619 maxlen = pfvf->maxlen; 4620 if (req->update_minlen && 4621 pfvf->minlen && pfvf->minlen < minlen) 4622 minlen = pfvf->minlen; 4623 } 4624 4625 /* Compare requested max/minlen with PF's max/minlen */ 4626 pfvf = &rvu->pf[pf]; 4627 if (pfvf->maxlen > maxlen) 4628 maxlen = pfvf->maxlen; 4629 if (req->update_minlen && 4630 pfvf->minlen && pfvf->minlen < minlen) 4631 minlen = pfvf->minlen; 4632 4633 /* Update the request with max/min PF's and it's VF's max/min */ 4634 req->maxlen = maxlen; 4635 if (req->update_minlen) 4636 req->minlen = minlen; 4637 } 4638 4639 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req, 4640 struct msg_rsp *rsp) 4641 { 4642 struct rvu_hwinfo *hw = rvu->hw; 4643 u16 pcifunc = req->hdr.pcifunc; 4644 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4645 int blkaddr, link = -1; 4646 struct nix_hw *nix_hw; 4647 struct rvu_pfvf *pfvf; 4648 u8 cgx = 0, lmac = 0; 4649 u16 max_mtu; 4650 u64 cfg; 4651 4652 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 4653 if (blkaddr < 0) 4654 return NIX_AF_ERR_AF_LF_INVALID; 4655 4656 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4657 if (!nix_hw) 4658 return NIX_AF_ERR_INVALID_NIXBLK; 4659 4660 if (is_lbk_vf(rvu, pcifunc) || is_rep_dev(rvu, pcifunc)) 4661 rvu_get_lbk_link_max_frs(rvu, &max_mtu); 4662 else 4663 rvu_get_lmac_link_max_frs(rvu, &max_mtu); 4664 4665 if (!req->sdp_link && req->maxlen > max_mtu) 4666 return NIX_AF_ERR_FRS_INVALID; 4667 4668 if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS) 4669 return NIX_AF_ERR_FRS_INVALID; 4670 4671 /* Check if config is for SDP link */ 4672 if (req->sdp_link) { 4673 if (!hw->sdp_links) 4674 return NIX_AF_ERR_RX_LINK_INVALID; 4675 link = hw->cgx_links + hw->lbk_links; 4676 goto linkcfg; 4677 } 4678 4679 /* Check if the request is from CGX mapped RVU PF */ 4680 if (is_pf_cgxmapped(rvu, pf)) { 4681 /* Get CGX and LMAC to which this PF is mapped and find link */ 4682 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac); 4683 link = (cgx * hw->lmac_per_cgx) + lmac; 4684 } else if (pf == 0) { 4685 /* For VFs of PF0 ingress is LBK port, so config LBK link */ 4686 pfvf = rvu_get_pfvf(rvu, pcifunc); 4687 link = hw->cgx_links + pfvf->lbkid; 4688 } else if (is_rep_dev(rvu, pcifunc)) { 4689 link = hw->cgx_links + 0; 4690 } 4691 4692 if (link < 0) 4693 return NIX_AF_ERR_RX_LINK_INVALID; 4694 4695 linkcfg: 4696 nix_find_link_frs(rvu, req, pcifunc); 4697 4698 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link)); 4699 cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16); 4700 if (req->update_minlen) 4701 cfg = (cfg & ~0xFFFFULL) | req->minlen; 4702 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg); 4703 4704 return 0; 4705 } 4706 4707 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req, 4708 struct msg_rsp *rsp) 4709 { 4710 int nixlf, blkaddr, err; 4711 u64 cfg; 4712 4713 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr); 4714 if (err) 4715 return err; 4716 4717 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf)); 4718 /* Set the interface configuration */ 4719 if (req->len_verify & BIT(0)) 4720 cfg |= BIT_ULL(41); 4721 else 4722 cfg &= ~BIT_ULL(41); 4723 4724 if (req->len_verify & BIT(1)) 4725 cfg |= BIT_ULL(40); 4726 else 4727 cfg &= ~BIT_ULL(40); 4728 4729 if (req->len_verify & NIX_RX_DROP_RE) 4730 cfg |= BIT_ULL(32); 4731 else 4732 cfg &= ~BIT_ULL(32); 4733 4734 if (req->csum_verify & BIT(0)) 4735 cfg |= BIT_ULL(37); 4736 else 4737 cfg &= ~BIT_ULL(37); 4738 4739 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg); 4740 4741 return 0; 4742 } 4743 4744 static u64 rvu_get_lbk_link_credits(struct rvu *rvu, u16 lbk_max_frs) 4745 { 4746 return 1600; /* 16 * max LBK datarate = 16 * 100Gbps */ 4747 } 4748 4749 static void nix_link_config(struct rvu *rvu, int blkaddr, 4750 struct nix_hw *nix_hw) 4751 { 4752 struct rvu_hwinfo *hw = rvu->hw; 4753 int cgx, lmac_cnt, slink, link; 4754 u16 lbk_max_frs, lmac_max_frs; 4755 unsigned long lmac_bmap; 4756 u64 tx_credits, cfg; 4757 u64 lmac_fifo_len; 4758 int iter; 4759 4760 rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs); 4761 rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs); 4762 4763 /* Set SDP link credit */ 4764 rvu_write64(rvu, blkaddr, NIX_AF_SDP_LINK_CREDIT, SDP_LINK_CREDIT); 4765 4766 /* Set default min/max packet lengths allowed on NIX Rx links. 4767 * 4768 * With HW reset minlen value of 60byte, HW will treat ARP pkts 4769 * as undersize and report them to SW as error pkts, hence 4770 * setting it to 40 bytes. 4771 */ 4772 for (link = 0; link < hw->cgx_links; link++) { 4773 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4774 ((u64)lmac_max_frs << 16) | NIC_HW_MIN_FRS); 4775 } 4776 4777 for (link = hw->cgx_links; link < hw->lbk_links; link++) { 4778 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4779 ((u64)lbk_max_frs << 16) | NIC_HW_MIN_FRS); 4780 } 4781 if (hw->sdp_links) { 4782 link = hw->cgx_links + hw->lbk_links; 4783 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4784 SDP_HW_MAX_FRS << 16 | SDP_HW_MIN_FRS); 4785 } 4786 4787 /* Get MCS external bypass status for CN10K-B */ 4788 if (mcs_get_blkcnt() == 1) { 4789 /* Adjust for 2 credits when external bypass is disabled */ 4790 nix_hw->cc_mcs_cnt = is_mcs_bypass(0) ? 0 : 2; 4791 } 4792 4793 /* Set credits for Tx links assuming max packet length allowed. 4794 * This will be reconfigured based on MTU set for PF/VF. 4795 */ 4796 for (cgx = 0; cgx < hw->cgx; cgx++) { 4797 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu)); 4798 /* Skip when cgx is not available or lmac cnt is zero */ 4799 if (lmac_cnt <= 0) 4800 continue; 4801 slink = cgx * hw->lmac_per_cgx; 4802 4803 /* Get LMAC id's from bitmap */ 4804 lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); 4805 for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) { 4806 lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); 4807 if (!lmac_fifo_len) { 4808 dev_err(rvu->dev, 4809 "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", 4810 __func__, cgx, iter); 4811 continue; 4812 } 4813 tx_credits = (lmac_fifo_len - lmac_max_frs) / 16; 4814 /* Enable credits and set credit pkt count to max allowed */ 4815 cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4816 cfg |= FIELD_PREP(NIX_AF_LINKX_MCS_CNT_MASK, nix_hw->cc_mcs_cnt); 4817 4818 link = iter + slink; 4819 nix_hw->tx_credits[link] = tx_credits; 4820 rvu_write64(rvu, blkaddr, 4821 NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); 4822 } 4823 } 4824 4825 /* Set Tx credits for LBK link */ 4826 slink = hw->cgx_links; 4827 for (link = slink; link < (slink + hw->lbk_links); link++) { 4828 tx_credits = rvu_get_lbk_link_credits(rvu, lbk_max_frs); 4829 nix_hw->tx_credits[link] = tx_credits; 4830 /* Enable credits and set credit pkt count to max allowed */ 4831 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4832 rvu_write64(rvu, blkaddr, 4833 NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits); 4834 } 4835 } 4836 4837 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr) 4838 { 4839 int idx, err; 4840 u64 status; 4841 4842 /* Start X2P bus calibration */ 4843 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4844 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9)); 4845 /* Wait for calibration to complete */ 4846 err = rvu_poll_reg(rvu, blkaddr, 4847 NIX_AF_STATUS, BIT_ULL(10), false); 4848 if (err) { 4849 dev_err(rvu->dev, "NIX X2P bus calibration failed\n"); 4850 return err; 4851 } 4852 4853 status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS); 4854 /* Check if CGX devices are ready */ 4855 for (idx = 0; idx < rvu->cgx_cnt_max; idx++) { 4856 /* Skip when cgx port is not available */ 4857 if (!rvu_cgx_pdata(idx, rvu) || 4858 (status & (BIT_ULL(16 + idx)))) 4859 continue; 4860 dev_err(rvu->dev, 4861 "CGX%d didn't respond to NIX X2P calibration\n", idx); 4862 err = -EBUSY; 4863 } 4864 4865 /* Check if LBK is ready */ 4866 if (!(status & BIT_ULL(19))) { 4867 dev_err(rvu->dev, 4868 "LBK didn't respond to NIX X2P calibration\n"); 4869 err = -EBUSY; 4870 } 4871 4872 /* Clear 'calibrate_x2p' bit */ 4873 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4874 rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9)); 4875 if (err || (status & 0x3FFULL)) 4876 dev_err(rvu->dev, 4877 "NIX X2P calibration failed, status 0x%llx\n", status); 4878 if (err) 4879 return err; 4880 return 0; 4881 } 4882 4883 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block) 4884 { 4885 u64 cfg; 4886 int err; 4887 4888 /* Set admin queue endianness */ 4889 cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG); 4890 #ifdef __BIG_ENDIAN 4891 cfg |= BIT_ULL(8); 4892 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4893 #else 4894 cfg &= ~BIT_ULL(8); 4895 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4896 #endif 4897 4898 /* Do not bypass NDC cache */ 4899 cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG); 4900 cfg &= ~0x3FFEULL; 4901 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 4902 /* Disable caching of SQB aka SQEs */ 4903 cfg |= 0x04ULL; 4904 #endif 4905 rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg); 4906 4907 /* Result structure can be followed by RQ/SQ/CQ context at 4908 * RES + 128bytes and a write mask at RES + 256 bytes, depending on 4909 * operation type. Alloc sufficient result memory for all operations. 4910 */ 4911 err = rvu_aq_alloc(rvu, &block->aq, 4912 Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s), 4913 ALIGN(sizeof(struct nix_aq_res_s), 128) + 256); 4914 if (err) 4915 return err; 4916 4917 rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE); 4918 rvu_write64(rvu, block->addr, 4919 NIX_AF_AQ_BASE, (u64)block->aq->inst->iova); 4920 return 0; 4921 } 4922 4923 static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr) 4924 { 4925 struct rvu_hwinfo *hw = rvu->hw; 4926 u64 hw_const; 4927 4928 hw_const = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 4929 4930 /* On OcteonTx2 DWRR quantum is directly configured into each of 4931 * the transmit scheduler queues. And PF/VF drivers were free to 4932 * config any value upto 2^24. 4933 * On CN10K, HW is modified, the quantum configuration at scheduler 4934 * queues is in terms of weight. And SW needs to setup a base DWRR MTU 4935 * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do 4936 * 'DWRR MTU * weight' to get the quantum. 4937 * 4938 * Check if HW uses a common MTU for all DWRR quantum configs. 4939 * On OcteonTx2 this register field is '0'. 4940 */ 4941 if ((((hw_const >> 56) & 0x10) == 0x10) && !(hw_const & BIT_ULL(61))) 4942 hw->cap.nix_common_dwrr_mtu = true; 4943 4944 if (hw_const & BIT_ULL(61)) 4945 hw->cap.nix_multiple_dwrr_mtu = true; 4946 } 4947 4948 static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) 4949 { 4950 const struct npc_lt_def_cfg *ltdefs; 4951 struct rvu_hwinfo *hw = rvu->hw; 4952 int blkaddr = nix_hw->blkaddr; 4953 struct rvu_block *block; 4954 int err; 4955 u64 cfg; 4956 4957 block = &hw->block[blkaddr]; 4958 4959 if (is_rvu_96xx_B0(rvu)) { 4960 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt 4961 * internal state when conditional clocks are turned off. 4962 * Hence enable them. 4963 */ 4964 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4965 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL); 4966 } 4967 4968 /* Set chan/link to backpressure TL3 instead of TL2 */ 4969 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); 4970 4971 /* Disable SQ manager's sticky mode operation (set TM6 = 0, TM11 = 0) 4972 * This sticky mode is known to cause SQ stalls when multiple 4973 * SQs are mapped to same SMQ and transmitting pkts simultaneously. 4974 * NIX PSE may deadlock when there are any sticky to non-sticky 4975 * transmission. Hence disable it (TM5 = 0). 4976 */ 4977 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); 4978 cfg &= ~(BIT_ULL(15) | BIT_ULL(14) | BIT_ULL(23)); 4979 /* NIX may drop credits when condition clocks are turned off. 4980 * Hence enable control flow clk (set TM9 = 1). 4981 */ 4982 cfg |= BIT_ULL(21); 4983 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); 4984 4985 ltdefs = rvu->kpu.lt_def; 4986 /* Calibrate X2P bus to check if CGX/LBK links are fine */ 4987 err = nix_calibrate_x2p(rvu, blkaddr); 4988 if (err) 4989 return err; 4990 4991 /* Setup capabilities of the NIX block */ 4992 rvu_nix_setup_capabilities(rvu, blkaddr); 4993 4994 /* Initialize admin queue */ 4995 err = nix_aq_init(rvu, block); 4996 if (err) 4997 return err; 4998 4999 /* Restore CINT timer delay to HW reset values */ 5000 rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); 5001 5002 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); 5003 5004 /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ 5005 cfg |= 1ULL; 5006 if (!is_rvu_otx2(rvu)) 5007 cfg |= NIX_PTP_1STEP_EN; 5008 5009 rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); 5010 5011 if (!is_rvu_otx2(rvu)) 5012 rvu_nix_block_cn10k_init(rvu, nix_hw); 5013 5014 if (is_block_implemented(hw, blkaddr)) { 5015 err = nix_setup_txschq(rvu, nix_hw, blkaddr); 5016 if (err) 5017 return err; 5018 5019 err = nix_setup_ipolicers(rvu, nix_hw, blkaddr); 5020 if (err) 5021 return err; 5022 5023 err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr); 5024 if (err) 5025 return err; 5026 5027 err = nix_setup_mcast(rvu, nix_hw, blkaddr); 5028 if (err) 5029 return err; 5030 5031 err = nix_setup_txvlan(rvu, nix_hw); 5032 if (err) 5033 return err; 5034 5035 err = nix_setup_bpids(rvu, nix_hw, blkaddr); 5036 if (err) 5037 return err; 5038 5039 /* Configure segmentation offload formats */ 5040 nix_setup_lso(rvu, nix_hw, blkaddr); 5041 5042 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info. 5043 * This helps HW protocol checker to identify headers 5044 * and validate length and checksums. 5045 */ 5046 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2, 5047 (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) | 5048 ltdefs->rx_ol2.ltype_mask); 5049 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4, 5050 (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) | 5051 ltdefs->rx_oip4.ltype_mask); 5052 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4, 5053 (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) | 5054 ltdefs->rx_iip4.ltype_mask); 5055 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6, 5056 (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) | 5057 ltdefs->rx_oip6.ltype_mask); 5058 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6, 5059 (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) | 5060 ltdefs->rx_iip6.ltype_mask); 5061 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP, 5062 (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) | 5063 ltdefs->rx_otcp.ltype_mask); 5064 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP, 5065 (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) | 5066 ltdefs->rx_itcp.ltype_mask); 5067 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP, 5068 (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) | 5069 ltdefs->rx_oudp.ltype_mask); 5070 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP, 5071 (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) | 5072 ltdefs->rx_iudp.ltype_mask); 5073 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP, 5074 (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) | 5075 ltdefs->rx_osctp.ltype_mask); 5076 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP, 5077 (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) | 5078 ltdefs->rx_isctp.ltype_mask); 5079 5080 if (!is_rvu_otx2(rvu)) { 5081 /* Enable APAD calculation for other protocols 5082 * matching APAD0 and APAD1 lt def registers. 5083 */ 5084 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0, 5085 (ltdefs->rx_apad0.valid << 11) | 5086 (ltdefs->rx_apad0.lid << 8) | 5087 (ltdefs->rx_apad0.ltype_match << 4) | 5088 ltdefs->rx_apad0.ltype_mask); 5089 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1, 5090 (ltdefs->rx_apad1.valid << 11) | 5091 (ltdefs->rx_apad1.lid << 8) | 5092 (ltdefs->rx_apad1.ltype_match << 4) | 5093 ltdefs->rx_apad1.ltype_mask); 5094 5095 /* Receive ethertype definition register defines layer 5096 * information in NPC_RESULT_S to identify the Ethertype 5097 * location in L2 header. Used for Ethertype overwriting 5098 * in inline IPsec flow. 5099 */ 5100 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0), 5101 (ltdefs->rx_et[0].offset << 12) | 5102 (ltdefs->rx_et[0].valid << 11) | 5103 (ltdefs->rx_et[0].lid << 8) | 5104 (ltdefs->rx_et[0].ltype_match << 4) | 5105 ltdefs->rx_et[0].ltype_mask); 5106 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1), 5107 (ltdefs->rx_et[1].offset << 12) | 5108 (ltdefs->rx_et[1].valid << 11) | 5109 (ltdefs->rx_et[1].lid << 8) | 5110 (ltdefs->rx_et[1].ltype_match << 4) | 5111 ltdefs->rx_et[1].ltype_mask); 5112 } 5113 5114 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr); 5115 if (err) 5116 return err; 5117 5118 nix_hw->tx_credits = kcalloc(hw->cgx_links + hw->lbk_links, 5119 sizeof(u64), GFP_KERNEL); 5120 if (!nix_hw->tx_credits) 5121 return -ENOMEM; 5122 5123 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */ 5124 nix_link_config(rvu, blkaddr, nix_hw); 5125 5126 /* Enable Channel backpressure */ 5127 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0)); 5128 } 5129 return 0; 5130 } 5131 5132 int rvu_nix_init(struct rvu *rvu) 5133 { 5134 struct rvu_hwinfo *hw = rvu->hw; 5135 struct nix_hw *nix_hw; 5136 int blkaddr = 0, err; 5137 int i = 0; 5138 5139 hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw), 5140 GFP_KERNEL); 5141 if (!hw->nix) 5142 return -ENOMEM; 5143 5144 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5145 while (blkaddr) { 5146 nix_hw = &hw->nix[i]; 5147 nix_hw->rvu = rvu; 5148 nix_hw->blkaddr = blkaddr; 5149 err = rvu_nix_block_init(rvu, nix_hw); 5150 if (err) 5151 return err; 5152 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5153 i++; 5154 } 5155 5156 return 0; 5157 } 5158 5159 static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr, 5160 struct rvu_block *block) 5161 { 5162 struct nix_txsch *txsch; 5163 struct nix_mcast *mcast; 5164 struct nix_txvlan *vlan; 5165 struct nix_hw *nix_hw; 5166 int lvl; 5167 5168 rvu_aq_free(rvu, block->aq); 5169 5170 if (is_block_implemented(rvu->hw, blkaddr)) { 5171 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5172 if (!nix_hw) 5173 return; 5174 5175 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 5176 txsch = &nix_hw->txsch[lvl]; 5177 kfree(txsch->schq.bmap); 5178 } 5179 5180 kfree(nix_hw->tx_credits); 5181 5182 nix_ipolicer_freemem(rvu, nix_hw); 5183 5184 vlan = &nix_hw->txvlan; 5185 kfree(vlan->rsrc.bmap); 5186 mutex_destroy(&vlan->rsrc_lock); 5187 5188 mcast = &nix_hw->mcast; 5189 qmem_free(rvu->dev, mcast->mce_ctx); 5190 qmem_free(rvu->dev, mcast->mcast_buf); 5191 mutex_destroy(&mcast->mce_lock); 5192 } 5193 } 5194 5195 void rvu_nix_freemem(struct rvu *rvu) 5196 { 5197 struct rvu_hwinfo *hw = rvu->hw; 5198 struct rvu_block *block; 5199 int blkaddr = 0; 5200 5201 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5202 while (blkaddr) { 5203 block = &hw->block[blkaddr]; 5204 rvu_nix_block_freemem(rvu, blkaddr, block); 5205 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5206 } 5207 } 5208 5209 static void nix_mcast_update_action(struct rvu *rvu, 5210 struct nix_mcast_grp_elem *elem) 5211 { 5212 struct npc_mcam *mcam = &rvu->hw->mcam; 5213 struct nix_rx_action rx_action = { 0 }; 5214 struct nix_tx_action tx_action = { 0 }; 5215 int npc_blkaddr; 5216 5217 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 5218 if (elem->dir == NIX_MCAST_INGRESS) { 5219 *(u64 *)&rx_action = npc_get_mcam_action(rvu, mcam, 5220 npc_blkaddr, 5221 elem->mcam_index); 5222 rx_action.index = elem->mce_start_index; 5223 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index, 5224 *(u64 *)&rx_action); 5225 } else { 5226 *(u64 *)&tx_action = npc_get_mcam_action(rvu, mcam, 5227 npc_blkaddr, 5228 elem->mcam_index); 5229 tx_action.index = elem->mce_start_index; 5230 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index, 5231 *(u64 *)&tx_action); 5232 } 5233 } 5234 5235 static void nix_mcast_update_mce_entry(struct rvu *rvu, u16 pcifunc, u8 is_active) 5236 { 5237 struct nix_mcast_grp_elem *elem; 5238 struct nix_mcast_grp *mcast_grp; 5239 struct nix_hw *nix_hw; 5240 int blkaddr; 5241 5242 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5243 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5244 if (!nix_hw) 5245 return; 5246 5247 mcast_grp = &nix_hw->mcast_grp; 5248 5249 mutex_lock(&mcast_grp->mcast_grp_lock); 5250 list_for_each_entry(elem, &mcast_grp->mcast_grp_head, list) { 5251 struct nix_mce_list *mce_list; 5252 struct mce *mce; 5253 5254 /* Iterate the group elements and disable the element which 5255 * received the disable request. 5256 */ 5257 mce_list = &elem->mcast_mce_list; 5258 hlist_for_each_entry(mce, &mce_list->head, node) { 5259 if (mce->pcifunc == pcifunc) { 5260 mce->is_active = is_active; 5261 break; 5262 } 5263 } 5264 5265 /* Dump the updated list to HW */ 5266 if (elem->dir == NIX_MCAST_INGRESS) 5267 nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 5268 else 5269 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 5270 5271 /* Update the multicast index in NPC rule */ 5272 nix_mcast_update_action(rvu, elem); 5273 } 5274 mutex_unlock(&mcast_grp->mcast_grp_lock); 5275 } 5276 5277 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req, 5278 struct msg_rsp *rsp) 5279 { 5280 u16 pcifunc = req->hdr.pcifunc; 5281 struct rvu_pfvf *pfvf; 5282 int nixlf, err, pf; 5283 5284 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 5285 if (err) 5286 return err; 5287 5288 /* Enable the interface if it is in any multicast list */ 5289 nix_mcast_update_mce_entry(rvu, pcifunc, 1); 5290 5291 rvu_npc_enable_default_entries(rvu, pcifunc, nixlf); 5292 5293 npc_mcam_enable_flows(rvu, pcifunc); 5294 5295 pfvf = rvu_get_pfvf(rvu, pcifunc); 5296 set_bit(NIXLF_INITIALIZED, &pfvf->flags); 5297 5298 rvu_switch_update_rules(rvu, pcifunc, true); 5299 5300 pf = rvu_get_pf(rvu->pdev, pcifunc); 5301 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5302 rvu_rep_notify_pfvf_state(rvu, pcifunc, true); 5303 5304 return rvu_cgx_start_stop_io(rvu, pcifunc, true); 5305 } 5306 5307 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, 5308 struct msg_rsp *rsp) 5309 { 5310 u16 pcifunc = req->hdr.pcifunc; 5311 struct rvu_pfvf *pfvf; 5312 int nixlf, err, pf; 5313 5314 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 5315 if (err) 5316 return err; 5317 5318 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 5319 /* Disable the interface if it is in any multicast list */ 5320 nix_mcast_update_mce_entry(rvu, pcifunc, 0); 5321 5322 pfvf = rvu_get_pfvf(rvu, pcifunc); 5323 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 5324 5325 err = rvu_cgx_start_stop_io(rvu, pcifunc, false); 5326 if (err) 5327 return err; 5328 5329 rvu_switch_update_rules(rvu, pcifunc, false); 5330 rvu_cgx_tx_enable(rvu, pcifunc, true); 5331 5332 pf = rvu_get_pf(rvu->pdev, pcifunc); 5333 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5334 rvu_rep_notify_pfvf_state(rvu, pcifunc, false); 5335 return 0; 5336 } 5337 5338 #define RX_SA_BASE GENMASK_ULL(52, 7) 5339 5340 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) 5341 { 5342 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 5343 struct hwctx_disable_req ctx_req; 5344 int pf = rvu_get_pf(rvu->pdev, pcifunc); 5345 struct mac_ops *mac_ops; 5346 u8 cgx_id, lmac_id; 5347 u64 sa_base; 5348 void *cgxd; 5349 int err; 5350 5351 ctx_req.hdr.pcifunc = pcifunc; 5352 5353 /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */ 5354 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 5355 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 5356 nix_interface_deinit(rvu, pcifunc, nixlf); 5357 nix_rx_sync(rvu, blkaddr); 5358 nix_txschq_free(rvu, pcifunc); 5359 5360 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 5361 5362 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5363 rvu_rep_notify_pfvf_state(rvu, pcifunc, false); 5364 5365 rvu_cgx_start_stop_io(rvu, pcifunc, false); 5366 5367 if (pfvf->sq_ctx) { 5368 ctx_req.ctype = NIX_AQ_CTYPE_SQ; 5369 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5370 if (err) 5371 dev_err(rvu->dev, "SQ ctx disable failed\n"); 5372 } 5373 5374 if (pfvf->rq_ctx) { 5375 ctx_req.ctype = NIX_AQ_CTYPE_RQ; 5376 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5377 if (err) 5378 dev_err(rvu->dev, "RQ ctx disable failed\n"); 5379 } 5380 5381 if (pfvf->cq_ctx) { 5382 ctx_req.ctype = NIX_AQ_CTYPE_CQ; 5383 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5384 if (err) 5385 dev_err(rvu->dev, "CQ ctx disable failed\n"); 5386 } 5387 5388 /* reset HW config done for Switch headers */ 5389 rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT, 5390 (PKIND_TX | PKIND_RX), 0, 0, 0, 0); 5391 5392 /* Disabling CGX and NPC config done for PTP */ 5393 if (pfvf->hw_rx_tstamp_en) { 5394 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 5395 cgxd = rvu_cgx_pdata(cgx_id, rvu); 5396 mac_ops = get_mac_ops(cgxd); 5397 mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, false); 5398 /* Undo NPC config done for PTP */ 5399 if (npc_config_ts_kpuaction(rvu, pf, pcifunc, false)) 5400 dev_err(rvu->dev, "NPC config for PTP failed\n"); 5401 pfvf->hw_rx_tstamp_en = false; 5402 } 5403 5404 /* reset priority flow control config */ 5405 rvu_cgx_prio_flow_ctrl_cfg(rvu, pcifunc, 0, 0, 0); 5406 5407 /* reset 802.3x flow control config */ 5408 rvu_cgx_cfg_pause_frm(rvu, pcifunc, 0, 0); 5409 5410 nix_ctx_free(rvu, pfvf); 5411 5412 nix_free_all_bandprof(rvu, pcifunc); 5413 5414 sa_base = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(nixlf)); 5415 if (FIELD_GET(RX_SA_BASE, sa_base)) { 5416 err = rvu_cpt_ctx_flush(rvu, pcifunc); 5417 if (err) 5418 dev_err(rvu->dev, 5419 "CPT ctx flush failed with error: %d\n", err); 5420 } 5421 } 5422 5423 #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32) 5424 5425 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable) 5426 { 5427 struct rvu_hwinfo *hw = rvu->hw; 5428 struct rvu_block *block; 5429 int blkaddr, pf; 5430 int nixlf; 5431 u64 cfg; 5432 5433 pf = rvu_get_pf(rvu->pdev, pcifunc); 5434 if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_PTP)) 5435 return 0; 5436 5437 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5438 if (blkaddr < 0) 5439 return NIX_AF_ERR_AF_LF_INVALID; 5440 5441 block = &hw->block[blkaddr]; 5442 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 5443 if (nixlf < 0) 5444 return NIX_AF_ERR_AF_LF_INVALID; 5445 5446 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf)); 5447 5448 if (enable) 5449 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN; 5450 else 5451 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN; 5452 5453 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 5454 5455 return 0; 5456 } 5457 5458 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req, 5459 struct msg_rsp *rsp) 5460 { 5461 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true); 5462 } 5463 5464 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req, 5465 struct msg_rsp *rsp) 5466 { 5467 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false); 5468 } 5469 5470 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu, 5471 struct nix_lso_format_cfg *req, 5472 struct nix_lso_format_cfg_rsp *rsp) 5473 { 5474 u16 pcifunc = req->hdr.pcifunc; 5475 struct nix_hw *nix_hw; 5476 struct rvu_pfvf *pfvf; 5477 int blkaddr, idx, f; 5478 u64 reg; 5479 5480 pfvf = rvu_get_pfvf(rvu, pcifunc); 5481 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5482 if (!pfvf->nixlf || blkaddr < 0) 5483 return NIX_AF_ERR_AF_LF_INVALID; 5484 5485 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5486 if (!nix_hw) 5487 return NIX_AF_ERR_INVALID_NIXBLK; 5488 5489 /* Find existing matching LSO format, if any */ 5490 for (idx = 0; idx < nix_hw->lso.in_use; idx++) { 5491 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) { 5492 reg = rvu_read64(rvu, blkaddr, 5493 NIX_AF_LSO_FORMATX_FIELDX(idx, f)); 5494 if (req->fields[f] != (reg & req->field_mask)) 5495 break; 5496 } 5497 5498 if (f == NIX_LSO_FIELD_MAX) 5499 break; 5500 } 5501 5502 if (idx < nix_hw->lso.in_use) { 5503 /* Match found */ 5504 rsp->lso_format_idx = idx; 5505 return 0; 5506 } 5507 5508 if (nix_hw->lso.in_use == nix_hw->lso.total) 5509 return NIX_AF_ERR_LSO_CFG_FAIL; 5510 5511 rsp->lso_format_idx = nix_hw->lso.in_use++; 5512 5513 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) 5514 rvu_write64(rvu, blkaddr, 5515 NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f), 5516 req->fields[f]); 5517 5518 return 0; 5519 } 5520 5521 #define IPSEC_GEN_CFG_EGRP GENMASK_ULL(50, 48) 5522 #define IPSEC_GEN_CFG_OPCODE GENMASK_ULL(47, 32) 5523 #define IPSEC_GEN_CFG_PARAM1 GENMASK_ULL(31, 16) 5524 #define IPSEC_GEN_CFG_PARAM2 GENMASK_ULL(15, 0) 5525 5526 #define CPT_INST_QSEL_BLOCK GENMASK_ULL(28, 24) 5527 #define CPT_INST_QSEL_PF_FUNC GENMASK_ULL(23, 8) 5528 #define CPT_INST_QSEL_SLOT GENMASK_ULL(7, 0) 5529 5530 #define CPT_INST_CREDIT_TH GENMASK_ULL(53, 32) 5531 #define CPT_INST_CREDIT_BPID GENMASK_ULL(30, 22) 5532 #define CPT_INST_CREDIT_CNT GENMASK_ULL(21, 0) 5533 5534 static void nix_inline_ipsec_cfg(struct rvu *rvu, struct nix_inline_ipsec_cfg *req, 5535 int blkaddr) 5536 { 5537 u8 cpt_idx, cpt_blkaddr; 5538 u64 val; 5539 5540 cpt_idx = (blkaddr == BLKADDR_NIX0) ? 0 : 1; 5541 if (req->enable) { 5542 val = 0; 5543 /* Enable context prefetching */ 5544 if (!is_rvu_otx2(rvu)) 5545 val |= BIT_ULL(51); 5546 5547 /* Set OPCODE and EGRP */ 5548 val |= FIELD_PREP(IPSEC_GEN_CFG_EGRP, req->gen_cfg.egrp); 5549 val |= FIELD_PREP(IPSEC_GEN_CFG_OPCODE, req->gen_cfg.opcode); 5550 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM1, req->gen_cfg.param1); 5551 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM2, req->gen_cfg.param2); 5552 5553 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, val); 5554 5555 /* Set CPT queue for inline IPSec */ 5556 val = FIELD_PREP(CPT_INST_QSEL_SLOT, req->inst_qsel.cpt_slot); 5557 val |= FIELD_PREP(CPT_INST_QSEL_PF_FUNC, 5558 req->inst_qsel.cpt_pf_func); 5559 5560 if (!is_rvu_otx2(rvu)) { 5561 cpt_blkaddr = (cpt_idx == 0) ? BLKADDR_CPT0 : 5562 BLKADDR_CPT1; 5563 val |= FIELD_PREP(CPT_INST_QSEL_BLOCK, cpt_blkaddr); 5564 } 5565 5566 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 5567 val); 5568 5569 /* Set CPT credit */ 5570 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 5571 if ((val & 0x3FFFFF) != 0x3FFFFF) 5572 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 5573 0x3FFFFF - val); 5574 5575 val = FIELD_PREP(CPT_INST_CREDIT_CNT, req->cpt_credit); 5576 val |= FIELD_PREP(CPT_INST_CREDIT_BPID, req->bpid); 5577 val |= FIELD_PREP(CPT_INST_CREDIT_TH, req->credit_th); 5578 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), val); 5579 } else { 5580 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, 0x0); 5581 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 5582 0x0); 5583 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 5584 if ((val & 0x3FFFFF) != 0x3FFFFF) 5585 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 5586 0x3FFFFF - val); 5587 } 5588 } 5589 5590 int rvu_mbox_handler_nix_inline_ipsec_cfg(struct rvu *rvu, 5591 struct nix_inline_ipsec_cfg *req, 5592 struct msg_rsp *rsp) 5593 { 5594 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5595 return 0; 5596 5597 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX0); 5598 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 5599 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX1); 5600 5601 return 0; 5602 } 5603 5604 int rvu_mbox_handler_nix_read_inline_ipsec_cfg(struct rvu *rvu, 5605 struct msg_req *req, 5606 struct nix_inline_ipsec_cfg *rsp) 5607 5608 { 5609 u64 val; 5610 5611 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5612 return 0; 5613 5614 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_IPSEC_GEN_CFG); 5615 rsp->gen_cfg.egrp = FIELD_GET(IPSEC_GEN_CFG_EGRP, val); 5616 rsp->gen_cfg.opcode = FIELD_GET(IPSEC_GEN_CFG_OPCODE, val); 5617 rsp->gen_cfg.param1 = FIELD_GET(IPSEC_GEN_CFG_PARAM1, val); 5618 rsp->gen_cfg.param2 = FIELD_GET(IPSEC_GEN_CFG_PARAM2, val); 5619 5620 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_CPTX_CREDIT(0)); 5621 rsp->cpt_credit = FIELD_GET(CPT_INST_CREDIT_CNT, val); 5622 rsp->credit_th = FIELD_GET(CPT_INST_CREDIT_TH, val); 5623 rsp->bpid = FIELD_GET(CPT_INST_CREDIT_BPID, val); 5624 5625 return 0; 5626 } 5627 5628 int rvu_mbox_handler_nix_inline_ipsec_lf_cfg(struct rvu *rvu, 5629 struct nix_inline_ipsec_lf_cfg *req, 5630 struct msg_rsp *rsp) 5631 { 5632 int lf, blkaddr, err; 5633 u64 val; 5634 5635 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5636 return 0; 5637 5638 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &lf, &blkaddr); 5639 if (err) 5640 return err; 5641 5642 if (req->enable) { 5643 /* Set TT, TAG_CONST, SA_POW2_SIZE and LENM1_MAX */ 5644 val = (u64)req->ipsec_cfg0.tt << 44 | 5645 (u64)req->ipsec_cfg0.tag_const << 20 | 5646 (u64)req->ipsec_cfg0.sa_pow2_size << 16 | 5647 req->ipsec_cfg0.lenm1_max; 5648 5649 if (blkaddr == BLKADDR_NIX1) 5650 val |= BIT_ULL(46); 5651 5652 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), val); 5653 5654 /* Set SA_IDX_W and SA_IDX_MAX */ 5655 val = (u64)req->ipsec_cfg1.sa_idx_w << 32 | 5656 req->ipsec_cfg1.sa_idx_max; 5657 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), val); 5658 5659 /* Set SA base address */ 5660 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 5661 req->sa_base_addr); 5662 } else { 5663 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), 0x0); 5664 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), 0x0); 5665 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 5666 0x0); 5667 } 5668 5669 return 0; 5670 } 5671 5672 void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc) 5673 { 5674 bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 5675 5676 /* overwrite vf mac address with default_mac */ 5677 if (from_vf) 5678 ether_addr_copy(pfvf->mac_addr, pfvf->default_mac); 5679 } 5680 5681 /* NIX ingress policers or bandwidth profiles APIs */ 5682 static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr) 5683 { 5684 struct npc_lt_def_cfg defs, *ltdefs; 5685 5686 ltdefs = &defs; 5687 memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg)); 5688 5689 /* Extract PCP and DEI fields from outer VLAN from byte offset 5690 * 2 from the start of LB_PTR (ie TAG). 5691 * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN 5692 * fields are considered when 'Tunnel enable' is set in profile. 5693 */ 5694 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI, 5695 (2UL << 12) | (ltdefs->ovlan.lid << 8) | 5696 (ltdefs->ovlan.ltype_match << 4) | 5697 ltdefs->ovlan.ltype_mask); 5698 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI, 5699 (2UL << 12) | (ltdefs->ivlan.lid << 8) | 5700 (ltdefs->ivlan.ltype_match << 4) | 5701 ltdefs->ivlan.ltype_mask); 5702 5703 /* DSCP field in outer and tunneled IPv4 packets */ 5704 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP, 5705 (1UL << 12) | (ltdefs->rx_oip4.lid << 8) | 5706 (ltdefs->rx_oip4.ltype_match << 4) | 5707 ltdefs->rx_oip4.ltype_mask); 5708 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP, 5709 (1UL << 12) | (ltdefs->rx_iip4.lid << 8) | 5710 (ltdefs->rx_iip4.ltype_match << 4) | 5711 ltdefs->rx_iip4.ltype_mask); 5712 5713 /* DSCP field (traffic class) in outer and tunneled IPv6 packets */ 5714 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP, 5715 (1UL << 11) | (ltdefs->rx_oip6.lid << 8) | 5716 (ltdefs->rx_oip6.ltype_match << 4) | 5717 ltdefs->rx_oip6.ltype_mask); 5718 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP, 5719 (1UL << 11) | (ltdefs->rx_iip6.lid << 8) | 5720 (ltdefs->rx_iip6.ltype_match << 4) | 5721 ltdefs->rx_iip6.ltype_mask); 5722 } 5723 5724 static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw, 5725 int layer, int prof_idx) 5726 { 5727 struct nix_cn10k_aq_enq_req aq_req; 5728 int rc; 5729 5730 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5731 5732 aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14); 5733 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 5734 aq_req.op = NIX_AQ_INSTOP_INIT; 5735 5736 /* Context is all zeros, submit to AQ */ 5737 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5738 (struct nix_aq_enq_req *)&aq_req, NULL); 5739 if (rc) 5740 dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n", 5741 layer, prof_idx); 5742 return rc; 5743 } 5744 5745 static int nix_setup_ipolicers(struct rvu *rvu, 5746 struct nix_hw *nix_hw, int blkaddr) 5747 { 5748 struct rvu_hwinfo *hw = rvu->hw; 5749 struct nix_ipolicer *ipolicer; 5750 int err, layer, prof_idx; 5751 u64 cfg; 5752 5753 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); 5754 if (!(cfg & BIT_ULL(61))) { 5755 hw->cap.ipolicer = false; 5756 return 0; 5757 } 5758 5759 hw->cap.ipolicer = true; 5760 nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS, 5761 sizeof(*ipolicer), GFP_KERNEL); 5762 if (!nix_hw->ipolicer) 5763 return -ENOMEM; 5764 5765 cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST); 5766 5767 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5768 ipolicer = &nix_hw->ipolicer[layer]; 5769 switch (layer) { 5770 case BAND_PROF_LEAF_LAYER: 5771 ipolicer->band_prof.max = cfg & 0XFFFF; 5772 break; 5773 case BAND_PROF_MID_LAYER: 5774 ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF; 5775 break; 5776 case BAND_PROF_TOP_LAYER: 5777 ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF; 5778 break; 5779 } 5780 5781 if (!ipolicer->band_prof.max) 5782 continue; 5783 5784 err = rvu_alloc_bitmap(&ipolicer->band_prof); 5785 if (err) 5786 return err; 5787 5788 ipolicer->pfvf_map = devm_kcalloc(rvu->dev, 5789 ipolicer->band_prof.max, 5790 sizeof(u16), GFP_KERNEL); 5791 if (!ipolicer->pfvf_map) 5792 return -ENOMEM; 5793 5794 ipolicer->match_id = devm_kcalloc(rvu->dev, 5795 ipolicer->band_prof.max, 5796 sizeof(u16), GFP_KERNEL); 5797 if (!ipolicer->match_id) 5798 return -ENOMEM; 5799 5800 for (prof_idx = 0; 5801 prof_idx < ipolicer->band_prof.max; prof_idx++) { 5802 /* Set AF as current owner for INIT ops to succeed */ 5803 ipolicer->pfvf_map[prof_idx] = 0x00; 5804 5805 /* There is no enable bit in the profile context, 5806 * so no context disable. So let's INIT them here 5807 * so that PF/VF later on have to just do WRITE to 5808 * setup policer rates and config. 5809 */ 5810 err = nix_init_policer_context(rvu, nix_hw, 5811 layer, prof_idx); 5812 if (err) 5813 return err; 5814 } 5815 5816 /* Allocate memory for maintaining ref_counts for MID level 5817 * profiles, this will be needed for leaf layer profiles' 5818 * aggregation. 5819 */ 5820 if (layer != BAND_PROF_MID_LAYER) 5821 continue; 5822 5823 ipolicer->ref_count = devm_kcalloc(rvu->dev, 5824 ipolicer->band_prof.max, 5825 sizeof(u16), GFP_KERNEL); 5826 if (!ipolicer->ref_count) 5827 return -ENOMEM; 5828 } 5829 5830 /* Set policer timeunit to 2us ie (19 + 1) * 100 nsec = 2us */ 5831 rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19); 5832 5833 nix_config_rx_pkt_policer_precolor(rvu, blkaddr); 5834 5835 return 0; 5836 } 5837 5838 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw) 5839 { 5840 struct nix_ipolicer *ipolicer; 5841 int layer; 5842 5843 if (!rvu->hw->cap.ipolicer) 5844 return; 5845 5846 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5847 ipolicer = &nix_hw->ipolicer[layer]; 5848 5849 if (!ipolicer->band_prof.max) 5850 continue; 5851 5852 kfree(ipolicer->band_prof.bmap); 5853 } 5854 } 5855 5856 #define NIX_BW_PROF_HI_MASK GENMASK(10, 7) 5857 5858 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 5859 struct nix_hw *nix_hw, u16 pcifunc) 5860 { 5861 struct nix_ipolicer *ipolicer; 5862 int layer, hi_layer, prof_idx; 5863 5864 /* Bits [15:14] in profile index represent layer */ 5865 layer = (req->qidx >> 14) & 0x03; 5866 prof_idx = req->qidx & 0x3FFF; 5867 5868 ipolicer = &nix_hw->ipolicer[layer]; 5869 if (prof_idx >= ipolicer->band_prof.max) 5870 return -EINVAL; 5871 5872 /* Check if the profile is allocated to the requesting PCIFUNC or not 5873 * with the exception of AF. AF is allowed to read and update contexts. 5874 */ 5875 if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc) 5876 return -EINVAL; 5877 5878 /* If this profile is linked to higher layer profile then check 5879 * if that profile is also allocated to the requesting PCIFUNC 5880 * or not. 5881 */ 5882 if (!req->prof.hl_en) 5883 return 0; 5884 5885 /* Leaf layer profile can link only to mid layer and 5886 * mid layer to top layer. 5887 */ 5888 if (layer == BAND_PROF_LEAF_LAYER) 5889 hi_layer = BAND_PROF_MID_LAYER; 5890 else if (layer == BAND_PROF_MID_LAYER) 5891 hi_layer = BAND_PROF_TOP_LAYER; 5892 else 5893 return -EINVAL; 5894 5895 ipolicer = &nix_hw->ipolicer[hi_layer]; 5896 prof_idx = FIELD_PREP(NIX_BW_PROF_HI_MASK, req->prof.band_prof_id_h); 5897 prof_idx |= req->prof.band_prof_id; 5898 if (prof_idx >= ipolicer->band_prof.max || 5899 ipolicer->pfvf_map[prof_idx] != pcifunc) 5900 return -EINVAL; 5901 5902 return 0; 5903 } 5904 5905 int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu, 5906 struct nix_bandprof_alloc_req *req, 5907 struct nix_bandprof_alloc_rsp *rsp) 5908 { 5909 int blkaddr, layer, prof, idx, err; 5910 u16 pcifunc = req->hdr.pcifunc; 5911 struct nix_ipolicer *ipolicer; 5912 struct nix_hw *nix_hw; 5913 5914 if (!rvu->hw->cap.ipolicer) 5915 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5916 5917 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5918 if (err) 5919 return err; 5920 5921 mutex_lock(&rvu->rsrc_lock); 5922 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5923 if (layer == BAND_PROF_INVAL_LAYER) 5924 continue; 5925 if (!req->prof_count[layer]) 5926 continue; 5927 5928 ipolicer = &nix_hw->ipolicer[layer]; 5929 for (idx = 0; idx < req->prof_count[layer]; idx++) { 5930 /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */ 5931 if (idx == MAX_BANDPROF_PER_PFFUNC) 5932 break; 5933 5934 prof = rvu_alloc_rsrc(&ipolicer->band_prof); 5935 if (prof < 0) 5936 break; 5937 rsp->prof_count[layer]++; 5938 rsp->prof_idx[layer][idx] = prof; 5939 ipolicer->pfvf_map[prof] = pcifunc; 5940 } 5941 } 5942 mutex_unlock(&rvu->rsrc_lock); 5943 return 0; 5944 } 5945 5946 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc) 5947 { 5948 int blkaddr, layer, prof_idx, err; 5949 struct nix_ipolicer *ipolicer; 5950 struct nix_hw *nix_hw; 5951 5952 if (!rvu->hw->cap.ipolicer) 5953 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5954 5955 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5956 if (err) 5957 return err; 5958 5959 mutex_lock(&rvu->rsrc_lock); 5960 /* Free all the profiles allocated to the PCIFUNC */ 5961 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5962 if (layer == BAND_PROF_INVAL_LAYER) 5963 continue; 5964 ipolicer = &nix_hw->ipolicer[layer]; 5965 5966 for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) { 5967 if (ipolicer->pfvf_map[prof_idx] != pcifunc) 5968 continue; 5969 5970 /* Clear ratelimit aggregation, if any */ 5971 if (layer == BAND_PROF_LEAF_LAYER && 5972 ipolicer->match_id[prof_idx]) 5973 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 5974 5975 ipolicer->pfvf_map[prof_idx] = 0x00; 5976 ipolicer->match_id[prof_idx] = 0; 5977 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 5978 } 5979 } 5980 mutex_unlock(&rvu->rsrc_lock); 5981 return 0; 5982 } 5983 5984 int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu, 5985 struct nix_bandprof_free_req *req, 5986 struct msg_rsp *rsp) 5987 { 5988 int blkaddr, layer, prof_idx, idx, err; 5989 u16 pcifunc = req->hdr.pcifunc; 5990 struct nix_ipolicer *ipolicer; 5991 struct nix_hw *nix_hw; 5992 5993 if (req->free_all) 5994 return nix_free_all_bandprof(rvu, pcifunc); 5995 5996 if (!rvu->hw->cap.ipolicer) 5997 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5998 5999 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 6000 if (err) 6001 return err; 6002 6003 mutex_lock(&rvu->rsrc_lock); 6004 /* Free the requested profile indices */ 6005 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 6006 if (layer == BAND_PROF_INVAL_LAYER) 6007 continue; 6008 if (!req->prof_count[layer]) 6009 continue; 6010 6011 ipolicer = &nix_hw->ipolicer[layer]; 6012 for (idx = 0; idx < req->prof_count[layer]; idx++) { 6013 if (idx == MAX_BANDPROF_PER_PFFUNC) 6014 break; 6015 prof_idx = req->prof_idx[layer][idx]; 6016 if (prof_idx >= ipolicer->band_prof.max || 6017 ipolicer->pfvf_map[prof_idx] != pcifunc) 6018 continue; 6019 6020 /* Clear ratelimit aggregation, if any */ 6021 if (layer == BAND_PROF_LEAF_LAYER && 6022 ipolicer->match_id[prof_idx]) 6023 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 6024 6025 ipolicer->pfvf_map[prof_idx] = 0x00; 6026 ipolicer->match_id[prof_idx] = 0; 6027 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 6028 } 6029 } 6030 mutex_unlock(&rvu->rsrc_lock); 6031 return 0; 6032 } 6033 6034 int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw, 6035 struct nix_cn10k_aq_enq_req *aq_req, 6036 struct nix_cn10k_aq_enq_rsp *aq_rsp, 6037 u16 pcifunc, u8 ctype, u32 qidx) 6038 { 6039 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6040 aq_req->hdr.pcifunc = pcifunc; 6041 aq_req->ctype = ctype; 6042 aq_req->op = NIX_AQ_INSTOP_READ; 6043 aq_req->qidx = qidx; 6044 6045 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6046 (struct nix_aq_enq_req *)aq_req, 6047 (struct nix_aq_enq_rsp *)aq_rsp); 6048 } 6049 6050 static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu, 6051 struct nix_hw *nix_hw, 6052 struct nix_cn10k_aq_enq_req *aq_req, 6053 struct nix_cn10k_aq_enq_rsp *aq_rsp, 6054 u32 leaf_prof, u16 mid_prof) 6055 { 6056 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6057 aq_req->hdr.pcifunc = 0x00; 6058 aq_req->ctype = NIX_AQ_CTYPE_BANDPROF; 6059 aq_req->op = NIX_AQ_INSTOP_WRITE; 6060 aq_req->qidx = leaf_prof; 6061 6062 aq_req->prof.band_prof_id = mid_prof & 0x7F; 6063 aq_req->prof_mask.band_prof_id = GENMASK(6, 0); 6064 aq_req->prof.band_prof_id_h = FIELD_GET(NIX_BW_PROF_HI_MASK, mid_prof); 6065 aq_req->prof_mask.band_prof_id_h = GENMASK(3, 0); 6066 aq_req->prof.hl_en = 1; 6067 aq_req->prof_mask.hl_en = 1; 6068 6069 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6070 (struct nix_aq_enq_req *)aq_req, 6071 (struct nix_aq_enq_rsp *)aq_rsp); 6072 } 6073 6074 #define NIX_RQ_PROF_HI_MASK GENMASK(13, 10) 6075 6076 int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, 6077 u16 rq_idx, u16 match_id) 6078 { 6079 int leaf_prof, mid_prof, leaf_match; 6080 struct nix_cn10k_aq_enq_req aq_req; 6081 struct nix_cn10k_aq_enq_rsp aq_rsp; 6082 struct nix_ipolicer *ipolicer; 6083 struct nix_hw *nix_hw; 6084 int blkaddr, idx, rc; 6085 6086 if (!rvu->hw->cap.ipolicer) 6087 return 0; 6088 6089 rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 6090 if (rc) 6091 return rc; 6092 6093 /* Fetch the RQ's context to see if policing is enabled */ 6094 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc, 6095 NIX_AQ_CTYPE_RQ, rq_idx); 6096 if (rc) { 6097 dev_err(rvu->dev, 6098 "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n", 6099 __func__, rq_idx, pcifunc); 6100 return rc; 6101 } 6102 6103 if (!aq_rsp.rq.policer_ena) 6104 return 0; 6105 6106 /* Get the bandwidth profile ID mapped to this RQ */ 6107 leaf_prof = FIELD_PREP(NIX_RQ_PROF_HI_MASK, aq_rsp.rq.band_prof_id_h); 6108 leaf_prof |= aq_rsp.rq.band_prof_id; 6109 6110 ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER]; 6111 ipolicer->match_id[leaf_prof] = match_id; 6112 6113 /* Check if any other leaf profile is marked with same match_id */ 6114 for (idx = 0; idx < ipolicer->band_prof.max; idx++) { 6115 if (idx == leaf_prof) 6116 continue; 6117 if (ipolicer->match_id[idx] != match_id) 6118 continue; 6119 6120 leaf_match = idx; 6121 break; 6122 } 6123 6124 if (idx == ipolicer->band_prof.max) 6125 return 0; 6126 6127 /* Fetch the matching profile's context to check if it's already 6128 * mapped to a mid level profile. 6129 */ 6130 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6131 NIX_AQ_CTYPE_BANDPROF, leaf_match); 6132 if (rc) { 6133 dev_err(rvu->dev, 6134 "%s: Failed to fetch context of leaf profile %d\n", 6135 __func__, leaf_match); 6136 return rc; 6137 } 6138 6139 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 6140 if (aq_rsp.prof.hl_en) { 6141 /* Get Mid layer prof index and map leaf_prof index 6142 * also such that flows that are being steered 6143 * to different RQs and marked with same match_id 6144 * are rate limited in a aggregate fashion 6145 */ 6146 mid_prof = FIELD_PREP(NIX_BW_PROF_HI_MASK, 6147 aq_rsp.prof.band_prof_id_h); 6148 mid_prof |= aq_rsp.prof.band_prof_id; 6149 6150 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6151 &aq_req, &aq_rsp, 6152 leaf_prof, mid_prof); 6153 if (rc) { 6154 dev_err(rvu->dev, 6155 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6156 __func__, leaf_prof, mid_prof); 6157 goto exit; 6158 } 6159 6160 mutex_lock(&rvu->rsrc_lock); 6161 ipolicer->ref_count[mid_prof]++; 6162 mutex_unlock(&rvu->rsrc_lock); 6163 goto exit; 6164 } 6165 6166 /* Allocate a mid layer profile and 6167 * map both 'leaf_prof' and 'leaf_match' profiles to it. 6168 */ 6169 mutex_lock(&rvu->rsrc_lock); 6170 mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof); 6171 if (mid_prof < 0) { 6172 dev_err(rvu->dev, 6173 "%s: Unable to allocate mid layer profile\n", __func__); 6174 mutex_unlock(&rvu->rsrc_lock); 6175 goto exit; 6176 } 6177 mutex_unlock(&rvu->rsrc_lock); 6178 ipolicer->pfvf_map[mid_prof] = 0x00; 6179 ipolicer->ref_count[mid_prof] = 0; 6180 6181 /* Initialize mid layer profile same as 'leaf_prof' */ 6182 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6183 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 6184 if (rc) { 6185 dev_err(rvu->dev, 6186 "%s: Failed to fetch context of leaf profile %d\n", 6187 __func__, leaf_prof); 6188 goto exit; 6189 } 6190 6191 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6192 aq_req.hdr.pcifunc = 0x00; 6193 aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14); 6194 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 6195 aq_req.op = NIX_AQ_INSTOP_WRITE; 6196 memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s)); 6197 memset((char *)&aq_req.prof_mask, 0xff, sizeof(struct nix_bandprof_s)); 6198 /* Clear higher layer enable bit in the mid profile, just in case */ 6199 aq_req.prof.hl_en = 0; 6200 aq_req.prof_mask.hl_en = 1; 6201 6202 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6203 (struct nix_aq_enq_req *)&aq_req, NULL); 6204 if (rc) { 6205 dev_err(rvu->dev, 6206 "%s: Failed to INIT context of mid layer profile %d\n", 6207 __func__, mid_prof); 6208 goto exit; 6209 } 6210 6211 /* Map both leaf profiles to this mid layer profile */ 6212 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6213 &aq_req, &aq_rsp, 6214 leaf_prof, mid_prof); 6215 if (rc) { 6216 dev_err(rvu->dev, 6217 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6218 __func__, leaf_prof, mid_prof); 6219 goto exit; 6220 } 6221 6222 mutex_lock(&rvu->rsrc_lock); 6223 ipolicer->ref_count[mid_prof]++; 6224 mutex_unlock(&rvu->rsrc_lock); 6225 6226 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6227 &aq_req, &aq_rsp, 6228 leaf_match, mid_prof); 6229 if (rc) { 6230 dev_err(rvu->dev, 6231 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6232 __func__, leaf_match, mid_prof); 6233 ipolicer->ref_count[mid_prof]--; 6234 goto exit; 6235 } 6236 6237 mutex_lock(&rvu->rsrc_lock); 6238 ipolicer->ref_count[mid_prof]++; 6239 mutex_unlock(&rvu->rsrc_lock); 6240 6241 exit: 6242 return rc; 6243 } 6244 6245 /* Called with mutex rsrc_lock */ 6246 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 6247 u32 leaf_prof) 6248 { 6249 struct nix_cn10k_aq_enq_req aq_req; 6250 struct nix_cn10k_aq_enq_rsp aq_rsp; 6251 struct nix_ipolicer *ipolicer; 6252 u16 mid_prof; 6253 int rc; 6254 6255 mutex_unlock(&rvu->rsrc_lock); 6256 6257 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6258 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 6259 6260 mutex_lock(&rvu->rsrc_lock); 6261 if (rc) { 6262 dev_err(rvu->dev, 6263 "%s: Failed to fetch context of leaf profile %d\n", 6264 __func__, leaf_prof); 6265 return; 6266 } 6267 6268 if (!aq_rsp.prof.hl_en) 6269 return; 6270 6271 mid_prof = FIELD_PREP(NIX_BW_PROF_HI_MASK, aq_rsp.prof.band_prof_id_h); 6272 mid_prof |= aq_rsp.prof.band_prof_id; 6273 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 6274 ipolicer->ref_count[mid_prof]--; 6275 /* If ref_count is zero, free mid layer profile */ 6276 if (!ipolicer->ref_count[mid_prof]) { 6277 ipolicer->pfvf_map[mid_prof] = 0x00; 6278 rvu_free_rsrc(&ipolicer->band_prof, mid_prof); 6279 } 6280 } 6281 6282 int rvu_mbox_handler_nix_bandprof_get_hwinfo(struct rvu *rvu, struct msg_req *req, 6283 struct nix_bandprof_get_hwinfo_rsp *rsp) 6284 { 6285 struct nix_ipolicer *ipolicer; 6286 int blkaddr, layer, err; 6287 struct nix_hw *nix_hw; 6288 u64 tu; 6289 6290 if (!rvu->hw->cap.ipolicer) 6291 return NIX_AF_ERR_IPOLICER_NOTSUPP; 6292 6293 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6294 if (err) 6295 return err; 6296 6297 /* Return number of bandwidth profiles free at each layer */ 6298 mutex_lock(&rvu->rsrc_lock); 6299 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 6300 if (layer == BAND_PROF_INVAL_LAYER) 6301 continue; 6302 6303 ipolicer = &nix_hw->ipolicer[layer]; 6304 rsp->prof_count[layer] = rvu_rsrc_free_count(&ipolicer->band_prof); 6305 } 6306 mutex_unlock(&rvu->rsrc_lock); 6307 6308 /* Set the policer timeunit in nanosec */ 6309 tu = rvu_read64(rvu, blkaddr, NIX_AF_PL_TS) & GENMASK_ULL(9, 0); 6310 rsp->policer_timeunit = (tu + 1) * 100; 6311 6312 return 0; 6313 } 6314 6315 static struct nix_mcast_grp_elem *rvu_nix_mcast_find_grp_elem(struct nix_mcast_grp *mcast_grp, 6316 u32 mcast_grp_idx) 6317 { 6318 struct nix_mcast_grp_elem *iter; 6319 bool is_found = false; 6320 6321 list_for_each_entry(iter, &mcast_grp->mcast_grp_head, list) { 6322 if (iter->mcast_grp_idx == mcast_grp_idx) { 6323 is_found = true; 6324 break; 6325 } 6326 } 6327 6328 if (is_found) 6329 return iter; 6330 6331 return NULL; 6332 } 6333 6334 int rvu_nix_mcast_get_mce_index(struct rvu *rvu, u16 pcifunc, u32 mcast_grp_idx) 6335 { 6336 struct nix_mcast_grp_elem *elem; 6337 struct nix_mcast_grp *mcast_grp; 6338 struct nix_hw *nix_hw; 6339 int blkaddr, ret; 6340 6341 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6342 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6343 if (!nix_hw) 6344 return NIX_AF_ERR_INVALID_NIXBLK; 6345 6346 mcast_grp = &nix_hw->mcast_grp; 6347 mutex_lock(&mcast_grp->mcast_grp_lock); 6348 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); 6349 if (!elem) 6350 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6351 else 6352 ret = elem->mce_start_index; 6353 6354 mutex_unlock(&mcast_grp->mcast_grp_lock); 6355 return ret; 6356 } 6357 6358 void rvu_nix_mcast_flr_free_entries(struct rvu *rvu, u16 pcifunc) 6359 { 6360 struct nix_mcast_grp_destroy_req dreq = { 0 }; 6361 struct nix_mcast_grp_update_req ureq = { 0 }; 6362 struct nix_mcast_grp_update_rsp ursp = { 0 }; 6363 struct nix_mcast_grp_elem *elem, *tmp; 6364 struct nix_mcast_grp *mcast_grp; 6365 struct nix_hw *nix_hw; 6366 int blkaddr; 6367 6368 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6369 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6370 if (!nix_hw) 6371 return; 6372 6373 mcast_grp = &nix_hw->mcast_grp; 6374 6375 mutex_lock(&mcast_grp->mcast_grp_lock); 6376 list_for_each_entry_safe(elem, tmp, &mcast_grp->mcast_grp_head, list) { 6377 struct nix_mce_list *mce_list; 6378 struct hlist_node *tmp; 6379 struct mce *mce; 6380 6381 /* If the pcifunc which created the multicast/mirror 6382 * group received an FLR, then delete the entire group. 6383 */ 6384 if (elem->pcifunc == pcifunc) { 6385 /* Delete group */ 6386 dreq.hdr.pcifunc = elem->pcifunc; 6387 dreq.mcast_grp_idx = elem->mcast_grp_idx; 6388 dreq.is_af = 1; 6389 rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL); 6390 continue; 6391 } 6392 6393 /* Iterate the group elements and delete the element which 6394 * received the FLR. 6395 */ 6396 mce_list = &elem->mcast_mce_list; 6397 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 6398 if (mce->pcifunc == pcifunc) { 6399 ureq.hdr.pcifunc = pcifunc; 6400 ureq.num_mce_entry = 1; 6401 ureq.mcast_grp_idx = elem->mcast_grp_idx; 6402 ureq.op = NIX_MCAST_OP_DEL_ENTRY; 6403 ureq.pcifunc[0] = pcifunc; 6404 ureq.is_af = 1; 6405 rvu_mbox_handler_nix_mcast_grp_update(rvu, &ureq, &ursp); 6406 break; 6407 } 6408 } 6409 } 6410 mutex_unlock(&mcast_grp->mcast_grp_lock); 6411 } 6412 6413 int rvu_nix_mcast_update_mcam_entry(struct rvu *rvu, u16 pcifunc, 6414 u32 mcast_grp_idx, u16 mcam_index) 6415 { 6416 struct nix_mcast_grp_elem *elem; 6417 struct nix_mcast_grp *mcast_grp; 6418 struct nix_hw *nix_hw; 6419 int blkaddr, ret = 0; 6420 6421 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6422 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6423 if (!nix_hw) 6424 return NIX_AF_ERR_INVALID_NIXBLK; 6425 6426 mcast_grp = &nix_hw->mcast_grp; 6427 mutex_lock(&mcast_grp->mcast_grp_lock); 6428 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); 6429 if (!elem) 6430 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6431 else 6432 elem->mcam_index = mcam_index; 6433 6434 mutex_unlock(&mcast_grp->mcast_grp_lock); 6435 return ret; 6436 } 6437 6438 int rvu_mbox_handler_nix_mcast_grp_create(struct rvu *rvu, 6439 struct nix_mcast_grp_create_req *req, 6440 struct nix_mcast_grp_create_rsp *rsp) 6441 { 6442 struct nix_mcast_grp_elem *elem; 6443 struct nix_mcast_grp *mcast_grp; 6444 struct nix_hw *nix_hw; 6445 int blkaddr, err; 6446 6447 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6448 if (err) 6449 return err; 6450 6451 mcast_grp = &nix_hw->mcast_grp; 6452 elem = kzalloc_obj(*elem); 6453 if (!elem) 6454 return -ENOMEM; 6455 6456 INIT_HLIST_HEAD(&elem->mcast_mce_list.head); 6457 elem->mcam_index = -1; 6458 elem->mce_start_index = -1; 6459 elem->pcifunc = req->hdr.pcifunc; 6460 elem->dir = req->dir; 6461 elem->mcast_grp_idx = mcast_grp->next_grp_index++; 6462 6463 mutex_lock(&mcast_grp->mcast_grp_lock); 6464 list_add_tail(&elem->list, &mcast_grp->mcast_grp_head); 6465 mcast_grp->count++; 6466 mutex_unlock(&mcast_grp->mcast_grp_lock); 6467 6468 rsp->mcast_grp_idx = elem->mcast_grp_idx; 6469 return 0; 6470 } 6471 6472 int rvu_mbox_handler_nix_mcast_grp_destroy(struct rvu *rvu, 6473 struct nix_mcast_grp_destroy_req *req, 6474 struct msg_rsp *rsp) 6475 { 6476 struct npc_delete_flow_req uninstall_req = { 0 }; 6477 struct npc_delete_flow_rsp uninstall_rsp = { 0 }; 6478 struct nix_mcast_grp_elem *elem; 6479 struct nix_mcast_grp *mcast_grp; 6480 int blkaddr, err, ret = 0; 6481 struct nix_mcast *mcast; 6482 struct nix_hw *nix_hw; 6483 6484 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6485 if (err) 6486 return err; 6487 6488 mcast_grp = &nix_hw->mcast_grp; 6489 6490 /* If AF is requesting for the deletion, 6491 * then AF is already taking the lock 6492 */ 6493 if (!req->is_af) 6494 mutex_lock(&mcast_grp->mcast_grp_lock); 6495 6496 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); 6497 if (!elem) { 6498 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6499 goto unlock_grp; 6500 } 6501 6502 /* If no mce entries are associated with the group 6503 * then just remove it from the global list. 6504 */ 6505 if (!elem->mcast_mce_list.count) 6506 goto delete_grp; 6507 6508 /* Delete the associated mcam entry and 6509 * remove all mce entries from the group 6510 */ 6511 mcast = &nix_hw->mcast; 6512 mutex_lock(&mcast->mce_lock); 6513 if (elem->mcam_index != -1) { 6514 uninstall_req.hdr.pcifunc = req->hdr.pcifunc; 6515 uninstall_req.entry = elem->mcam_index; 6516 rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &uninstall_rsp); 6517 } 6518 6519 nix_free_mce_list(mcast, elem->mcast_mce_list.count, 6520 elem->mce_start_index, elem->dir); 6521 nix_delete_mcast_mce_list(&elem->mcast_mce_list); 6522 mutex_unlock(&mcast->mce_lock); 6523 6524 delete_grp: 6525 list_del(&elem->list); 6526 kfree(elem); 6527 mcast_grp->count--; 6528 6529 unlock_grp: 6530 if (!req->is_af) 6531 mutex_unlock(&mcast_grp->mcast_grp_lock); 6532 6533 return ret; 6534 } 6535 6536 int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, 6537 struct nix_mcast_grp_update_req *req, 6538 struct nix_mcast_grp_update_rsp *rsp) 6539 { 6540 struct nix_mcast_grp_destroy_req dreq = { 0 }; 6541 struct npc_mcam *mcam = &rvu->hw->mcam; 6542 struct nix_mcast_grp_elem *elem; 6543 struct nix_mcast_grp *mcast_grp; 6544 int blkaddr, err, npc_blkaddr; 6545 u16 prev_count, new_count; 6546 struct nix_mcast *mcast; 6547 struct nix_hw *nix_hw; 6548 int i, ret; 6549 6550 if (!req->num_mce_entry) 6551 return 0; 6552 6553 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6554 if (err) 6555 return err; 6556 6557 mcast_grp = &nix_hw->mcast_grp; 6558 6559 /* If AF is requesting for the updation, 6560 * then AF is already taking the lock 6561 */ 6562 if (!req->is_af) 6563 mutex_lock(&mcast_grp->mcast_grp_lock); 6564 6565 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); 6566 if (!elem) { 6567 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6568 goto unlock_grp; 6569 } 6570 6571 /* If any pcifunc matches the group's pcifunc, then we can 6572 * delete the entire group. 6573 */ 6574 if (req->op == NIX_MCAST_OP_DEL_ENTRY) { 6575 for (i = 0; i < req->num_mce_entry; i++) { 6576 if (elem->pcifunc == req->pcifunc[i]) { 6577 /* Delete group */ 6578 dreq.hdr.pcifunc = elem->pcifunc; 6579 dreq.mcast_grp_idx = elem->mcast_grp_idx; 6580 dreq.is_af = 1; 6581 rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL); 6582 ret = 0; 6583 goto unlock_grp; 6584 } 6585 } 6586 } 6587 6588 mcast = &nix_hw->mcast; 6589 mutex_lock(&mcast->mce_lock); 6590 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 6591 if (elem->mcam_index != -1) 6592 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, false); 6593 6594 prev_count = elem->mcast_mce_list.count; 6595 if (req->op == NIX_MCAST_OP_ADD_ENTRY) { 6596 new_count = prev_count + req->num_mce_entry; 6597 if (prev_count) 6598 nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir); 6599 6600 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir); 6601 6602 /* It is possible not to get contiguous memory */ 6603 if (elem->mce_start_index < 0) { 6604 if (elem->mcam_index != -1) { 6605 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6606 elem->mcam_index, true); 6607 ret = NIX_AF_ERR_NON_CONTIG_MCE_LIST; 6608 goto unlock_mce; 6609 } 6610 } 6611 6612 ret = nix_add_mce_list_entry(rvu, nix_hw, elem, req); 6613 if (ret) { 6614 nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir); 6615 if (prev_count) 6616 elem->mce_start_index = nix_alloc_mce_list(mcast, 6617 prev_count, 6618 elem->dir); 6619 6620 if (elem->mcam_index != -1) 6621 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6622 elem->mcam_index, true); 6623 6624 goto unlock_mce; 6625 } 6626 } else { 6627 if (!prev_count || prev_count < req->num_mce_entry) { 6628 if (elem->mcam_index != -1) 6629 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6630 elem->mcam_index, true); 6631 ret = NIX_AF_ERR_INVALID_MCAST_DEL_REQ; 6632 goto unlock_mce; 6633 } 6634 6635 nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir); 6636 new_count = prev_count - req->num_mce_entry; 6637 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir); 6638 ret = nix_del_mce_list_entry(rvu, nix_hw, elem, req); 6639 if (ret) { 6640 nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir); 6641 elem->mce_start_index = nix_alloc_mce_list(mcast, prev_count, elem->dir); 6642 if (elem->mcam_index != -1) 6643 npc_enable_mcam_entry(rvu, mcam, 6644 npc_blkaddr, 6645 elem->mcam_index, 6646 true); 6647 6648 goto unlock_mce; 6649 } 6650 } 6651 6652 if (elem->mcam_index == -1) { 6653 rsp->mce_start_index = elem->mce_start_index; 6654 ret = 0; 6655 goto unlock_mce; 6656 } 6657 6658 nix_mcast_update_action(rvu, elem); 6659 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true); 6660 rsp->mce_start_index = elem->mce_start_index; 6661 ret = 0; 6662 6663 unlock_mce: 6664 mutex_unlock(&mcast->mce_lock); 6665 6666 unlock_grp: 6667 if (!req->is_af) 6668 mutex_unlock(&mcast_grp->mcast_grp_lock); 6669 6670 return ret; 6671 } 6672 6673 /* On CN10k and older series of silicons, hardware may incorrectly 6674 * assert XOFF on certain channels. Issue a write on NIX_AF_RX_CHANX_CFG 6675 * to broadcacst XON on the same. 6676 */ 6677 void rvu_block_bcast_xon(struct rvu *rvu, int blkaddr) 6678 { 6679 struct rvu_block *block = &rvu->hw->block[blkaddr]; 6680 u64 cfg; 6681 6682 if (!block->implemented || is_cn20k(rvu->pdev)) 6683 return; 6684 6685 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(0)); 6686 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(0), cfg); 6687 } 6688