1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/module.h> 9 #include <linux/pci.h> 10 11 #include "rvu_struct.h" 12 #include "rvu_reg.h" 13 #include "rvu.h" 14 #include "npc.h" 15 #include "mcs.h" 16 #include "cgx.h" 17 #include "lmac_common.h" 18 #include "rvu_npc_hash.h" 19 #include "cn20k/npc.h" 20 21 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc); 22 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 23 int type, int chan_id); 24 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 25 int type, bool add); 26 static int nix_setup_ipolicers(struct rvu *rvu, 27 struct nix_hw *nix_hw, int blkaddr); 28 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw); 29 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 30 struct nix_hw *nix_hw, u16 pcifunc); 31 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc); 32 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 33 u32 leaf_prof); 34 static const char *nix_get_ctx_name(int ctype); 35 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc); 36 37 enum mc_tbl_sz { 38 MC_TBL_SZ_256, 39 MC_TBL_SZ_512, 40 MC_TBL_SZ_1K, 41 MC_TBL_SZ_2K, 42 MC_TBL_SZ_4K, 43 MC_TBL_SZ_8K, 44 MC_TBL_SZ_16K, 45 MC_TBL_SZ_32K, 46 MC_TBL_SZ_64K, 47 }; 48 49 enum mc_buf_cnt { 50 MC_BUF_CNT_8, 51 MC_BUF_CNT_16, 52 MC_BUF_CNT_32, 53 MC_BUF_CNT_64, 54 MC_BUF_CNT_128, 55 MC_BUF_CNT_256, 56 MC_BUF_CNT_512, 57 MC_BUF_CNT_1024, 58 MC_BUF_CNT_2048, 59 }; 60 61 enum nix_makr_fmt_indexes { 62 NIX_MARK_CFG_IP_DSCP_RED, 63 NIX_MARK_CFG_IP_DSCP_YELLOW, 64 NIX_MARK_CFG_IP_DSCP_YELLOW_RED, 65 NIX_MARK_CFG_IP_ECN_RED, 66 NIX_MARK_CFG_IP_ECN_YELLOW, 67 NIX_MARK_CFG_IP_ECN_YELLOW_RED, 68 NIX_MARK_CFG_VLAN_DEI_RED, 69 NIX_MARK_CFG_VLAN_DEI_YELLOW, 70 NIX_MARK_CFG_VLAN_DEI_YELLOW_RED, 71 NIX_MARK_CFG_MAX, 72 }; 73 74 /* For now considering MC resources needed for broadcast 75 * pkt replication only. i.e 256 HWVFs + 12 PFs. 76 */ 77 #define MC_TBL_SIZE MC_TBL_SZ_2K 78 #define MC_BUF_CNT MC_BUF_CNT_1024 79 80 #define MC_TX_MAX 2048 81 82 struct mce { 83 struct hlist_node node; 84 u32 rq_rss_index; 85 u16 pcifunc; 86 u16 channel; 87 u8 dest_type; 88 u8 is_active; 89 u8 reserved[2]; 90 }; 91 92 int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr) 93 { 94 int i = 0; 95 96 /*If blkaddr is 0, return the first nix block address*/ 97 if (blkaddr == 0) 98 return rvu->nix_blkaddr[blkaddr]; 99 100 while (i + 1 < MAX_NIX_BLKS) { 101 if (rvu->nix_blkaddr[i] == blkaddr) 102 return rvu->nix_blkaddr[i + 1]; 103 i++; 104 } 105 106 return 0; 107 } 108 109 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc) 110 { 111 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 112 int blkaddr; 113 114 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 115 if (!pfvf->nixlf || blkaddr < 0) 116 return false; 117 return true; 118 } 119 120 int rvu_get_nixlf_count(struct rvu *rvu) 121 { 122 int blkaddr = 0, max = 0; 123 struct rvu_block *block; 124 125 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 126 while (blkaddr) { 127 block = &rvu->hw->block[blkaddr]; 128 max += block->lf.max; 129 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 130 } 131 return max; 132 } 133 134 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr) 135 { 136 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 137 struct rvu_hwinfo *hw = rvu->hw; 138 int blkaddr; 139 140 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 141 if (!pfvf->nixlf || blkaddr < 0) 142 return NIX_AF_ERR_AF_LF_INVALID; 143 144 *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 145 if (*nixlf < 0) 146 return NIX_AF_ERR_AF_LF_INVALID; 147 148 if (nix_blkaddr) 149 *nix_blkaddr = blkaddr; 150 151 return 0; 152 } 153 154 int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc, 155 struct nix_hw **nix_hw, int *blkaddr) 156 { 157 struct rvu_pfvf *pfvf; 158 159 pfvf = rvu_get_pfvf(rvu, pcifunc); 160 *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 161 if (!pfvf->nixlf || *blkaddr < 0) 162 return NIX_AF_ERR_AF_LF_INVALID; 163 164 *nix_hw = get_nix_hw(rvu->hw, *blkaddr); 165 if (!*nix_hw) 166 return NIX_AF_ERR_INVALID_NIXBLK; 167 return 0; 168 } 169 170 static void nix_mce_list_init(struct nix_mce_list *list, int max) 171 { 172 INIT_HLIST_HEAD(&list->head); 173 list->count = 0; 174 list->max = max; 175 } 176 177 static int nix_alloc_mce_list(struct nix_mcast *mcast, int count, u8 dir) 178 { 179 struct rsrc_bmap *mce_counter; 180 int idx; 181 182 if (!mcast) 183 return -EINVAL; 184 185 mce_counter = &mcast->mce_counter[dir]; 186 if (!rvu_rsrc_check_contig(mce_counter, count)) 187 return -ENOSPC; 188 189 idx = rvu_alloc_rsrc_contig(mce_counter, count); 190 return idx; 191 } 192 193 static void nix_free_mce_list(struct nix_mcast *mcast, int count, int start, u8 dir) 194 { 195 struct rsrc_bmap *mce_counter; 196 197 if (!mcast) 198 return; 199 200 mce_counter = &mcast->mce_counter[dir]; 201 rvu_free_rsrc_contig(mce_counter, count, start); 202 } 203 204 struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr) 205 { 206 int nix_blkaddr = 0, i = 0; 207 struct rvu *rvu = hw->rvu; 208 209 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 210 while (nix_blkaddr) { 211 if (blkaddr == nix_blkaddr && hw->nix) 212 return &hw->nix[i]; 213 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 214 i++; 215 } 216 return NULL; 217 } 218 219 int nix_get_dwrr_mtu_reg(struct rvu_hwinfo *hw, int smq_link_type) 220 { 221 if (hw->cap.nix_multiple_dwrr_mtu) 222 return NIX_AF_DWRR_MTUX(smq_link_type); 223 224 if (smq_link_type == SMQ_LINK_TYPE_SDP) 225 return NIX_AF_DWRR_SDP_MTU; 226 227 /* Here it's same reg for RPM and LBK */ 228 return NIX_AF_DWRR_RPM_MTU; 229 } 230 231 u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu) 232 { 233 dwrr_mtu &= 0x1FULL; 234 235 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 236 * Value of 4 is reserved for MTU value of 9728 bytes. 237 * Value of 5 is reserved for MTU value of 10240 bytes. 238 */ 239 switch (dwrr_mtu) { 240 case 4: 241 return 9728; 242 case 5: 243 return 10240; 244 default: 245 return BIT_ULL(dwrr_mtu); 246 } 247 248 return 0; 249 } 250 251 u32 convert_bytes_to_dwrr_mtu(u32 bytes) 252 { 253 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 254 * Value of 4 is reserved for MTU value of 9728 bytes. 255 * Value of 5 is reserved for MTU value of 10240 bytes. 256 */ 257 if (bytes > BIT_ULL(16)) 258 return 0; 259 260 switch (bytes) { 261 case 9728: 262 return 4; 263 case 10240: 264 return 5; 265 default: 266 return ilog2(bytes); 267 } 268 269 return 0; 270 } 271 272 static void nix_rx_sync(struct rvu *rvu, int blkaddr) 273 { 274 int err; 275 276 /* Sync all in flight RX packets to LLC/DRAM */ 277 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 278 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 279 if (err) 280 dev_err(rvu->dev, "SYNC1: NIX RX software sync failed\n"); 281 282 /* SW_SYNC ensures all existing transactions are finished and pkts 283 * are written to LLC/DRAM, queues should be teared down after 284 * successful SW_SYNC. Due to a HW errata, in some rare scenarios 285 * an existing transaction might end after SW_SYNC operation. To 286 * ensure operation is fully done, do the SW_SYNC twice. 287 */ 288 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 289 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 290 if (err) 291 dev_err(rvu->dev, "SYNC2: NIX RX software sync failed\n"); 292 } 293 294 static bool is_valid_txschq(struct rvu *rvu, int blkaddr, 295 int lvl, u16 pcifunc, u16 schq) 296 { 297 struct rvu_hwinfo *hw = rvu->hw; 298 struct nix_txsch *txsch; 299 struct nix_hw *nix_hw; 300 u16 map_func; 301 302 nix_hw = get_nix_hw(rvu->hw, blkaddr); 303 if (!nix_hw) 304 return false; 305 306 txsch = &nix_hw->txsch[lvl]; 307 /* Check out of bounds */ 308 if (schq >= txsch->schq.max) 309 return false; 310 311 mutex_lock(&rvu->rsrc_lock); 312 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 313 mutex_unlock(&rvu->rsrc_lock); 314 315 /* TLs aggegating traffic are shared across PF and VFs */ 316 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 317 if ((nix_get_tx_link(rvu, map_func) != 318 nix_get_tx_link(rvu, pcifunc)) && 319 (rvu_get_pf(rvu->pdev, map_func) != 320 rvu_get_pf(rvu->pdev, pcifunc))) 321 return false; 322 else 323 return true; 324 } 325 326 if (map_func != pcifunc) 327 return false; 328 329 return true; 330 } 331 332 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf, 333 struct nix_lf_alloc_rsp *rsp, bool loop) 334 { 335 struct rvu_pfvf *parent_pf, *pfvf = rvu_get_pfvf(rvu, pcifunc); 336 u16 req_chan_base, req_chan_end, req_chan_cnt; 337 struct rvu_hwinfo *hw = rvu->hw; 338 struct sdp_node_info *sdp_info; 339 int pkind, pf, vf, lbkid, vfid; 340 u8 cgx_id, lmac_id; 341 bool from_vf; 342 int err; 343 344 pf = rvu_get_pf(rvu->pdev, pcifunc); 345 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 346 type != NIX_INTF_TYPE_SDP) 347 return 0; 348 349 switch (type) { 350 case NIX_INTF_TYPE_CGX: 351 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf]; 352 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 353 354 pkind = rvu_npc_get_pkind(rvu, pf); 355 if (pkind < 0) { 356 dev_err(rvu->dev, 357 "PF_Func 0x%x: Invalid pkind\n", pcifunc); 358 return -EINVAL; 359 } 360 pfvf->rx_chan_base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0); 361 pfvf->tx_chan_base = pfvf->rx_chan_base; 362 pfvf->rx_chan_cnt = 1; 363 pfvf->tx_chan_cnt = 1; 364 rsp->tx_link = cgx_id * hw->lmac_per_cgx + lmac_id; 365 366 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind); 367 rvu_npc_set_pkind(rvu, pkind, pfvf); 368 break; 369 case NIX_INTF_TYPE_LBK: 370 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 371 372 /* If NIX1 block is present on the silicon then NIXes are 373 * assigned alternatively for lbk interfaces. NIX0 should 374 * send packets on lbk link 1 channels and NIX1 should send 375 * on lbk link 0 channels for the communication between 376 * NIX0 and NIX1. 377 */ 378 lbkid = 0; 379 if (rvu->hw->lbk_links > 1) 380 lbkid = vf & 0x1 ? 0 : 1; 381 382 /* By default NIX0 is configured to send packet on lbk link 1 383 * (which corresponds to LBK1), same packet will receive on 384 * NIX1 over lbk link 0. If NIX1 sends packet on lbk link 0 385 * (which corresponds to LBK2) packet will receive on NIX0 lbk 386 * link 1. 387 * But if lbk links for NIX0 and NIX1 are negated, i.e NIX0 388 * transmits and receives on lbk link 0, whick corresponds 389 * to LBK1 block, back to back connectivity between NIX and 390 * LBK can be achieved (which is similar to 96xx) 391 * 392 * RX TX 393 * NIX0 lbk link 1 (LBK2) 1 (LBK1) 394 * NIX0 lbk link 0 (LBK0) 0 (LBK0) 395 * NIX1 lbk link 0 (LBK1) 0 (LBK2) 396 * NIX1 lbk link 1 (LBK3) 1 (LBK3) 397 */ 398 if (loop) 399 lbkid = !lbkid; 400 401 /* Note that AF's VFs work in pairs and talk over consecutive 402 * loopback channels.Therefore if odd number of AF VFs are 403 * enabled then the last VF remains with no pair. 404 */ 405 pfvf->rx_chan_base = rvu_nix_chan_lbk(rvu, lbkid, vf); 406 pfvf->tx_chan_base = vf & 0x1 ? 407 rvu_nix_chan_lbk(rvu, lbkid, vf - 1) : 408 rvu_nix_chan_lbk(rvu, lbkid, vf + 1); 409 pfvf->rx_chan_cnt = 1; 410 pfvf->tx_chan_cnt = 1; 411 rsp->tx_link = hw->cgx_links + lbkid; 412 pfvf->lbkid = lbkid; 413 rvu_npc_set_pkind(rvu, NPC_RX_LBK_PKIND, pfvf); 414 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 415 pfvf->rx_chan_base, 416 pfvf->rx_chan_cnt); 417 418 break; 419 case NIX_INTF_TYPE_SDP: 420 from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 421 parent_pf = &rvu->pf[rvu_get_pf(rvu->pdev, pcifunc)]; 422 sdp_info = parent_pf->sdp_info; 423 if (!sdp_info) { 424 dev_err(rvu->dev, "Invalid sdp_info pointer\n"); 425 return -EINVAL; 426 } 427 if (from_vf) { 428 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn + 429 sdp_info->num_pf_rings; 430 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 431 for (vfid = 0; vfid < vf; vfid++) 432 req_chan_base += sdp_info->vf_rings[vfid]; 433 req_chan_cnt = sdp_info->vf_rings[vf]; 434 req_chan_end = req_chan_base + req_chan_cnt - 1; 435 if (req_chan_base < rvu_nix_chan_sdp(rvu, 0) || 436 req_chan_end > rvu_nix_chan_sdp(rvu, 255)) { 437 dev_err(rvu->dev, 438 "PF_Func 0x%x: Invalid channel base and count\n", 439 pcifunc); 440 return -EINVAL; 441 } 442 } else { 443 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn; 444 req_chan_cnt = sdp_info->num_pf_rings; 445 } 446 447 pfvf->rx_chan_base = req_chan_base; 448 pfvf->rx_chan_cnt = req_chan_cnt; 449 pfvf->tx_chan_base = pfvf->rx_chan_base; 450 pfvf->tx_chan_cnt = pfvf->rx_chan_cnt; 451 452 rsp->tx_link = hw->cgx_links + hw->lbk_links; 453 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 454 pfvf->rx_chan_base, 455 pfvf->rx_chan_cnt); 456 break; 457 } 458 459 /* Add a UCAST forwarding rule in MCAM with this NIXLF attached 460 * RVU PF/VF's MAC address. 461 */ 462 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 463 pfvf->rx_chan_base, pfvf->mac_addr); 464 465 /* Add this PF_FUNC to bcast pkt replication list */ 466 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true); 467 if (err) { 468 dev_err(rvu->dev, 469 "Bcast list, failed to enable PF_FUNC 0x%x\n", 470 pcifunc); 471 return err; 472 } 473 /* Install MCAM rule matching Ethernet broadcast mac address */ 474 rvu_npc_install_bcast_match_entry(rvu, pcifunc, 475 nixlf, pfvf->rx_chan_base); 476 477 pfvf->maxlen = NIC_HW_MIN_FRS; 478 pfvf->minlen = NIC_HW_MIN_FRS; 479 480 return 0; 481 } 482 483 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf) 484 { 485 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 486 int err; 487 488 pfvf->maxlen = 0; 489 pfvf->minlen = 0; 490 491 /* Remove this PF_FUNC from bcast pkt replication list */ 492 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false); 493 if (err) { 494 dev_err(rvu->dev, 495 "Bcast list, failed to disable PF_FUNC 0x%x\n", 496 pcifunc); 497 } 498 499 /* Free and disable any MCAM entries used by this NIX LF */ 500 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 501 502 /* Disable DMAC filters used */ 503 rvu_cgx_disable_dmac_entries(rvu, pcifunc); 504 } 505 506 #define NIX_BPIDS_PER_LMAC 8 507 #define NIX_BPIDS_PER_CPT 1 508 static int nix_setup_bpids(struct rvu *rvu, struct nix_hw *hw, int blkaddr) 509 { 510 struct nix_bp *bp = &hw->bp; 511 int err, max_bpids; 512 u64 cfg; 513 514 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 515 max_bpids = FIELD_GET(NIX_CONST_MAX_BPIDS, cfg); 516 517 /* Reserve the BPIds for CGX and SDP */ 518 bp->cgx_bpid_cnt = rvu->hw->cgx_links * NIX_BPIDS_PER_LMAC; 519 bp->sdp_bpid_cnt = rvu->hw->sdp_links * FIELD_GET(NIX_CONST_SDP_CHANS, cfg); 520 bp->free_pool_base = bp->cgx_bpid_cnt + bp->sdp_bpid_cnt + 521 NIX_BPIDS_PER_CPT; 522 bp->bpids.max = max_bpids - bp->free_pool_base; 523 524 err = rvu_alloc_bitmap(&bp->bpids); 525 if (err) 526 return err; 527 528 bp->fn_map = devm_kcalloc(rvu->dev, bp->bpids.max, 529 sizeof(u16), GFP_KERNEL); 530 if (!bp->fn_map) 531 goto free_bpids; 532 533 bp->intf_map = devm_kcalloc(rvu->dev, bp->bpids.max, 534 sizeof(u8), GFP_KERNEL); 535 if (!bp->intf_map) 536 goto free_bpids; 537 538 bp->ref_cnt = devm_kcalloc(rvu->dev, bp->bpids.max, 539 sizeof(u8), GFP_KERNEL); 540 if (!bp->ref_cnt) 541 goto free_bpids; 542 543 return 0; 544 545 free_bpids: 546 rvu_free_bitmap(&bp->bpids); 547 bp->bpids.bmap = NULL; 548 return -ENOMEM; 549 } 550 551 void rvu_nix_flr_free_bpids(struct rvu *rvu, u16 pcifunc) 552 { 553 int blkaddr, bpid, err; 554 struct nix_hw *nix_hw; 555 struct nix_bp *bp; 556 557 if (!is_lbk_vf(rvu, pcifunc)) 558 return; 559 560 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 561 if (err) 562 return; 563 564 bp = &nix_hw->bp; 565 566 mutex_lock(&rvu->rsrc_lock); 567 for (bpid = 0; bpid < bp->bpids.max; bpid++) { 568 if (bp->fn_map[bpid] == pcifunc) { 569 bp->ref_cnt[bpid]--; 570 if (bp->ref_cnt[bpid]) 571 continue; 572 rvu_free_rsrc(&bp->bpids, bpid); 573 bp->fn_map[bpid] = 0; 574 } 575 } 576 mutex_unlock(&rvu->rsrc_lock); 577 } 578 579 static u16 nix_get_channel(u16 chan, bool cpt_link) 580 { 581 /* CPT channel for a given link channel is always 582 * assumed to be BIT(11) set in link channel. 583 */ 584 return cpt_link ? chan | BIT(11) : chan; 585 } 586 587 static int nix_bp_disable(struct rvu *rvu, 588 struct nix_bp_cfg_req *req, 589 struct msg_rsp *rsp, bool cpt_link) 590 { 591 u16 pcifunc = req->hdr.pcifunc; 592 int blkaddr, pf, type, err; 593 u16 chan_base, chan, bpid; 594 struct rvu_pfvf *pfvf; 595 struct nix_hw *nix_hw; 596 struct nix_bp *bp; 597 u16 chan_v; 598 u64 cfg; 599 600 pf = rvu_get_pf(rvu->pdev, pcifunc); 601 type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 602 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK) 603 return 0; 604 605 if (is_sdp_pfvf(rvu, pcifunc)) 606 type = NIX_INTF_TYPE_SDP; 607 608 if (cpt_link && !rvu->hw->cpt_links) 609 return 0; 610 611 pfvf = rvu_get_pfvf(rvu, pcifunc); 612 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 613 if (err) 614 return err; 615 616 bp = &nix_hw->bp; 617 chan_base = pfvf->rx_chan_base + req->chan_base; 618 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 619 chan_v = nix_get_channel(chan, cpt_link); 620 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v)); 621 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v), 622 cfg & ~BIT_ULL(16)); 623 624 if (type == NIX_INTF_TYPE_LBK) { 625 bpid = cfg & GENMASK(8, 0); 626 mutex_lock(&rvu->rsrc_lock); 627 rvu_free_rsrc(&bp->bpids, bpid - bp->free_pool_base); 628 for (bpid = 0; bpid < bp->bpids.max; bpid++) { 629 if (bp->fn_map[bpid] == pcifunc) { 630 bp->fn_map[bpid] = 0; 631 bp->ref_cnt[bpid] = 0; 632 } 633 } 634 mutex_unlock(&rvu->rsrc_lock); 635 } 636 } 637 return 0; 638 } 639 640 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu, 641 struct nix_bp_cfg_req *req, 642 struct msg_rsp *rsp) 643 { 644 return nix_bp_disable(rvu, req, rsp, false); 645 } 646 647 int rvu_mbox_handler_nix_cpt_bp_disable(struct rvu *rvu, 648 struct nix_bp_cfg_req *req, 649 struct msg_rsp *rsp) 650 { 651 return nix_bp_disable(rvu, req, rsp, true); 652 } 653 654 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 655 int type, int chan_id) 656 { 657 int bpid, blkaddr, sdp_chan_base, err; 658 struct rvu_hwinfo *hw = rvu->hw; 659 struct rvu_pfvf *pfvf; 660 struct nix_hw *nix_hw; 661 u8 cgx_id, lmac_id; 662 struct nix_bp *bp; 663 664 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 665 666 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 667 if (err) 668 return err; 669 670 bp = &nix_hw->bp; 671 672 /* Backpressure IDs range division 673 * CGX channles are mapped to (0 - 191) BPIDs 674 * LBK channles are mapped to (192 - 255) BPIDs 675 * SDP channles are mapped to (256 - 511) BPIDs 676 * 677 * Lmac channles and bpids mapped as follows 678 * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15) 679 * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) .... 680 * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) .... 681 */ 682 switch (type) { 683 case NIX_INTF_TYPE_CGX: 684 if ((req->chan_base + req->chan_cnt) > NIX_BPIDS_PER_LMAC) 685 return NIX_AF_ERR_INVALID_BPID_REQ; 686 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 687 /* Assign bpid based on cgx, lmac and chan id */ 688 bpid = (cgx_id * hw->lmac_per_cgx * NIX_BPIDS_PER_LMAC) + 689 (lmac_id * NIX_BPIDS_PER_LMAC) + req->chan_base; 690 691 if (req->bpid_per_chan) 692 bpid += chan_id; 693 if (bpid > bp->cgx_bpid_cnt) 694 return NIX_AF_ERR_INVALID_BPID; 695 break; 696 697 case NIX_INTF_TYPE_LBK: 698 /* Alloc bpid from the free pool */ 699 mutex_lock(&rvu->rsrc_lock); 700 bpid = rvu_alloc_rsrc(&bp->bpids); 701 if (bpid < 0) { 702 mutex_unlock(&rvu->rsrc_lock); 703 return NIX_AF_ERR_INVALID_BPID; 704 } 705 bp->fn_map[bpid] = req->hdr.pcifunc; 706 bp->ref_cnt[bpid]++; 707 bpid += bp->free_pool_base; 708 mutex_unlock(&rvu->rsrc_lock); 709 break; 710 case NIX_INTF_TYPE_SDP: 711 if ((req->chan_base + req->chan_cnt) > bp->sdp_bpid_cnt) 712 return NIX_AF_ERR_INVALID_BPID_REQ; 713 714 /* Handle usecase of 2 SDP blocks */ 715 if (!hw->cap.programmable_chans) 716 sdp_chan_base = pfvf->rx_chan_base - NIX_CHAN_SDP_CH_START; 717 else 718 sdp_chan_base = pfvf->rx_chan_base - hw->sdp_chan_base; 719 720 bpid = bp->cgx_bpid_cnt + req->chan_base + sdp_chan_base; 721 if (req->bpid_per_chan) 722 bpid += chan_id; 723 724 if (bpid > (bp->cgx_bpid_cnt + bp->sdp_bpid_cnt)) 725 return NIX_AF_ERR_INVALID_BPID; 726 break; 727 default: 728 return -EINVAL; 729 } 730 return bpid; 731 } 732 733 static int nix_bp_enable(struct rvu *rvu, 734 struct nix_bp_cfg_req *req, 735 struct nix_bp_cfg_rsp *rsp, 736 bool cpt_link) 737 { 738 int blkaddr, pf, type, chan_id = 0; 739 u16 pcifunc = req->hdr.pcifunc; 740 struct rvu_pfvf *pfvf; 741 u16 chan_base, chan; 742 s16 bpid, bpid_base; 743 u16 chan_v; 744 u64 cfg; 745 746 pf = rvu_get_pf(rvu->pdev, pcifunc); 747 type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 748 if (is_sdp_pfvf(rvu, pcifunc)) 749 type = NIX_INTF_TYPE_SDP; 750 751 /* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */ 752 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 753 type != NIX_INTF_TYPE_SDP) 754 return 0; 755 756 if (cpt_link && !rvu->hw->cpt_links) 757 return 0; 758 759 pfvf = rvu_get_pfvf(rvu, pcifunc); 760 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 761 762 bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id); 763 chan_base = pfvf->rx_chan_base + req->chan_base; 764 bpid = bpid_base; 765 766 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 767 if (bpid < 0) { 768 dev_warn(rvu->dev, "Fail to enable backpressure\n"); 769 return -EINVAL; 770 } 771 772 chan_v = nix_get_channel(chan, cpt_link); 773 774 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v)); 775 cfg &= ~GENMASK_ULL(8, 0); 776 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v), 777 cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16)); 778 chan_id++; 779 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id); 780 } 781 782 for (chan = 0; chan < req->chan_cnt; chan++) { 783 /* Map channel and bpid assign to it */ 784 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 | 785 (bpid_base & 0x3FF); 786 if (req->bpid_per_chan) 787 bpid_base++; 788 } 789 rsp->chan_cnt = req->chan_cnt; 790 791 return 0; 792 } 793 794 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu, 795 struct nix_bp_cfg_req *req, 796 struct nix_bp_cfg_rsp *rsp) 797 { 798 return nix_bp_enable(rvu, req, rsp, false); 799 } 800 801 int rvu_mbox_handler_nix_cpt_bp_enable(struct rvu *rvu, 802 struct nix_bp_cfg_req *req, 803 struct nix_bp_cfg_rsp *rsp) 804 { 805 return nix_bp_enable(rvu, req, rsp, true); 806 } 807 808 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr, 809 u64 format, bool v4, u64 *fidx) 810 { 811 struct nix_lso_format field = {0}; 812 813 /* IP's Length field */ 814 field.layer = NIX_TXLAYER_OL3; 815 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ 816 field.offset = v4 ? 2 : 4; 817 field.sizem1 = 1; /* i.e 2 bytes */ 818 field.alg = NIX_LSOALG_ADD_PAYLEN; 819 rvu_write64(rvu, blkaddr, 820 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 821 *(u64 *)&field); 822 823 /* No ID field in IPv6 header */ 824 if (!v4) 825 return; 826 827 /* IP's ID field */ 828 field.layer = NIX_TXLAYER_OL3; 829 field.offset = 4; 830 field.sizem1 = 1; /* i.e 2 bytes */ 831 field.alg = NIX_LSOALG_ADD_SEGNUM; 832 rvu_write64(rvu, blkaddr, 833 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 834 *(u64 *)&field); 835 } 836 837 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr, 838 u64 format, u64 *fidx) 839 { 840 struct nix_lso_format field = {0}; 841 842 /* TCP's sequence number field */ 843 field.layer = NIX_TXLAYER_OL4; 844 field.offset = 4; 845 field.sizem1 = 3; /* i.e 4 bytes */ 846 field.alg = NIX_LSOALG_ADD_OFFSET; 847 rvu_write64(rvu, blkaddr, 848 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 849 *(u64 *)&field); 850 851 /* TCP's flags field */ 852 field.layer = NIX_TXLAYER_OL4; 853 field.offset = 12; 854 field.sizem1 = 1; /* 2 bytes */ 855 field.alg = NIX_LSOALG_TCP_FLAGS; 856 rvu_write64(rvu, blkaddr, 857 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 858 *(u64 *)&field); 859 } 860 861 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 862 { 863 u64 cfg, idx, fidx = 0; 864 865 /* Get max HW supported format indices */ 866 cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF; 867 nix_hw->lso.total = cfg; 868 869 /* Enable LSO */ 870 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG); 871 /* For TSO, set first and middle segment flags to 872 * mask out PSH, RST & FIN flags in TCP packet 873 */ 874 cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16)); 875 cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16); 876 rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63)); 877 878 /* Setup default static LSO formats 879 * 880 * Configure format fields for TCPv4 segmentation offload 881 */ 882 idx = NIX_LSO_FORMAT_IDX_TSOV4; 883 nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx); 884 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 885 886 /* Set rest of the fields to NOP */ 887 for (; fidx < 8; fidx++) { 888 rvu_write64(rvu, blkaddr, 889 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 890 } 891 nix_hw->lso.in_use++; 892 893 /* Configure format fields for TCPv6 segmentation offload */ 894 idx = NIX_LSO_FORMAT_IDX_TSOV6; 895 fidx = 0; 896 nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx); 897 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 898 899 /* Set rest of the fields to NOP */ 900 for (; fidx < 8; fidx++) { 901 rvu_write64(rvu, blkaddr, 902 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 903 } 904 nix_hw->lso.in_use++; 905 } 906 907 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) 908 { 909 kfree(pfvf->rq_bmap); 910 kfree(pfvf->sq_bmap); 911 kfree(pfvf->cq_bmap); 912 if (pfvf->rq_ctx) 913 qmem_free(rvu->dev, pfvf->rq_ctx); 914 if (pfvf->sq_ctx) 915 qmem_free(rvu->dev, pfvf->sq_ctx); 916 if (pfvf->cq_ctx) 917 qmem_free(rvu->dev, pfvf->cq_ctx); 918 if (pfvf->rss_ctx) 919 qmem_free(rvu->dev, pfvf->rss_ctx); 920 if (pfvf->nix_qints_ctx) 921 qmem_free(rvu->dev, pfvf->nix_qints_ctx); 922 if (pfvf->cq_ints_ctx) 923 qmem_free(rvu->dev, pfvf->cq_ints_ctx); 924 925 pfvf->rq_bmap = NULL; 926 pfvf->cq_bmap = NULL; 927 pfvf->sq_bmap = NULL; 928 pfvf->rq_ctx = NULL; 929 pfvf->sq_ctx = NULL; 930 pfvf->cq_ctx = NULL; 931 pfvf->rss_ctx = NULL; 932 pfvf->nix_qints_ctx = NULL; 933 pfvf->cq_ints_ctx = NULL; 934 } 935 936 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr, 937 struct rvu_pfvf *pfvf, int nixlf, 938 int rss_sz, int rss_grps, int hwctx_size, 939 u64 way_mask, bool tag_lsb_as_adder) 940 { 941 int err, grp, num_indices; 942 u64 val; 943 944 /* RSS is not requested for this NIXLF */ 945 if (!rss_sz) 946 return 0; 947 num_indices = rss_sz * rss_grps; 948 949 /* Alloc NIX RSS HW context memory and config the base */ 950 err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size); 951 if (err) 952 return err; 953 954 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf), 955 (u64)pfvf->rss_ctx->iova); 956 957 /* Config full RSS table size, enable RSS and caching */ 958 val = BIT_ULL(36) | BIT_ULL(4) | way_mask << 20 | 959 ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE); 960 961 if (tag_lsb_as_adder) 962 val |= BIT_ULL(5); 963 964 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf), val); 965 /* Config RSS group offset and sizes */ 966 for (grp = 0; grp < rss_grps; grp++) 967 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp), 968 ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp)); 969 return 0; 970 } 971 972 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, 973 struct nix_aq_inst_s *inst) 974 { 975 struct admin_queue *aq = block->aq; 976 struct nix_aq_res_s *result; 977 int timeout = 1000; 978 u64 reg, head; 979 int ret; 980 981 result = (struct nix_aq_res_s *)aq->res->base; 982 983 /* Get current head pointer where to append this instruction */ 984 reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS); 985 head = (reg >> 4) & AQ_PTR_MASK; 986 987 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), 988 (void *)inst, aq->inst->entry_sz); 989 memset(result, 0, sizeof(*result)); 990 /* sync into memory */ 991 wmb(); 992 993 /* Ring the doorbell and wait for result */ 994 rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1); 995 while (result->compcode == NIX_AQ_COMP_NOTDONE) { 996 cpu_relax(); 997 udelay(1); 998 timeout--; 999 if (!timeout) 1000 return -EBUSY; 1001 } 1002 1003 if (result->compcode != NIX_AQ_COMP_GOOD) { 1004 /* TODO: Replace this with some error code */ 1005 if (result->compcode == NIX_AQ_COMP_CTX_FAULT || 1006 result->compcode == NIX_AQ_COMP_LOCKERR || 1007 result->compcode == NIX_AQ_COMP_CTX_POISON) { 1008 ret = rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_RX); 1009 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_TX); 1010 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_RX); 1011 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_TX); 1012 if (ret) 1013 dev_err(rvu->dev, 1014 "%s: Not able to unlock cachelines\n", __func__); 1015 } 1016 1017 return -EBUSY; 1018 } 1019 1020 return 0; 1021 } 1022 1023 static void nix_get_aq_req_smq(struct rvu *rvu, struct nix_aq_enq_req *req, 1024 u16 *smq, u16 *smq_mask) 1025 { 1026 struct nix_cn10k_aq_enq_req *aq_req; 1027 1028 if (is_cn20k(rvu->pdev)) { 1029 *smq = ((struct nix_cn20k_aq_enq_req *)req)->sq.smq; 1030 *smq_mask = ((struct nix_cn20k_aq_enq_req *)req)->sq_mask.smq; 1031 return; 1032 } 1033 1034 if (!is_rvu_otx2(rvu)) { 1035 aq_req = (struct nix_cn10k_aq_enq_req *)req; 1036 *smq = aq_req->sq.smq; 1037 *smq_mask = aq_req->sq_mask.smq; 1038 } else { 1039 *smq = req->sq.smq; 1040 *smq_mask = req->sq_mask.smq; 1041 } 1042 } 1043 1044 static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, 1045 struct nix_aq_enq_req *req, 1046 struct nix_aq_enq_rsp *rsp) 1047 { 1048 struct rvu_hwinfo *hw = rvu->hw; 1049 u16 pcifunc = req->hdr.pcifunc; 1050 int nixlf, blkaddr, rc = 0; 1051 struct nix_aq_inst_s inst; 1052 struct rvu_block *block; 1053 struct admin_queue *aq; 1054 struct rvu_pfvf *pfvf; 1055 u16 smq, smq_mask; 1056 void *ctx, *mask; 1057 bool ena; 1058 u64 cfg; 1059 1060 blkaddr = nix_hw->blkaddr; 1061 block = &hw->block[blkaddr]; 1062 aq = block->aq; 1063 if (!aq) { 1064 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__); 1065 return NIX_AF_ERR_AQ_ENQUEUE; 1066 } 1067 1068 pfvf = rvu_get_pfvf(rvu, pcifunc); 1069 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1070 1071 /* Skip NIXLF check for broadcast MCE entry and bandwidth profile 1072 * operations done by AF itself. 1073 */ 1074 if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) || 1075 (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) { 1076 if (!pfvf->nixlf || nixlf < 0) 1077 return NIX_AF_ERR_AF_LF_INVALID; 1078 } 1079 1080 switch (req->ctype) { 1081 case NIX_AQ_CTYPE_RQ: 1082 /* Check if index exceeds max no of queues */ 1083 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize) 1084 rc = NIX_AF_ERR_AQ_ENQUEUE; 1085 break; 1086 case NIX_AQ_CTYPE_SQ: 1087 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize) 1088 rc = NIX_AF_ERR_AQ_ENQUEUE; 1089 break; 1090 case NIX_AQ_CTYPE_CQ: 1091 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize) 1092 rc = NIX_AF_ERR_AQ_ENQUEUE; 1093 break; 1094 case NIX_AQ_CTYPE_RSS: 1095 /* Check if RSS is enabled and qidx is within range */ 1096 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf)); 1097 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx || 1098 (req->qidx >= (256UL << (cfg & 0xF)))) 1099 rc = NIX_AF_ERR_AQ_ENQUEUE; 1100 break; 1101 case NIX_AQ_CTYPE_MCE: 1102 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG); 1103 1104 /* Check if index exceeds MCE list length */ 1105 if (!nix_hw->mcast.mce_ctx || 1106 (req->qidx >= (256UL << (cfg & 0xF)))) 1107 rc = NIX_AF_ERR_AQ_ENQUEUE; 1108 1109 /* Adding multicast lists for requests from PF/VFs is not 1110 * yet supported, so ignore this. 1111 */ 1112 if (rsp) 1113 rc = NIX_AF_ERR_AQ_ENQUEUE; 1114 break; 1115 case NIX_AQ_CTYPE_BANDPROF: 1116 if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req, 1117 nix_hw, pcifunc)) 1118 rc = NIX_AF_ERR_INVALID_BANDPROF; 1119 break; 1120 default: 1121 rc = NIX_AF_ERR_AQ_ENQUEUE; 1122 } 1123 1124 if (rc) 1125 return rc; 1126 1127 nix_get_aq_req_smq(rvu, req, &smq, &smq_mask); 1128 /* Check if SQ pointed SMQ belongs to this PF/VF or not */ 1129 if (req->ctype == NIX_AQ_CTYPE_SQ && 1130 ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) || 1131 (req->op == NIX_AQ_INSTOP_WRITE && 1132 req->sq_mask.ena && req->sq.ena && smq_mask))) { 1133 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ, 1134 pcifunc, smq)) 1135 return NIX_AF_ERR_AQ_ENQUEUE; 1136 } 1137 1138 memset(&inst, 0, sizeof(struct nix_aq_inst_s)); 1139 inst.lf = nixlf; 1140 inst.cindex = req->qidx; 1141 inst.ctype = req->ctype; 1142 inst.op = req->op; 1143 /* Currently we are not supporting enqueuing multiple instructions, 1144 * so always choose first entry in result memory. 1145 */ 1146 inst.res_addr = (u64)aq->res->iova; 1147 1148 /* Hardware uses same aq->res->base for updating result of 1149 * previous instruction hence wait here till it is done. 1150 */ 1151 spin_lock(&aq->lock); 1152 1153 /* Clean result + context memory */ 1154 memset(aq->res->base, 0, aq->res->entry_sz); 1155 /* Context needs to be written at RES_ADDR + 128 */ 1156 ctx = aq->res->base + 128; 1157 /* Mask needs to be written at RES_ADDR + 256 */ 1158 mask = aq->res->base + 256; 1159 1160 switch (req->op) { 1161 case NIX_AQ_INSTOP_WRITE: 1162 if (req->ctype == NIX_AQ_CTYPE_RQ) 1163 memcpy(mask, &req->rq_mask, 1164 NIX_MAX_CTX_SIZE); 1165 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1166 memcpy(mask, &req->sq_mask, 1167 NIX_MAX_CTX_SIZE); 1168 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1169 memcpy(mask, &req->cq_mask, 1170 NIX_MAX_CTX_SIZE); 1171 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1172 memcpy(mask, &req->rss_mask, 1173 NIX_MAX_CTX_SIZE); 1174 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1175 memcpy(mask, &req->mce_mask, 1176 NIX_MAX_CTX_SIZE); 1177 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1178 memcpy(mask, &req->prof_mask, 1179 NIX_MAX_CTX_SIZE); 1180 fallthrough; 1181 case NIX_AQ_INSTOP_INIT: 1182 if (req->ctype == NIX_AQ_CTYPE_RQ) 1183 memcpy(ctx, &req->rq, NIX_MAX_CTX_SIZE); 1184 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1185 memcpy(ctx, &req->sq, NIX_MAX_CTX_SIZE); 1186 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1187 memcpy(ctx, &req->cq, NIX_MAX_CTX_SIZE); 1188 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1189 memcpy(ctx, &req->rss, NIX_MAX_CTX_SIZE); 1190 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1191 memcpy(ctx, &req->mce, NIX_MAX_CTX_SIZE); 1192 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1193 memcpy(ctx, &req->prof, NIX_MAX_CTX_SIZE); 1194 break; 1195 case NIX_AQ_INSTOP_NOP: 1196 case NIX_AQ_INSTOP_READ: 1197 case NIX_AQ_INSTOP_LOCK: 1198 case NIX_AQ_INSTOP_UNLOCK: 1199 break; 1200 default: 1201 rc = NIX_AF_ERR_AQ_ENQUEUE; 1202 spin_unlock(&aq->lock); 1203 return rc; 1204 } 1205 1206 /* Submit the instruction to AQ */ 1207 rc = nix_aq_enqueue_wait(rvu, block, &inst); 1208 if (rc) { 1209 spin_unlock(&aq->lock); 1210 return rc; 1211 } 1212 1213 /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */ 1214 if (req->op == NIX_AQ_INSTOP_INIT) { 1215 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena) 1216 __set_bit(req->qidx, pfvf->rq_bmap); 1217 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena) 1218 __set_bit(req->qidx, pfvf->sq_bmap); 1219 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena) 1220 __set_bit(req->qidx, pfvf->cq_bmap); 1221 } 1222 1223 if (req->op == NIX_AQ_INSTOP_WRITE) { 1224 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1225 ena = (req->rq.ena & req->rq_mask.ena) | 1226 (test_bit(req->qidx, pfvf->rq_bmap) & 1227 ~req->rq_mask.ena); 1228 if (ena) 1229 __set_bit(req->qidx, pfvf->rq_bmap); 1230 else 1231 __clear_bit(req->qidx, pfvf->rq_bmap); 1232 } 1233 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1234 ena = (req->rq.ena & req->sq_mask.ena) | 1235 (test_bit(req->qidx, pfvf->sq_bmap) & 1236 ~req->sq_mask.ena); 1237 if (ena) 1238 __set_bit(req->qidx, pfvf->sq_bmap); 1239 else 1240 __clear_bit(req->qidx, pfvf->sq_bmap); 1241 } 1242 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1243 ena = (req->rq.ena & req->cq_mask.ena) | 1244 (test_bit(req->qidx, pfvf->cq_bmap) & 1245 ~req->cq_mask.ena); 1246 if (ena) 1247 __set_bit(req->qidx, pfvf->cq_bmap); 1248 else 1249 __clear_bit(req->qidx, pfvf->cq_bmap); 1250 } 1251 } 1252 1253 if (rsp) { 1254 /* Copy read context into mailbox */ 1255 if (req->op == NIX_AQ_INSTOP_READ) { 1256 if (req->ctype == NIX_AQ_CTYPE_RQ) 1257 memcpy(&rsp->rq, ctx, 1258 NIX_MAX_CTX_SIZE); 1259 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1260 memcpy(&rsp->sq, ctx, 1261 NIX_MAX_CTX_SIZE); 1262 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1263 memcpy(&rsp->cq, ctx, 1264 NIX_MAX_CTX_SIZE); 1265 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1266 memcpy(&rsp->rss, ctx, 1267 NIX_MAX_CTX_SIZE); 1268 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1269 memcpy(&rsp->mce, ctx, 1270 NIX_MAX_CTX_SIZE); 1271 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1272 memcpy(&rsp->prof, ctx, 1273 NIX_MAX_CTX_SIZE); 1274 } 1275 } 1276 1277 spin_unlock(&aq->lock); 1278 return 0; 1279 } 1280 1281 static int rvu_nix_verify_aq_ctx(struct rvu *rvu, struct nix_hw *nix_hw, 1282 struct nix_aq_enq_req *req, u8 ctype) 1283 { 1284 struct nix_cn10k_aq_enq_req aq_req; 1285 struct nix_cn10k_aq_enq_rsp aq_rsp; 1286 int rc, word; 1287 1288 if (req->ctype != NIX_AQ_CTYPE_CQ) 1289 return 0; 1290 1291 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 1292 req->hdr.pcifunc, ctype, req->qidx); 1293 if (rc) { 1294 dev_err(rvu->dev, 1295 "%s: Failed to fetch %s%d context of PFFUNC 0x%x\n", 1296 __func__, nix_get_ctx_name(ctype), req->qidx, 1297 req->hdr.pcifunc); 1298 return rc; 1299 } 1300 1301 /* Make copy of original context & mask which are required 1302 * for resubmission 1303 */ 1304 memcpy(&aq_req.cq_mask, &req->cq_mask, NIX_MAX_CTX_SIZE); 1305 memcpy(&aq_req.cq, &req->cq, NIX_MAX_CTX_SIZE); 1306 1307 /* exclude fields which HW can update */ 1308 aq_req.cq_mask.cq_err = 0; 1309 aq_req.cq_mask.wrptr = 0; 1310 aq_req.cq_mask.tail = 0; 1311 aq_req.cq_mask.head = 0; 1312 aq_req.cq_mask.avg_level = 0; 1313 aq_req.cq_mask.update_time = 0; 1314 aq_req.cq_mask.substream = 0; 1315 1316 /* Context mask (cq_mask) holds mask value of fields which 1317 * are changed in AQ WRITE operation. 1318 * for example cq.drop = 0xa; 1319 * cq_mask.drop = 0xff; 1320 * Below logic performs '&' between cq and cq_mask so that non 1321 * updated fields are masked out for request and response 1322 * comparison 1323 */ 1324 for (word = 0; word < NIX_MAX_CTX_SIZE / sizeof(u64); 1325 word++) { 1326 *(u64 *)((u8 *)&aq_rsp.cq + word * 8) &= 1327 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1328 *(u64 *)((u8 *)&aq_req.cq + word * 8) &= 1329 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1330 } 1331 1332 if (memcmp(&aq_req.cq, &aq_rsp.cq, NIX_MAX_CTX_SIZE)) 1333 return NIX_AF_ERR_AQ_CTX_RETRY_WRITE; 1334 1335 return 0; 1336 } 1337 1338 int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req, 1339 struct nix_aq_enq_rsp *rsp) 1340 { 1341 struct nix_hw *nix_hw; 1342 int err, retries = 5; 1343 int blkaddr; 1344 1345 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc); 1346 if (blkaddr < 0) 1347 return NIX_AF_ERR_AF_LF_INVALID; 1348 1349 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1350 if (!nix_hw) 1351 return NIX_AF_ERR_INVALID_NIXBLK; 1352 1353 retry: 1354 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp); 1355 1356 /* HW errata 'AQ Modification to CQ could be discarded on heavy traffic' 1357 * As a work around perfrom CQ context read after each AQ write. If AQ 1358 * read shows AQ write is not updated perform AQ write again. 1359 */ 1360 if (!err && req->op == NIX_AQ_INSTOP_WRITE) { 1361 err = rvu_nix_verify_aq_ctx(rvu, nix_hw, req, NIX_AQ_CTYPE_CQ); 1362 if (err == NIX_AF_ERR_AQ_CTX_RETRY_WRITE) { 1363 if (retries--) 1364 goto retry; 1365 else 1366 return NIX_AF_ERR_CQ_CTX_WRITE_ERR; 1367 } 1368 } 1369 1370 return err; 1371 } 1372 1373 static const char *nix_get_ctx_name(int ctype) 1374 { 1375 switch (ctype) { 1376 case NIX_AQ_CTYPE_CQ: 1377 return "CQ"; 1378 case NIX_AQ_CTYPE_SQ: 1379 return "SQ"; 1380 case NIX_AQ_CTYPE_RQ: 1381 return "RQ"; 1382 case NIX_AQ_CTYPE_RSS: 1383 return "RSS"; 1384 } 1385 return ""; 1386 } 1387 1388 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) 1389 { 1390 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1391 struct nix_aq_enq_req aq_req; 1392 unsigned long *bmap; 1393 int qidx, q_cnt = 0; 1394 int err = 0, rc; 1395 1396 if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx) 1397 return NIX_AF_ERR_AQ_ENQUEUE; 1398 1399 memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); 1400 aq_req.hdr.pcifunc = req->hdr.pcifunc; 1401 1402 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1403 aq_req.cq.ena = 0; 1404 aq_req.cq_mask.ena = 1; 1405 aq_req.cq.bp_ena = 0; 1406 aq_req.cq_mask.bp_ena = 1; 1407 q_cnt = pfvf->cq_ctx->qsize; 1408 bmap = pfvf->cq_bmap; 1409 } 1410 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1411 aq_req.sq.ena = 0; 1412 aq_req.sq_mask.ena = 1; 1413 q_cnt = pfvf->sq_ctx->qsize; 1414 bmap = pfvf->sq_bmap; 1415 } 1416 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1417 aq_req.rq.ena = 0; 1418 aq_req.rq_mask.ena = 1; 1419 q_cnt = pfvf->rq_ctx->qsize; 1420 bmap = pfvf->rq_bmap; 1421 } 1422 1423 aq_req.ctype = req->ctype; 1424 aq_req.op = NIX_AQ_INSTOP_WRITE; 1425 1426 for (qidx = 0; qidx < q_cnt; qidx++) { 1427 if (!test_bit(qidx, bmap)) 1428 continue; 1429 aq_req.qidx = qidx; 1430 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL); 1431 if (rc) { 1432 err = rc; 1433 dev_err(rvu->dev, "Failed to disable %s:%d context\n", 1434 nix_get_ctx_name(req->ctype), qidx); 1435 } 1436 } 1437 1438 return err; 1439 } 1440 1441 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 1442 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req) 1443 { 1444 struct nix_aq_enq_req lock_ctx_req; 1445 int err; 1446 1447 if (req->op != NIX_AQ_INSTOP_INIT) 1448 return 0; 1449 1450 if (req->ctype == NIX_AQ_CTYPE_MCE || 1451 req->ctype == NIX_AQ_CTYPE_DYNO) 1452 return 0; 1453 1454 memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req)); 1455 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; 1456 lock_ctx_req.ctype = req->ctype; 1457 lock_ctx_req.op = NIX_AQ_INSTOP_LOCK; 1458 lock_ctx_req.qidx = req->qidx; 1459 err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL); 1460 if (err) 1461 dev_err(rvu->dev, 1462 "PFUNC 0x%x: Failed to lock NIX %s:%d context\n", 1463 req->hdr.pcifunc, 1464 nix_get_ctx_name(req->ctype), req->qidx); 1465 return err; 1466 } 1467 1468 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1469 struct nix_aq_enq_req *req, 1470 struct nix_aq_enq_rsp *rsp) 1471 { 1472 int err; 1473 1474 err = rvu_nix_aq_enq_inst(rvu, req, rsp); 1475 if (!err) 1476 err = nix_lf_hwctx_lockdown(rvu, req); 1477 return err; 1478 } 1479 #else 1480 1481 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1482 struct nix_aq_enq_req *req, 1483 struct nix_aq_enq_rsp *rsp) 1484 { 1485 return rvu_nix_aq_enq_inst(rvu, req, rsp); 1486 } 1487 #endif 1488 /* CN10K mbox handler */ 1489 int rvu_mbox_handler_nix_cn10k_aq_enq(struct rvu *rvu, 1490 struct nix_cn10k_aq_enq_req *req, 1491 struct nix_cn10k_aq_enq_rsp *rsp) 1492 { 1493 return rvu_nix_aq_enq_inst(rvu, (struct nix_aq_enq_req *)req, 1494 (struct nix_aq_enq_rsp *)rsp); 1495 } 1496 1497 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu, 1498 struct hwctx_disable_req *req, 1499 struct msg_rsp *rsp) 1500 { 1501 return nix_lf_hwctx_disable(rvu, req); 1502 } 1503 1504 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu, 1505 struct nix_lf_alloc_req *req, 1506 struct nix_lf_alloc_rsp *rsp) 1507 { 1508 int nixlf, qints, hwctx_size, intf, rc = 0; 1509 u16 bcast, mcast, promisc, ucast; 1510 struct rvu_hwinfo *hw = rvu->hw; 1511 u16 pcifunc = req->hdr.pcifunc; 1512 bool rules_created = false; 1513 struct rvu_block *block; 1514 struct rvu_pfvf *pfvf; 1515 u64 cfg, ctx_cfg; 1516 int blkaddr; 1517 1518 if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt) 1519 return NIX_AF_ERR_PARAM; 1520 1521 if (req->way_mask) 1522 req->way_mask &= 0xFFFF; 1523 1524 pfvf = rvu_get_pfvf(rvu, pcifunc); 1525 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1526 if (!pfvf->nixlf || blkaddr < 0) 1527 return NIX_AF_ERR_AF_LF_INVALID; 1528 1529 block = &hw->block[blkaddr]; 1530 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1531 if (nixlf < 0) 1532 return NIX_AF_ERR_AF_LF_INVALID; 1533 1534 /* Check if requested 'NIXLF <=> NPALF' mapping is valid */ 1535 if (req->npa_func) { 1536 /* If default, use 'this' NIXLF's PFFUNC */ 1537 if (req->npa_func == RVU_DEFAULT_PF_FUNC) 1538 req->npa_func = pcifunc; 1539 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA)) 1540 return NIX_AF_INVAL_NPA_PF_FUNC; 1541 } 1542 1543 /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */ 1544 if (req->sso_func) { 1545 /* If default, use 'this' NIXLF's PFFUNC */ 1546 if (req->sso_func == RVU_DEFAULT_PF_FUNC) 1547 req->sso_func = pcifunc; 1548 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO)) 1549 return NIX_AF_INVAL_SSO_PF_FUNC; 1550 } 1551 1552 /* If RSS is being enabled, check if requested config is valid. 1553 * RSS table size should be power of two, otherwise 1554 * RSS_GRP::OFFSET + adder might go beyond that group or 1555 * won't be able to use entire table. 1556 */ 1557 if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE || 1558 !is_power_of_2(req->rss_sz))) 1559 return NIX_AF_ERR_RSS_SIZE_INVALID; 1560 1561 if (req->rss_sz && 1562 (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS)) 1563 return NIX_AF_ERR_RSS_GRPS_INVALID; 1564 1565 /* Reset this NIX LF */ 1566 rc = rvu_lf_reset(rvu, block, nixlf); 1567 if (rc) { 1568 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1569 block->addr - BLKADDR_NIX0, nixlf); 1570 return NIX_AF_ERR_LF_RESET; 1571 } 1572 1573 ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3); 1574 1575 /* Alloc NIX RQ HW context memory and config the base */ 1576 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); 1577 rc = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size); 1578 if (rc) 1579 goto free_mem; 1580 1581 pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL); 1582 if (!pfvf->rq_bmap) { 1583 rc = -ENOMEM; 1584 goto free_mem; 1585 } 1586 1587 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf), 1588 (u64)pfvf->rq_ctx->iova); 1589 1590 /* Set caching and queue count in HW */ 1591 cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20; 1592 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg); 1593 1594 /* Alloc NIX SQ HW context memory and config the base */ 1595 hwctx_size = 1UL << (ctx_cfg & 0xF); 1596 rc = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size); 1597 if (rc) 1598 goto free_mem; 1599 1600 pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL); 1601 if (!pfvf->sq_bmap) { 1602 rc = -ENOMEM; 1603 goto free_mem; 1604 } 1605 1606 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf), 1607 (u64)pfvf->sq_ctx->iova); 1608 1609 cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20; 1610 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg); 1611 1612 /* Alloc NIX CQ HW context memory and config the base */ 1613 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); 1614 rc = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size); 1615 if (rc) 1616 goto free_mem; 1617 1618 pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL); 1619 if (!pfvf->cq_bmap) { 1620 rc = -ENOMEM; 1621 goto free_mem; 1622 } 1623 1624 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf), 1625 (u64)pfvf->cq_ctx->iova); 1626 1627 cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20; 1628 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg); 1629 1630 /* Initialize receive side scaling (RSS) */ 1631 hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF); 1632 rc = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz, 1633 req->rss_grps, hwctx_size, req->way_mask, 1634 !!(req->flags & NIX_LF_RSS_TAG_LSB_AS_ADDER)); 1635 if (rc) 1636 goto free_mem; 1637 1638 /* Alloc memory for CQINT's HW contexts */ 1639 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1640 qints = (cfg >> 24) & 0xFFF; 1641 hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF); 1642 rc = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size); 1643 if (rc) 1644 goto free_mem; 1645 1646 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf), 1647 (u64)pfvf->cq_ints_ctx->iova); 1648 1649 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf), 1650 BIT_ULL(36) | req->way_mask << 20); 1651 1652 /* Alloc memory for QINT's HW contexts */ 1653 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1654 qints = (cfg >> 12) & 0xFFF; 1655 hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF); 1656 rc = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size); 1657 if (rc) 1658 goto free_mem; 1659 1660 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf), 1661 (u64)pfvf->nix_qints_ctx->iova); 1662 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf), 1663 BIT_ULL(36) | req->way_mask << 20); 1664 1665 /* Setup VLANX TPID's. 1666 * Use VLAN1 for 802.1Q 1667 * and VLAN0 for 802.1AD. 1668 */ 1669 cfg = (0x8100ULL << 16) | 0x88A8ULL; 1670 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 1671 1672 /* Enable LMTST for this NIX LF */ 1673 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0)); 1674 1675 /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */ 1676 if (req->npa_func) 1677 cfg = req->npa_func; 1678 if (req->sso_func) 1679 cfg |= (u64)req->sso_func << 16; 1680 1681 cfg |= (u64)req->xqe_sz << 33; 1682 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg); 1683 1684 /* Config Rx pkt length, csum checks and apad enable / disable */ 1685 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg); 1686 1687 /* Configure pkind for TX parse config */ 1688 cfg = NPC_TX_DEF_PKIND; 1689 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg); 1690 1691 if (is_rep_dev(rvu, pcifunc)) { 1692 pfvf->tx_chan_base = RVU_SWITCH_LBK_CHAN; 1693 pfvf->tx_chan_cnt = 1; 1694 goto exit; 1695 } 1696 1697 intf = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 1698 if (is_sdp_pfvf(rvu, pcifunc)) 1699 intf = NIX_INTF_TYPE_SDP; 1700 1701 if (is_cn20k(rvu->pdev)) { 1702 rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, 1703 &promisc, &ucast); 1704 if (rc) { 1705 rc = npc_cn20k_dft_rules_alloc(rvu, pcifunc); 1706 if (rc) 1707 goto free_mem; 1708 1709 rules_created = true; 1710 } 1711 } 1712 1713 rc = nix_interface_init(rvu, pcifunc, intf, nixlf, rsp, 1714 !!(req->flags & NIX_LF_LBK_BLK_SEL)); 1715 if (rc) 1716 goto free_dft; 1717 1718 /* Disable NPC entries as NIXLF's contexts are not initialized yet */ 1719 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1720 1721 /* Configure RX VTAG Type 7 (strip) for vf vlan */ 1722 rvu_write64(rvu, blkaddr, 1723 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7), 1724 VTAGSIZE_T4 | VTAG_STRIP); 1725 1726 goto exit; 1727 1728 free_dft: 1729 if (is_cn20k(rvu->pdev) && rules_created) 1730 npc_cn20k_dft_rules_free(rvu, pcifunc); 1731 1732 free_mem: 1733 nix_ctx_free(rvu, pfvf); 1734 1735 exit: 1736 /* Set macaddr of this PF/VF */ 1737 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 1738 1739 /* set SQB size info */ 1740 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST); 1741 rsp->sqb_size = (cfg >> 34) & 0xFFFF; 1742 rsp->rx_chan_base = pfvf->rx_chan_base; 1743 rsp->tx_chan_base = pfvf->tx_chan_base; 1744 rsp->rx_chan_cnt = pfvf->rx_chan_cnt; 1745 rsp->tx_chan_cnt = pfvf->tx_chan_cnt; 1746 rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4; 1747 rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6; 1748 /* Get HW supported stat count */ 1749 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 1750 rsp->lf_rx_stats = ((cfg >> 32) & 0xFF); 1751 rsp->lf_tx_stats = ((cfg >> 24) & 0xFF); 1752 /* Get count of CQ IRQs and error IRQs supported per LF */ 1753 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1754 rsp->qints = ((cfg >> 12) & 0xFFF); 1755 rsp->cints = ((cfg >> 24) & 0xFFF); 1756 rsp->cgx_links = hw->cgx_links; 1757 rsp->lbk_links = hw->lbk_links; 1758 rsp->sdp_links = hw->sdp_links; 1759 1760 return rc; 1761 } 1762 1763 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req, 1764 struct msg_rsp *rsp) 1765 { 1766 struct rvu_hwinfo *hw = rvu->hw; 1767 u16 pcifunc = req->hdr.pcifunc; 1768 struct rvu_block *block; 1769 int blkaddr, nixlf, err; 1770 struct rvu_pfvf *pfvf; 1771 1772 pfvf = rvu_get_pfvf(rvu, pcifunc); 1773 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1774 if (!pfvf->nixlf || blkaddr < 0) 1775 return NIX_AF_ERR_AF_LF_INVALID; 1776 1777 block = &hw->block[blkaddr]; 1778 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1779 if (nixlf < 0) 1780 return NIX_AF_ERR_AF_LF_INVALID; 1781 1782 if (is_rep_dev(rvu, pcifunc)) 1783 goto free_lf; 1784 1785 if (req->flags & NIX_LF_DISABLE_FLOWS) 1786 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 1787 else 1788 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 1789 1790 /* Free any tx vtag def entries used by this NIX LF */ 1791 if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG)) 1792 nix_free_tx_vtag_entries(rvu, pcifunc); 1793 1794 nix_interface_deinit(rvu, pcifunc, nixlf); 1795 1796 free_lf: 1797 /* Reset this NIX LF */ 1798 err = rvu_lf_reset(rvu, block, nixlf); 1799 if (err) { 1800 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1801 block->addr - BLKADDR_NIX0, nixlf); 1802 return NIX_AF_ERR_LF_RESET; 1803 } 1804 1805 nix_ctx_free(rvu, pfvf); 1806 1807 if (is_cn20k(rvu->pdev) && !(req->flags & NIX_LF_DONT_FREE_DFT_IDXS)) 1808 npc_cn20k_dft_rules_free(rvu, pcifunc); 1809 1810 return 0; 1811 } 1812 1813 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu, 1814 struct nix_mark_format_cfg *req, 1815 struct nix_mark_format_cfg_rsp *rsp) 1816 { 1817 u16 pcifunc = req->hdr.pcifunc; 1818 struct nix_hw *nix_hw; 1819 struct rvu_pfvf *pfvf; 1820 int blkaddr, rc; 1821 u32 cfg; 1822 1823 pfvf = rvu_get_pfvf(rvu, pcifunc); 1824 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1825 if (!pfvf->nixlf || blkaddr < 0) 1826 return NIX_AF_ERR_AF_LF_INVALID; 1827 1828 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1829 if (!nix_hw) 1830 return NIX_AF_ERR_INVALID_NIXBLK; 1831 1832 cfg = (((u32)req->offset & 0x7) << 16) | 1833 (((u32)req->y_mask & 0xF) << 12) | 1834 (((u32)req->y_val & 0xF) << 8) | 1835 (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF); 1836 1837 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg); 1838 if (rc < 0) { 1839 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)", 1840 rvu_get_pf(rvu->pdev, pcifunc), 1841 pcifunc & RVU_PFVF_FUNC_MASK); 1842 return NIX_AF_ERR_MARK_CFG_FAIL; 1843 } 1844 1845 rsp->mark_format_idx = rc; 1846 return 0; 1847 } 1848 1849 /* Handle shaper update specially for few revisions */ 1850 static bool 1851 handle_txschq_shaper_update(struct rvu *rvu, int blkaddr, int nixlf, 1852 int lvl, u64 reg, u64 regval) 1853 { 1854 u64 regbase, oldval, sw_xoff = 0; 1855 u64 dbgval, md_debug0 = 0; 1856 unsigned long poll_tmo; 1857 bool rate_reg = 0; 1858 u32 schq; 1859 1860 regbase = reg & 0xFFFF; 1861 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 1862 1863 /* Check for rate register */ 1864 switch (lvl) { 1865 case NIX_TXSCH_LVL_TL1: 1866 md_debug0 = NIX_AF_TL1X_MD_DEBUG0(schq); 1867 sw_xoff = NIX_AF_TL1X_SW_XOFF(schq); 1868 1869 rate_reg = !!(regbase == NIX_AF_TL1X_CIR(0)); 1870 break; 1871 case NIX_TXSCH_LVL_TL2: 1872 md_debug0 = NIX_AF_TL2X_MD_DEBUG0(schq); 1873 sw_xoff = NIX_AF_TL2X_SW_XOFF(schq); 1874 1875 rate_reg = (regbase == NIX_AF_TL2X_CIR(0) || 1876 regbase == NIX_AF_TL2X_PIR(0)); 1877 break; 1878 case NIX_TXSCH_LVL_TL3: 1879 md_debug0 = NIX_AF_TL3X_MD_DEBUG0(schq); 1880 sw_xoff = NIX_AF_TL3X_SW_XOFF(schq); 1881 1882 rate_reg = (regbase == NIX_AF_TL3X_CIR(0) || 1883 regbase == NIX_AF_TL3X_PIR(0)); 1884 break; 1885 case NIX_TXSCH_LVL_TL4: 1886 md_debug0 = NIX_AF_TL4X_MD_DEBUG0(schq); 1887 sw_xoff = NIX_AF_TL4X_SW_XOFF(schq); 1888 1889 rate_reg = (regbase == NIX_AF_TL4X_CIR(0) || 1890 regbase == NIX_AF_TL4X_PIR(0)); 1891 break; 1892 case NIX_TXSCH_LVL_MDQ: 1893 sw_xoff = NIX_AF_MDQX_SW_XOFF(schq); 1894 rate_reg = (regbase == NIX_AF_MDQX_CIR(0) || 1895 regbase == NIX_AF_MDQX_PIR(0)); 1896 break; 1897 } 1898 1899 if (!rate_reg) 1900 return false; 1901 1902 /* Nothing special to do when state is not toggled */ 1903 oldval = rvu_read64(rvu, blkaddr, reg); 1904 if ((oldval & 0x1) == (regval & 0x1)) { 1905 rvu_write64(rvu, blkaddr, reg, regval); 1906 return true; 1907 } 1908 1909 /* PIR/CIR disable */ 1910 if (!(regval & 0x1)) { 1911 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1912 rvu_write64(rvu, blkaddr, reg, 0); 1913 udelay(4); 1914 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1915 return true; 1916 } 1917 1918 /* PIR/CIR enable */ 1919 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1920 if (md_debug0) { 1921 poll_tmo = jiffies + usecs_to_jiffies(10000); 1922 /* Wait until VLD(bit32) == 1 or C_CON(bit48) == 0 */ 1923 do { 1924 if (time_after(jiffies, poll_tmo)) { 1925 dev_err(rvu->dev, 1926 "NIXLF%d: TLX%u(lvl %u) CIR/PIR enable failed\n", 1927 nixlf, schq, lvl); 1928 goto exit; 1929 } 1930 usleep_range(1, 5); 1931 dbgval = rvu_read64(rvu, blkaddr, md_debug0); 1932 } while (!(dbgval & BIT_ULL(32)) && (dbgval & BIT_ULL(48))); 1933 } 1934 rvu_write64(rvu, blkaddr, reg, regval); 1935 exit: 1936 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1937 return true; 1938 } 1939 1940 static void nix_reset_tx_schedule(struct rvu *rvu, int blkaddr, 1941 int lvl, int schq) 1942 { 1943 u64 tlx_parent = 0, tlx_schedule = 0; 1944 1945 switch (lvl) { 1946 case NIX_TXSCH_LVL_TL2: 1947 tlx_parent = NIX_AF_TL2X_PARENT(schq); 1948 tlx_schedule = NIX_AF_TL2X_SCHEDULE(schq); 1949 break; 1950 case NIX_TXSCH_LVL_TL3: 1951 tlx_parent = NIX_AF_TL3X_PARENT(schq); 1952 tlx_schedule = NIX_AF_TL3X_SCHEDULE(schq); 1953 break; 1954 case NIX_TXSCH_LVL_TL4: 1955 tlx_parent = NIX_AF_TL4X_PARENT(schq); 1956 tlx_schedule = NIX_AF_TL4X_SCHEDULE(schq); 1957 break; 1958 case NIX_TXSCH_LVL_MDQ: 1959 /* no need to reset SMQ_CFG as HW clears this CSR 1960 * on SMQ flush 1961 */ 1962 tlx_parent = NIX_AF_MDQX_PARENT(schq); 1963 tlx_schedule = NIX_AF_MDQX_SCHEDULE(schq); 1964 break; 1965 default: 1966 return; 1967 } 1968 1969 if (tlx_parent) 1970 rvu_write64(rvu, blkaddr, tlx_parent, 0x0); 1971 1972 if (tlx_schedule) 1973 rvu_write64(rvu, blkaddr, tlx_schedule, 0x0); 1974 } 1975 1976 /* Disable shaping of pkts by a scheduler queue 1977 * at a given scheduler level. 1978 */ 1979 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr, 1980 int nixlf, int lvl, int schq) 1981 { 1982 struct rvu_hwinfo *hw = rvu->hw; 1983 u64 cir_reg = 0, pir_reg = 0; 1984 u64 cfg; 1985 1986 switch (lvl) { 1987 case NIX_TXSCH_LVL_TL1: 1988 cir_reg = NIX_AF_TL1X_CIR(schq); 1989 pir_reg = 0; /* PIR not available at TL1 */ 1990 break; 1991 case NIX_TXSCH_LVL_TL2: 1992 cir_reg = NIX_AF_TL2X_CIR(schq); 1993 pir_reg = NIX_AF_TL2X_PIR(schq); 1994 break; 1995 case NIX_TXSCH_LVL_TL3: 1996 cir_reg = NIX_AF_TL3X_CIR(schq); 1997 pir_reg = NIX_AF_TL3X_PIR(schq); 1998 break; 1999 case NIX_TXSCH_LVL_TL4: 2000 cir_reg = NIX_AF_TL4X_CIR(schq); 2001 pir_reg = NIX_AF_TL4X_PIR(schq); 2002 break; 2003 case NIX_TXSCH_LVL_MDQ: 2004 cir_reg = NIX_AF_MDQX_CIR(schq); 2005 pir_reg = NIX_AF_MDQX_PIR(schq); 2006 break; 2007 } 2008 2009 /* Shaper state toggle needs wait/poll */ 2010 if (hw->cap.nix_shaper_toggle_wait) { 2011 if (cir_reg) 2012 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2013 lvl, cir_reg, 0); 2014 if (pir_reg) 2015 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2016 lvl, pir_reg, 0); 2017 return; 2018 } 2019 2020 if (!cir_reg) 2021 return; 2022 cfg = rvu_read64(rvu, blkaddr, cir_reg); 2023 rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0)); 2024 2025 if (!pir_reg) 2026 return; 2027 cfg = rvu_read64(rvu, blkaddr, pir_reg); 2028 rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0)); 2029 } 2030 2031 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr, 2032 int lvl, int schq) 2033 { 2034 struct rvu_hwinfo *hw = rvu->hw; 2035 int link_level; 2036 int link; 2037 2038 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2039 return; 2040 2041 /* Reset TL4's SDP link config */ 2042 if (lvl == NIX_TXSCH_LVL_TL4) 2043 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00); 2044 2045 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2046 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2047 if (lvl != link_level) 2048 return; 2049 2050 /* Reset TL2's CGX or LBK link config */ 2051 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++) 2052 rvu_write64(rvu, blkaddr, 2053 NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00); 2054 } 2055 2056 static void nix_clear_tx_xoff(struct rvu *rvu, int blkaddr, 2057 int lvl, int schq) 2058 { 2059 struct rvu_hwinfo *hw = rvu->hw; 2060 u64 reg; 2061 2062 /* Skip this if shaping is not supported */ 2063 if (!hw->cap.nix_shaping) 2064 return; 2065 2066 /* Clear level specific SW_XOFF */ 2067 switch (lvl) { 2068 case NIX_TXSCH_LVL_TL1: 2069 reg = NIX_AF_TL1X_SW_XOFF(schq); 2070 break; 2071 case NIX_TXSCH_LVL_TL2: 2072 reg = NIX_AF_TL2X_SW_XOFF(schq); 2073 break; 2074 case NIX_TXSCH_LVL_TL3: 2075 reg = NIX_AF_TL3X_SW_XOFF(schq); 2076 break; 2077 case NIX_TXSCH_LVL_TL4: 2078 reg = NIX_AF_TL4X_SW_XOFF(schq); 2079 break; 2080 case NIX_TXSCH_LVL_MDQ: 2081 reg = NIX_AF_MDQX_SW_XOFF(schq); 2082 break; 2083 default: 2084 return; 2085 } 2086 2087 rvu_write64(rvu, blkaddr, reg, 0x0); 2088 } 2089 2090 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc) 2091 { 2092 struct rvu_hwinfo *hw = rvu->hw; 2093 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2094 u8 cgx_id = 0, lmac_id = 0; 2095 2096 if (is_lbk_vf(rvu, pcifunc)) {/* LBK links */ 2097 return hw->cgx_links; 2098 } else if (is_pf_cgxmapped(rvu, pf)) { 2099 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 2100 return (cgx_id * hw->lmac_per_cgx) + lmac_id; 2101 } 2102 2103 /* SDP link */ 2104 return hw->cgx_links + hw->lbk_links; 2105 } 2106 2107 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc, 2108 int link, int *start, int *end) 2109 { 2110 struct rvu_hwinfo *hw = rvu->hw; 2111 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2112 2113 /* LBK links */ 2114 if (is_lbk_vf(rvu, pcifunc) || is_rep_dev(rvu, pcifunc)) { 2115 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 2116 *end = *start + hw->cap.nix_txsch_per_lbk_lmac; 2117 } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */ 2118 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 2119 *end = *start + hw->cap.nix_txsch_per_cgx_lmac; 2120 } else { /* SDP link */ 2121 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) + 2122 (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links); 2123 *end = *start + hw->cap.nix_txsch_per_sdp_lmac; 2124 } 2125 } 2126 2127 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc, 2128 struct nix_hw *nix_hw, 2129 struct nix_txsch_alloc_req *req) 2130 { 2131 struct rvu_hwinfo *hw = rvu->hw; 2132 int schq, req_schq, free_cnt; 2133 struct nix_txsch *txsch; 2134 int link, start, end; 2135 2136 txsch = &nix_hw->txsch[lvl]; 2137 req_schq = req->schq_contig[lvl] + req->schq[lvl]; 2138 2139 if (!req_schq) 2140 return 0; 2141 2142 link = nix_get_tx_link(rvu, pcifunc); 2143 2144 /* For traffic aggregating scheduler level, one queue is enough */ 2145 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2146 if (req_schq != 1) 2147 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2148 return 0; 2149 } 2150 2151 /* Get free SCHQ count and check if request can be accomodated */ 2152 if (hw->cap.nix_fixed_txschq_mapping) { 2153 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 2154 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK); 2155 if (end <= txsch->schq.max && schq < end && 2156 !test_bit(schq, txsch->schq.bmap)) 2157 free_cnt = 1; 2158 else 2159 free_cnt = 0; 2160 } else { 2161 free_cnt = rvu_rsrc_free_count(&txsch->schq); 2162 } 2163 2164 if (free_cnt < req_schq || req->schq[lvl] > MAX_TXSCHQ_PER_FUNC || 2165 req->schq_contig[lvl] > MAX_TXSCHQ_PER_FUNC) 2166 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2167 2168 /* If contiguous queues are needed, check for availability */ 2169 if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] && 2170 !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl])) 2171 return NIX_AF_ERR_TLX_ALLOC_FAIL; 2172 2173 return 0; 2174 } 2175 2176 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch, 2177 struct nix_txsch_alloc_rsp *rsp, 2178 int lvl, int start, int end) 2179 { 2180 struct rvu_hwinfo *hw = rvu->hw; 2181 u16 pcifunc = rsp->hdr.pcifunc; 2182 int idx, schq; 2183 2184 /* For traffic aggregating levels, queue alloc is based 2185 * on transmit link to which PF_FUNC is mapped to. 2186 */ 2187 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2188 /* A single TL queue is allocated */ 2189 if (rsp->schq_contig[lvl]) { 2190 rsp->schq_contig[lvl] = 1; 2191 rsp->schq_contig_list[lvl][0] = start; 2192 } 2193 2194 /* Both contig and non-contig reqs doesn't make sense here */ 2195 if (rsp->schq_contig[lvl]) 2196 rsp->schq[lvl] = 0; 2197 2198 if (rsp->schq[lvl]) { 2199 rsp->schq[lvl] = 1; 2200 rsp->schq_list[lvl][0] = start; 2201 } 2202 return; 2203 } 2204 2205 /* Adjust the queue request count if HW supports 2206 * only one queue per level configuration. 2207 */ 2208 if (hw->cap.nix_fixed_txschq_mapping) { 2209 idx = pcifunc & RVU_PFVF_FUNC_MASK; 2210 schq = start + idx; 2211 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) { 2212 rsp->schq_contig[lvl] = 0; 2213 rsp->schq[lvl] = 0; 2214 return; 2215 } 2216 2217 if (rsp->schq_contig[lvl]) { 2218 rsp->schq_contig[lvl] = 1; 2219 set_bit(schq, txsch->schq.bmap); 2220 rsp->schq_contig_list[lvl][0] = schq; 2221 rsp->schq[lvl] = 0; 2222 } else if (rsp->schq[lvl]) { 2223 rsp->schq[lvl] = 1; 2224 set_bit(schq, txsch->schq.bmap); 2225 rsp->schq_list[lvl][0] = schq; 2226 } 2227 return; 2228 } 2229 2230 /* Allocate contiguous queue indices requesty first */ 2231 if (rsp->schq_contig[lvl]) { 2232 schq = bitmap_find_next_zero_area(txsch->schq.bmap, 2233 txsch->schq.max, start, 2234 rsp->schq_contig[lvl], 0); 2235 if (schq >= end) 2236 rsp->schq_contig[lvl] = 0; 2237 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) { 2238 set_bit(schq, txsch->schq.bmap); 2239 rsp->schq_contig_list[lvl][idx] = schq; 2240 schq++; 2241 } 2242 } 2243 2244 /* Allocate non-contiguous queue indices */ 2245 if (rsp->schq[lvl]) { 2246 idx = 0; 2247 for (schq = start; schq < end; schq++) { 2248 if (!test_bit(schq, txsch->schq.bmap)) { 2249 set_bit(schq, txsch->schq.bmap); 2250 rsp->schq_list[lvl][idx++] = schq; 2251 } 2252 if (idx == rsp->schq[lvl]) 2253 break; 2254 } 2255 /* Update how many were allocated */ 2256 rsp->schq[lvl] = idx; 2257 } 2258 } 2259 2260 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu, 2261 struct nix_txsch_alloc_req *req, 2262 struct nix_txsch_alloc_rsp *rsp) 2263 { 2264 struct rvu_hwinfo *hw = rvu->hw; 2265 u16 pcifunc = req->hdr.pcifunc; 2266 int link, blkaddr, rc = 0; 2267 int lvl, idx, start, end; 2268 struct nix_txsch *txsch; 2269 struct nix_hw *nix_hw; 2270 u32 *pfvf_map; 2271 int nixlf; 2272 u16 schq; 2273 2274 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2275 if (rc) 2276 return rc; 2277 2278 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2279 if (!nix_hw) 2280 return NIX_AF_ERR_INVALID_NIXBLK; 2281 2282 mutex_lock(&rvu->rsrc_lock); 2283 2284 /* Check if request is valid as per HW capabilities 2285 * and can be accomodated. 2286 */ 2287 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2288 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req); 2289 if (rc) 2290 goto err; 2291 } 2292 2293 /* Allocate requested Tx scheduler queues */ 2294 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2295 txsch = &nix_hw->txsch[lvl]; 2296 pfvf_map = txsch->pfvf_map; 2297 2298 if (!req->schq[lvl] && !req->schq_contig[lvl]) 2299 continue; 2300 2301 rsp->schq[lvl] = req->schq[lvl]; 2302 rsp->schq_contig[lvl] = req->schq_contig[lvl]; 2303 2304 link = nix_get_tx_link(rvu, pcifunc); 2305 2306 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2307 start = link; 2308 end = link; 2309 } else if (hw->cap.nix_fixed_txschq_mapping) { 2310 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 2311 } else { 2312 start = 0; 2313 end = txsch->schq.max; 2314 } 2315 2316 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end); 2317 2318 /* Reset queue config */ 2319 for (idx = 0; idx < req->schq_contig[lvl]; idx++) { 2320 schq = rsp->schq_contig_list[lvl][idx]; 2321 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2322 NIX_TXSCHQ_CFG_DONE)) 2323 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2324 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2325 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2326 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2327 } 2328 2329 for (idx = 0; idx < req->schq[lvl]; idx++) { 2330 schq = rsp->schq_list[lvl][idx]; 2331 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2332 NIX_TXSCHQ_CFG_DONE)) 2333 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2334 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2335 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2336 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2337 } 2338 } 2339 2340 rsp->aggr_level = hw->cap.nix_tx_aggr_lvl; 2341 rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO; 2342 rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr, 2343 NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2344 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2345 goto exit; 2346 err: 2347 rc = NIX_AF_ERR_TLX_ALLOC_FAIL; 2348 exit: 2349 mutex_unlock(&rvu->rsrc_lock); 2350 return rc; 2351 } 2352 2353 static void nix_smq_flush_fill_ctx(struct rvu *rvu, int blkaddr, int smq, 2354 struct nix_smq_flush_ctx *smq_flush_ctx) 2355 { 2356 struct nix_smq_tree_ctx *smq_tree_ctx; 2357 u64 parent_off, regval; 2358 u16 schq; 2359 int lvl; 2360 2361 smq_flush_ctx->smq = smq; 2362 2363 schq = smq; 2364 for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) { 2365 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl]; 2366 smq_tree_ctx->schq = schq; 2367 if (lvl == NIX_TXSCH_LVL_TL1) { 2368 smq_tree_ctx->cir_off = NIX_AF_TL1X_CIR(schq); 2369 smq_tree_ctx->pir_off = 0; 2370 smq_tree_ctx->pir_val = 0; 2371 parent_off = 0; 2372 } else if (lvl == NIX_TXSCH_LVL_TL2) { 2373 smq_tree_ctx->cir_off = NIX_AF_TL2X_CIR(schq); 2374 smq_tree_ctx->pir_off = NIX_AF_TL2X_PIR(schq); 2375 parent_off = NIX_AF_TL2X_PARENT(schq); 2376 } else if (lvl == NIX_TXSCH_LVL_TL3) { 2377 smq_tree_ctx->cir_off = NIX_AF_TL3X_CIR(schq); 2378 smq_tree_ctx->pir_off = NIX_AF_TL3X_PIR(schq); 2379 parent_off = NIX_AF_TL3X_PARENT(schq); 2380 } else if (lvl == NIX_TXSCH_LVL_TL4) { 2381 smq_tree_ctx->cir_off = NIX_AF_TL4X_CIR(schq); 2382 smq_tree_ctx->pir_off = NIX_AF_TL4X_PIR(schq); 2383 parent_off = NIX_AF_TL4X_PARENT(schq); 2384 } else if (lvl == NIX_TXSCH_LVL_MDQ) { 2385 smq_tree_ctx->cir_off = NIX_AF_MDQX_CIR(schq); 2386 smq_tree_ctx->pir_off = NIX_AF_MDQX_PIR(schq); 2387 parent_off = NIX_AF_MDQX_PARENT(schq); 2388 } 2389 /* save cir/pir register values */ 2390 smq_tree_ctx->cir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->cir_off); 2391 if (smq_tree_ctx->pir_off) 2392 smq_tree_ctx->pir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->pir_off); 2393 2394 /* get parent txsch node */ 2395 if (parent_off) { 2396 regval = rvu_read64(rvu, blkaddr, parent_off); 2397 schq = (regval >> 16) & 0x1FF; 2398 } 2399 } 2400 } 2401 2402 static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr, 2403 struct nix_smq_flush_ctx *smq_flush_ctx, bool enable) 2404 { 2405 struct nix_txsch *txsch; 2406 struct nix_hw *nix_hw; 2407 int tl2, tl2_schq; 2408 u64 regoff; 2409 2410 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2411 if (!nix_hw) 2412 return; 2413 2414 /* loop through all TL2s with matching PF_FUNC */ 2415 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2]; 2416 tl2_schq = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL2].schq; 2417 for (tl2 = 0; tl2 < txsch->schq.max; tl2++) { 2418 /* skip the smq(flush) TL2 */ 2419 if (tl2 == tl2_schq) 2420 continue; 2421 /* skip unused TL2s */ 2422 if (TXSCH_MAP_FLAGS(txsch->pfvf_map[tl2]) & NIX_TXSCHQ_FREE) 2423 continue; 2424 /* skip if PF_FUNC doesn't match */ 2425 if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) != 2426 (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq]) & 2427 ~RVU_PFVF_FUNC_MASK)) 2428 continue; 2429 /* enable/disable XOFF */ 2430 regoff = NIX_AF_TL2X_SW_XOFF(tl2); 2431 if (enable) 2432 rvu_write64(rvu, blkaddr, regoff, 0x1); 2433 else 2434 rvu_write64(rvu, blkaddr, regoff, 0x0); 2435 } 2436 } 2437 2438 static void nix_smq_flush_enadis_rate(struct rvu *rvu, int blkaddr, 2439 struct nix_smq_flush_ctx *smq_flush_ctx, bool enable) 2440 { 2441 u64 cir_off, pir_off, cir_val, pir_val; 2442 struct nix_smq_tree_ctx *smq_tree_ctx; 2443 int lvl; 2444 2445 for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) { 2446 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl]; 2447 cir_off = smq_tree_ctx->cir_off; 2448 cir_val = smq_tree_ctx->cir_val; 2449 pir_off = smq_tree_ctx->pir_off; 2450 pir_val = smq_tree_ctx->pir_val; 2451 2452 if (enable) { 2453 rvu_write64(rvu, blkaddr, cir_off, cir_val); 2454 if (lvl != NIX_TXSCH_LVL_TL1) 2455 rvu_write64(rvu, blkaddr, pir_off, pir_val); 2456 } else { 2457 rvu_write64(rvu, blkaddr, cir_off, 0x0); 2458 if (lvl != NIX_TXSCH_LVL_TL1) 2459 rvu_write64(rvu, blkaddr, pir_off, 0x0); 2460 } 2461 } 2462 } 2463 2464 static int nix_smq_flush(struct rvu *rvu, int blkaddr, 2465 int smq, u16 pcifunc, int nixlf) 2466 { 2467 struct nix_smq_flush_ctx *smq_flush_ctx; 2468 int err, restore_tx_en = 0, i; 2469 int pf = rvu_get_pf(rvu->pdev, pcifunc); 2470 u8 cgx_id = 0, lmac_id = 0; 2471 u16 tl2_tl3_link_schq; 2472 u8 link, link_level; 2473 u64 cfg, bmap = 0; 2474 2475 if (!is_rvu_otx2(rvu)) { 2476 /* Skip SMQ flush if pkt count is zero */ 2477 cfg = rvu_read64(rvu, blkaddr, NIX_AF_MDQX_IN_MD_COUNT(smq)); 2478 if (!cfg) 2479 return 0; 2480 } 2481 2482 /* enable cgx tx if disabled */ 2483 if (is_pf_cgxmapped(rvu, pf)) { 2484 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 2485 restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), 2486 lmac_id, true); 2487 } 2488 2489 /* XOFF all TL2s whose parent TL1 matches SMQ tree TL1 */ 2490 smq_flush_ctx = kzalloc_obj(*smq_flush_ctx); 2491 if (!smq_flush_ctx) 2492 return -ENOMEM; 2493 nix_smq_flush_fill_ctx(rvu, blkaddr, smq, smq_flush_ctx); 2494 nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, true); 2495 nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, false); 2496 2497 /* Disable backpressure from physical link, 2498 * otherwise SMQ flush may stall. 2499 */ 2500 rvu_cgx_enadis_rx_bp(rvu, pf, false); 2501 2502 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2503 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2504 tl2_tl3_link_schq = smq_flush_ctx->smq_tree_ctx[link_level].schq; 2505 link = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL1].schq; 2506 2507 /* SMQ set enqueue xoff */ 2508 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); 2509 cfg |= BIT_ULL(50); 2510 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg); 2511 2512 /* Clear all NIX_AF_TL3_TL2_LINK_CFG[ENA] for the TL3/TL2 queue */ 2513 for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { 2514 cfg = rvu_read64(rvu, blkaddr, 2515 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); 2516 if (!(cfg & BIT_ULL(12))) 2517 continue; 2518 bmap |= BIT_ULL(i); 2519 cfg &= ~BIT_ULL(12); 2520 rvu_write64(rvu, blkaddr, 2521 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); 2522 } 2523 2524 /* Do SMQ flush and set enqueue xoff */ 2525 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); 2526 cfg |= BIT_ULL(50) | BIT_ULL(49); 2527 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg); 2528 2529 /* Wait for flush to complete */ 2530 err = rvu_poll_reg(rvu, blkaddr, 2531 NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true); 2532 if (err) 2533 dev_info(rvu->dev, 2534 "NIXLF%d: SMQ%d flush failed, txlink might be busy\n", 2535 nixlf, smq); 2536 2537 /* Set NIX_AF_TL3_TL2_LINKX_CFG[ENA] for the TL3/TL2 queue */ 2538 for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { 2539 if (!(bmap & BIT_ULL(i))) 2540 continue; 2541 cfg = rvu_read64(rvu, blkaddr, 2542 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); 2543 cfg |= BIT_ULL(12); 2544 rvu_write64(rvu, blkaddr, 2545 NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); 2546 } 2547 2548 /* clear XOFF on TL2s */ 2549 nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, true); 2550 nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, false); 2551 kfree(smq_flush_ctx); 2552 2553 rvu_cgx_enadis_rx_bp(rvu, pf, true); 2554 /* restore cgx tx state */ 2555 if (restore_tx_en) 2556 rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); 2557 return err; 2558 } 2559 2560 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc) 2561 { 2562 int blkaddr, nixlf, lvl, schq, err; 2563 struct rvu_hwinfo *hw = rvu->hw; 2564 struct nix_txsch *txsch; 2565 struct nix_hw *nix_hw; 2566 u16 map_func; 2567 2568 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2569 if (blkaddr < 0) 2570 return NIX_AF_ERR_AF_LF_INVALID; 2571 2572 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2573 if (!nix_hw) 2574 return NIX_AF_ERR_INVALID_NIXBLK; 2575 2576 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2577 if (nixlf < 0) 2578 return NIX_AF_ERR_AF_LF_INVALID; 2579 2580 /* Disable TL2/3 queue links and all XOFF's before SMQ flush*/ 2581 mutex_lock(&rvu->rsrc_lock); 2582 for (lvl = NIX_TXSCH_LVL_MDQ; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2583 txsch = &nix_hw->txsch[lvl]; 2584 2585 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2586 continue; 2587 2588 for (schq = 0; schq < txsch->schq.max; schq++) { 2589 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2590 continue; 2591 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2592 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2593 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2594 } 2595 } 2596 nix_clear_tx_xoff(rvu, blkaddr, NIX_TXSCH_LVL_TL1, 2597 nix_get_tx_link(rvu, pcifunc)); 2598 2599 /* On PF cleanup, clear cfg done flag as 2600 * PF would have changed default config. 2601 */ 2602 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 2603 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL1]; 2604 schq = nix_get_tx_link(rvu, pcifunc); 2605 /* Do not clear pcifunc in txsch->pfvf_map[schq] because 2606 * VF might be using this TL1 queue 2607 */ 2608 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 2609 txsch->pfvf_map[schq] = TXSCH_SET_FLAG(map_func, 0x0); 2610 } 2611 2612 /* Flush SMQs */ 2613 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ]; 2614 for (schq = 0; schq < txsch->schq.max; schq++) { 2615 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2616 continue; 2617 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2618 } 2619 2620 /* Now free scheduler queues to free pool */ 2621 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2622 /* TLs above aggregation level are shared across all PF 2623 * and it's VFs, hence skip freeing them. 2624 */ 2625 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2626 continue; 2627 2628 txsch = &nix_hw->txsch[lvl]; 2629 for (schq = 0; schq < txsch->schq.max; schq++) { 2630 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2631 continue; 2632 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2633 rvu_free_rsrc(&txsch->schq, schq); 2634 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2635 } 2636 } 2637 mutex_unlock(&rvu->rsrc_lock); 2638 2639 err = rvu_ndc_sync(rvu, blkaddr, nixlf, NIX_AF_NDC_TX_SYNC); 2640 if (err) 2641 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf); 2642 2643 return 0; 2644 } 2645 2646 static int nix_txschq_free_one(struct rvu *rvu, 2647 struct nix_txsch_free_req *req) 2648 { 2649 struct rvu_hwinfo *hw = rvu->hw; 2650 u16 pcifunc = req->hdr.pcifunc; 2651 int lvl, schq, nixlf, blkaddr; 2652 struct nix_txsch *txsch; 2653 struct nix_hw *nix_hw; 2654 u32 *pfvf_map; 2655 int rc; 2656 2657 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2658 if (blkaddr < 0) 2659 return NIX_AF_ERR_AF_LF_INVALID; 2660 2661 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2662 if (!nix_hw) 2663 return NIX_AF_ERR_INVALID_NIXBLK; 2664 2665 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2666 if (nixlf < 0) 2667 return NIX_AF_ERR_AF_LF_INVALID; 2668 2669 lvl = req->schq_lvl; 2670 schq = req->schq; 2671 txsch = &nix_hw->txsch[lvl]; 2672 2673 if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max) 2674 return 0; 2675 2676 pfvf_map = txsch->pfvf_map; 2677 mutex_lock(&rvu->rsrc_lock); 2678 2679 if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) { 2680 rc = NIX_AF_ERR_TLX_INVALID; 2681 goto err; 2682 } 2683 2684 /* Clear SW_XOFF of this resource only. 2685 * For SMQ level, all path XOFF's 2686 * need to be made clear by user 2687 */ 2688 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2689 2690 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2691 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2692 2693 /* Flush if it is a SMQ. Onus of disabling 2694 * TL2/3 queue links before SMQ flush is on user 2695 */ 2696 if (lvl == NIX_TXSCH_LVL_SMQ && 2697 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf)) { 2698 rc = NIX_AF_SMQ_FLUSH_FAILED; 2699 goto err; 2700 } 2701 2702 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2703 2704 /* Free the resource */ 2705 rvu_free_rsrc(&txsch->schq, schq); 2706 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2707 mutex_unlock(&rvu->rsrc_lock); 2708 return 0; 2709 err: 2710 mutex_unlock(&rvu->rsrc_lock); 2711 return rc; 2712 } 2713 2714 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu, 2715 struct nix_txsch_free_req *req, 2716 struct msg_rsp *rsp) 2717 { 2718 if (req->flags & TXSCHQ_FREE_ALL) 2719 return nix_txschq_free(rvu, req->hdr.pcifunc); 2720 else 2721 return nix_txschq_free_one(rvu, req); 2722 } 2723 2724 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr, 2725 int lvl, u64 reg, u64 regval) 2726 { 2727 u64 regbase = reg & 0xFFFF; 2728 u16 schq, parent; 2729 2730 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg)) 2731 return false; 2732 2733 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2734 /* Check if this schq belongs to this PF/VF or not */ 2735 if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq)) 2736 return false; 2737 2738 parent = (regval >> 16) & 0x1FF; 2739 /* Validate MDQ's TL4 parent */ 2740 if (regbase == NIX_AF_MDQX_PARENT(0) && 2741 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent)) 2742 return false; 2743 2744 /* Validate TL4's TL3 parent */ 2745 if (regbase == NIX_AF_TL4X_PARENT(0) && 2746 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent)) 2747 return false; 2748 2749 /* Validate TL3's TL2 parent */ 2750 if (regbase == NIX_AF_TL3X_PARENT(0) && 2751 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent)) 2752 return false; 2753 2754 /* Validate TL2's TL1 parent */ 2755 if (regbase == NIX_AF_TL2X_PARENT(0) && 2756 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent)) 2757 return false; 2758 2759 return true; 2760 } 2761 2762 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg) 2763 { 2764 u64 regbase; 2765 2766 if (hw->cap.nix_shaping) 2767 return true; 2768 2769 /* If shaping and coloring is not supported, then 2770 * *_CIR and *_PIR registers should not be configured. 2771 */ 2772 regbase = reg & 0xFFFF; 2773 2774 switch (lvl) { 2775 case NIX_TXSCH_LVL_TL1: 2776 if (regbase == NIX_AF_TL1X_CIR(0)) 2777 return false; 2778 break; 2779 case NIX_TXSCH_LVL_TL2: 2780 if (regbase == NIX_AF_TL2X_CIR(0) || 2781 regbase == NIX_AF_TL2X_PIR(0)) 2782 return false; 2783 break; 2784 case NIX_TXSCH_LVL_TL3: 2785 if (regbase == NIX_AF_TL3X_CIR(0) || 2786 regbase == NIX_AF_TL3X_PIR(0)) 2787 return false; 2788 break; 2789 case NIX_TXSCH_LVL_TL4: 2790 if (regbase == NIX_AF_TL4X_CIR(0) || 2791 regbase == NIX_AF_TL4X_PIR(0)) 2792 return false; 2793 break; 2794 case NIX_TXSCH_LVL_MDQ: 2795 if (regbase == NIX_AF_MDQX_CIR(0) || 2796 regbase == NIX_AF_MDQX_PIR(0)) 2797 return false; 2798 break; 2799 } 2800 return true; 2801 } 2802 2803 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw, 2804 u16 pcifunc, int blkaddr) 2805 { 2806 u32 *pfvf_map; 2807 int schq; 2808 2809 schq = nix_get_tx_link(rvu, pcifunc); 2810 pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map; 2811 /* Skip if PF has already done the config */ 2812 if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE) 2813 return; 2814 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq), 2815 (TXSCH_TL1_DFLT_RR_PRIO << 1)); 2816 2817 /* On OcteonTx2 the config was in bytes and newer silcons 2818 * it's changed to weight. 2819 */ 2820 if (!rvu->hw->cap.nix_common_dwrr_mtu) 2821 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2822 TXSCH_TL1_DFLT_RR_QTM); 2823 else 2824 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2825 CN10K_MAX_DWRR_WEIGHT); 2826 2827 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00); 2828 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE); 2829 } 2830 2831 /* Register offset - [15:0] 2832 * Scheduler Queue number - [25:16] 2833 */ 2834 #define NIX_TX_SCHQ_MASK GENMASK_ULL(25, 0) 2835 2836 static int nix_txschq_cfg_read(struct rvu *rvu, struct nix_hw *nix_hw, 2837 int blkaddr, struct nix_txschq_config *req, 2838 struct nix_txschq_config *rsp) 2839 { 2840 u16 pcifunc = req->hdr.pcifunc; 2841 int idx, schq; 2842 u64 reg; 2843 2844 for (idx = 0; idx < req->num_regs; idx++) { 2845 reg = req->reg[idx]; 2846 reg &= NIX_TX_SCHQ_MASK; 2847 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2848 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, req->lvl, reg) || 2849 !is_valid_txschq(rvu, blkaddr, req->lvl, pcifunc, schq)) 2850 return NIX_AF_INVAL_TXSCHQ_CFG; 2851 rsp->regval[idx] = rvu_read64(rvu, blkaddr, reg); 2852 } 2853 rsp->lvl = req->lvl; 2854 rsp->num_regs = req->num_regs; 2855 return 0; 2856 } 2857 2858 void rvu_nix_tx_tl2_cfg(struct rvu *rvu, int blkaddr, u16 pcifunc, 2859 struct nix_txsch *txsch, bool enable) 2860 { 2861 struct rvu_hwinfo *hw = rvu->hw; 2862 int lbk_link_start, lbk_links; 2863 u8 pf = rvu_get_pf(rvu->pdev, pcifunc); 2864 int schq; 2865 u64 cfg; 2866 2867 if (!is_pf_cgxmapped(rvu, pf) && !is_rep_dev(rvu, pcifunc)) 2868 return; 2869 2870 cfg = enable ? (BIT_ULL(12) | RVU_SWITCH_LBK_CHAN) : 0; 2871 lbk_link_start = hw->cgx_links; 2872 2873 for (schq = 0; schq < txsch->schq.max; schq++) { 2874 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2875 continue; 2876 /* Enable all LBK links with channel 63 by default so that 2877 * packets can be sent to LBK with a NPC TX MCAM rule 2878 */ 2879 lbk_links = hw->lbk_links; 2880 while (lbk_links--) 2881 rvu_write64(rvu, blkaddr, 2882 NIX_AF_TL3_TL2X_LINKX_CFG(schq, 2883 lbk_link_start + 2884 lbk_links), cfg); 2885 } 2886 } 2887 2888 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu, 2889 struct nix_txschq_config *req, 2890 struct nix_txschq_config *rsp) 2891 { 2892 u64 reg, val, regval, schq_regbase, val_mask; 2893 struct rvu_hwinfo *hw = rvu->hw; 2894 u16 pcifunc = req->hdr.pcifunc; 2895 struct nix_txsch *txsch; 2896 struct nix_hw *nix_hw; 2897 int blkaddr, idx, err; 2898 int nixlf, schq; 2899 u32 *pfvf_map; 2900 2901 if (req->lvl >= NIX_TXSCH_LVL_CNT || 2902 req->num_regs > MAX_REGS_PER_MBOX_MSG) 2903 return NIX_AF_INVAL_TXSCHQ_CFG; 2904 2905 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2906 if (err) 2907 return err; 2908 2909 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2910 if (!nix_hw) 2911 return NIX_AF_ERR_INVALID_NIXBLK; 2912 2913 if (req->read) 2914 return nix_txschq_cfg_read(rvu, nix_hw, blkaddr, req, rsp); 2915 2916 txsch = &nix_hw->txsch[req->lvl]; 2917 pfvf_map = txsch->pfvf_map; 2918 2919 if (req->lvl >= hw->cap.nix_tx_aggr_lvl && 2920 pcifunc & RVU_PFVF_FUNC_MASK) { 2921 mutex_lock(&rvu->rsrc_lock); 2922 if (req->lvl == NIX_TXSCH_LVL_TL1) 2923 nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr); 2924 mutex_unlock(&rvu->rsrc_lock); 2925 return 0; 2926 } 2927 2928 for (idx = 0; idx < req->num_regs; idx++) { 2929 reg = req->reg[idx]; 2930 reg &= NIX_TX_SCHQ_MASK; 2931 regval = req->regval[idx]; 2932 schq_regbase = reg & 0xFFFF; 2933 val_mask = req->regval_mask[idx]; 2934 2935 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr, 2936 txsch->lvl, reg, regval)) 2937 return NIX_AF_INVAL_TXSCHQ_CFG; 2938 2939 /* Check if shaping and coloring is supported */ 2940 if (!is_txschq_shaping_valid(hw, req->lvl, reg)) 2941 continue; 2942 2943 val = rvu_read64(rvu, blkaddr, reg); 2944 regval = (val & val_mask) | (regval & ~val_mask); 2945 2946 /* Handle shaping state toggle specially */ 2947 if (hw->cap.nix_shaper_toggle_wait && 2948 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2949 req->lvl, reg, regval)) 2950 continue; 2951 2952 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */ 2953 if (schq_regbase == NIX_AF_SMQX_CFG(0)) { 2954 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], 2955 pcifunc, 0); 2956 regval &= ~(0x7FULL << 24); 2957 regval |= ((u64)nixlf << 24); 2958 } 2959 2960 /* Clear 'BP_ENA' config, if it's not allowed */ 2961 if (!hw->cap.nix_tx_link_bp) { 2962 if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) || 2963 (schq_regbase & 0xFF00) == 2964 NIX_AF_TL3_TL2X_LINKX_CFG(0, 0)) 2965 regval &= ~BIT_ULL(13); 2966 } 2967 2968 /* Mark config as done for TL1 by PF */ 2969 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) && 2970 schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) { 2971 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2972 mutex_lock(&rvu->rsrc_lock); 2973 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], 2974 NIX_TXSCHQ_CFG_DONE); 2975 mutex_unlock(&rvu->rsrc_lock); 2976 } 2977 2978 /* SMQ flush is special hence split register writes such 2979 * that flush first and write rest of the bits later. 2980 */ 2981 if (schq_regbase == NIX_AF_SMQX_CFG(0) && 2982 (regval & BIT_ULL(49))) { 2983 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2984 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2985 regval &= ~BIT_ULL(49); 2986 } 2987 rvu_write64(rvu, blkaddr, reg, regval); 2988 } 2989 2990 return 0; 2991 } 2992 2993 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr, 2994 struct nix_vtag_config *req) 2995 { 2996 u64 regval = req->vtag_size; 2997 2998 if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 || 2999 req->vtag_size > VTAGSIZE_T8) 3000 return -EINVAL; 3001 3002 /* RX VTAG Type 7 reserved for vf vlan */ 3003 if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7) 3004 return NIX_AF_ERR_RX_VTAG_INUSE; 3005 3006 if (req->rx.capture_vtag) 3007 regval |= BIT_ULL(5); 3008 if (req->rx.strip_vtag) 3009 regval |= BIT_ULL(4); 3010 3011 rvu_write64(rvu, blkaddr, 3012 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval); 3013 return 0; 3014 } 3015 3016 static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr, 3017 u16 pcifunc, int index) 3018 { 3019 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3020 struct nix_txvlan *vlan; 3021 3022 if (!nix_hw) 3023 return NIX_AF_ERR_INVALID_NIXBLK; 3024 3025 vlan = &nix_hw->txvlan; 3026 if (vlan->entry2pfvf_map[index] != pcifunc) 3027 return NIX_AF_ERR_PARAM; 3028 3029 rvu_write64(rvu, blkaddr, 3030 NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull); 3031 rvu_write64(rvu, blkaddr, 3032 NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull); 3033 3034 vlan->entry2pfvf_map[index] = 0; 3035 rvu_free_rsrc(&vlan->rsrc, index); 3036 3037 return 0; 3038 } 3039 3040 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc) 3041 { 3042 struct nix_txvlan *vlan; 3043 struct nix_hw *nix_hw; 3044 int index, blkaddr; 3045 3046 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3047 if (blkaddr < 0) 3048 return; 3049 3050 nix_hw = get_nix_hw(rvu->hw, blkaddr); 3051 if (!nix_hw) 3052 return; 3053 3054 vlan = &nix_hw->txvlan; 3055 3056 mutex_lock(&vlan->rsrc_lock); 3057 /* Scan all the entries and free the ones mapped to 'pcifunc' */ 3058 for (index = 0; index < vlan->rsrc.max; index++) { 3059 if (vlan->entry2pfvf_map[index] == pcifunc) 3060 nix_tx_vtag_free(rvu, blkaddr, pcifunc, index); 3061 } 3062 mutex_unlock(&vlan->rsrc_lock); 3063 } 3064 3065 static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr, 3066 u64 vtag, u8 size) 3067 { 3068 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3069 struct nix_txvlan *vlan; 3070 u64 regval; 3071 int index; 3072 3073 if (!nix_hw) 3074 return NIX_AF_ERR_INVALID_NIXBLK; 3075 3076 vlan = &nix_hw->txvlan; 3077 3078 mutex_lock(&vlan->rsrc_lock); 3079 3080 index = rvu_alloc_rsrc(&vlan->rsrc); 3081 if (index < 0) { 3082 mutex_unlock(&vlan->rsrc_lock); 3083 return index; 3084 } 3085 3086 mutex_unlock(&vlan->rsrc_lock); 3087 3088 regval = size ? vtag : vtag << 32; 3089 3090 rvu_write64(rvu, blkaddr, 3091 NIX_AF_TX_VTAG_DEFX_DATA(index), regval); 3092 rvu_write64(rvu, blkaddr, 3093 NIX_AF_TX_VTAG_DEFX_CTL(index), size); 3094 3095 return index; 3096 } 3097 3098 static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr, 3099 struct nix_vtag_config *req) 3100 { 3101 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3102 u16 pcifunc = req->hdr.pcifunc; 3103 int idx0 = req->tx.vtag0_idx; 3104 int idx1 = req->tx.vtag1_idx; 3105 struct nix_txvlan *vlan; 3106 int err = 0; 3107 3108 if (!nix_hw) 3109 return NIX_AF_ERR_INVALID_NIXBLK; 3110 3111 vlan = &nix_hw->txvlan; 3112 if (req->tx.free_vtag0 && req->tx.free_vtag1) 3113 if (vlan->entry2pfvf_map[idx0] != pcifunc || 3114 vlan->entry2pfvf_map[idx1] != pcifunc) 3115 return NIX_AF_ERR_PARAM; 3116 3117 mutex_lock(&vlan->rsrc_lock); 3118 3119 if (req->tx.free_vtag0) { 3120 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0); 3121 if (err) 3122 goto exit; 3123 } 3124 3125 if (req->tx.free_vtag1) 3126 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1); 3127 3128 exit: 3129 mutex_unlock(&vlan->rsrc_lock); 3130 return err; 3131 } 3132 3133 static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr, 3134 struct nix_vtag_config *req, 3135 struct nix_vtag_config_rsp *rsp) 3136 { 3137 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 3138 struct nix_txvlan *vlan; 3139 u16 pcifunc = req->hdr.pcifunc; 3140 3141 if (!nix_hw) 3142 return NIX_AF_ERR_INVALID_NIXBLK; 3143 3144 vlan = &nix_hw->txvlan; 3145 if (req->tx.cfg_vtag0) { 3146 rsp->vtag0_idx = 3147 nix_tx_vtag_alloc(rvu, blkaddr, 3148 req->tx.vtag0, req->vtag_size); 3149 3150 if (rsp->vtag0_idx < 0) 3151 return NIX_AF_ERR_TX_VTAG_NOSPC; 3152 3153 vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc; 3154 } 3155 3156 if (req->tx.cfg_vtag1) { 3157 rsp->vtag1_idx = 3158 nix_tx_vtag_alloc(rvu, blkaddr, 3159 req->tx.vtag1, req->vtag_size); 3160 3161 if (rsp->vtag1_idx < 0) 3162 goto err_free; 3163 3164 vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc; 3165 } 3166 3167 return 0; 3168 3169 err_free: 3170 if (req->tx.cfg_vtag0) 3171 nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx); 3172 3173 return NIX_AF_ERR_TX_VTAG_NOSPC; 3174 } 3175 3176 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu, 3177 struct nix_vtag_config *req, 3178 struct nix_vtag_config_rsp *rsp) 3179 { 3180 u16 pcifunc = req->hdr.pcifunc; 3181 int blkaddr, nixlf, err; 3182 3183 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3184 if (err) 3185 return err; 3186 3187 if (req->cfg_type) { 3188 /* rx vtag configuration */ 3189 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req); 3190 if (err) 3191 return NIX_AF_ERR_PARAM; 3192 } else { 3193 /* tx vtag configuration */ 3194 if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) && 3195 (req->tx.free_vtag0 || req->tx.free_vtag1)) 3196 return NIX_AF_ERR_PARAM; 3197 3198 if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1) 3199 return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp); 3200 3201 if (req->tx.free_vtag0 || req->tx.free_vtag1) 3202 return nix_tx_vtag_decfg(rvu, blkaddr, req); 3203 } 3204 3205 return 0; 3206 } 3207 3208 static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw, 3209 int mce, u8 op, u16 pcifunc, int next, 3210 int index, u8 mce_op, bool eol) 3211 { 3212 struct nix_aq_enq_req aq_req; 3213 int err; 3214 3215 aq_req.hdr.pcifunc = 0; 3216 aq_req.ctype = NIX_AQ_CTYPE_MCE; 3217 aq_req.op = op; 3218 aq_req.qidx = mce; 3219 3220 /* Use RSS with RSS index 0 */ 3221 aq_req.mce.op = mce_op; 3222 aq_req.mce.index = index; 3223 aq_req.mce.eol = eol; 3224 aq_req.mce.pf_func = pcifunc; 3225 aq_req.mce.next = next; 3226 3227 /* All fields valid */ 3228 *(u64 *)(&aq_req.mce_mask) = ~0ULL; 3229 3230 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL); 3231 if (err) { 3232 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n", 3233 rvu_get_pf(rvu->pdev, pcifunc), 3234 pcifunc & RVU_PFVF_FUNC_MASK); 3235 return err; 3236 } 3237 return 0; 3238 } 3239 3240 static void nix_delete_mcast_mce_list(struct nix_mce_list *mce_list) 3241 { 3242 struct hlist_node *tmp; 3243 struct mce *mce; 3244 3245 /* Scan through the current list */ 3246 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 3247 hlist_del(&mce->node); 3248 kfree(mce); 3249 } 3250 3251 mce_list->count = 0; 3252 mce_list->max = 0; 3253 } 3254 3255 static int nix_get_last_mce_list_index(struct nix_mcast_grp_elem *elem) 3256 { 3257 return elem->mce_start_index + elem->mcast_mce_list.count - 1; 3258 } 3259 3260 static int nix_update_ingress_mce_list_hw(struct rvu *rvu, 3261 struct nix_hw *nix_hw, 3262 struct nix_mcast_grp_elem *elem) 3263 { 3264 int idx, last_idx, next_idx, err; 3265 struct nix_mce_list *mce_list; 3266 struct mce *mce, *prev_mce; 3267 3268 mce_list = &elem->mcast_mce_list; 3269 idx = elem->mce_start_index; 3270 last_idx = nix_get_last_mce_list_index(elem); 3271 hlist_for_each_entry(mce, &mce_list->head, node) { 3272 if (idx > last_idx) 3273 break; 3274 3275 if (!mce->is_active) { 3276 if (idx == elem->mce_start_index) { 3277 idx++; 3278 prev_mce = mce; 3279 elem->mce_start_index = idx; 3280 continue; 3281 } else if (idx == last_idx) { 3282 err = nix_blk_setup_mce(rvu, nix_hw, idx - 1, NIX_AQ_INSTOP_WRITE, 3283 prev_mce->pcifunc, next_idx, 3284 prev_mce->rq_rss_index, 3285 prev_mce->dest_type, 3286 false); 3287 if (err) 3288 return err; 3289 3290 break; 3291 } 3292 } 3293 3294 next_idx = idx + 1; 3295 /* EOL should be set in last MCE */ 3296 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE, 3297 mce->pcifunc, next_idx, 3298 mce->rq_rss_index, mce->dest_type, 3299 (next_idx > last_idx) ? true : false); 3300 if (err) 3301 return err; 3302 3303 idx++; 3304 prev_mce = mce; 3305 } 3306 3307 return 0; 3308 } 3309 3310 static void nix_update_egress_mce_list_hw(struct rvu *rvu, 3311 struct nix_hw *nix_hw, 3312 struct nix_mcast_grp_elem *elem) 3313 { 3314 struct nix_mce_list *mce_list; 3315 int idx, last_idx, next_idx; 3316 struct mce *mce, *prev_mce; 3317 u64 regval; 3318 u8 eol; 3319 3320 mce_list = &elem->mcast_mce_list; 3321 idx = elem->mce_start_index; 3322 last_idx = nix_get_last_mce_list_index(elem); 3323 hlist_for_each_entry(mce, &mce_list->head, node) { 3324 if (idx > last_idx) 3325 break; 3326 3327 if (!mce->is_active) { 3328 if (idx == elem->mce_start_index) { 3329 idx++; 3330 prev_mce = mce; 3331 elem->mce_start_index = idx; 3332 continue; 3333 } else if (idx == last_idx) { 3334 regval = (next_idx << 16) | (1 << 12) | prev_mce->channel; 3335 rvu_write64(rvu, nix_hw->blkaddr, 3336 NIX_AF_TX_MCASTX(idx - 1), 3337 regval); 3338 break; 3339 } 3340 } 3341 3342 eol = 0; 3343 next_idx = idx + 1; 3344 /* EOL should be set in last MCE */ 3345 if (next_idx > last_idx) 3346 eol = 1; 3347 3348 regval = (next_idx << 16) | (eol << 12) | mce->channel; 3349 rvu_write64(rvu, nix_hw->blkaddr, 3350 NIX_AF_TX_MCASTX(idx), 3351 regval); 3352 idx++; 3353 prev_mce = mce; 3354 } 3355 } 3356 3357 static int nix_del_mce_list_entry(struct rvu *rvu, 3358 struct nix_hw *nix_hw, 3359 struct nix_mcast_grp_elem *elem, 3360 struct nix_mcast_grp_update_req *req) 3361 { 3362 u32 num_entry = req->num_mce_entry; 3363 struct nix_mce_list *mce_list; 3364 struct mce *mce; 3365 bool is_found; 3366 int i; 3367 3368 mce_list = &elem->mcast_mce_list; 3369 for (i = 0; i < num_entry; i++) { 3370 is_found = false; 3371 hlist_for_each_entry(mce, &mce_list->head, node) { 3372 /* If already exists, then delete */ 3373 if (mce->pcifunc == req->pcifunc[i]) { 3374 hlist_del(&mce->node); 3375 kfree(mce); 3376 mce_list->count--; 3377 is_found = true; 3378 break; 3379 } 3380 } 3381 3382 if (!is_found) 3383 return NIX_AF_ERR_INVALID_MCAST_DEL_REQ; 3384 } 3385 3386 mce_list->max = mce_list->count; 3387 /* Dump the updated list to HW */ 3388 if (elem->dir == NIX_MCAST_INGRESS) 3389 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 3390 3391 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 3392 return 0; 3393 } 3394 3395 static int nix_add_mce_list_entry(struct rvu *rvu, 3396 struct nix_hw *nix_hw, 3397 struct nix_mcast_grp_elem *elem, 3398 struct nix_mcast_grp_update_req *req) 3399 { 3400 u32 num_entry = req->num_mce_entry; 3401 struct nix_mce_list *mce_list; 3402 struct hlist_node *tmp; 3403 struct mce *mce; 3404 int i; 3405 3406 mce_list = &elem->mcast_mce_list; 3407 for (i = 0; i < num_entry; i++) { 3408 mce = kzalloc_obj(*mce); 3409 if (!mce) 3410 goto free_mce; 3411 3412 mce->pcifunc = req->pcifunc[i]; 3413 mce->channel = req->channel[i]; 3414 mce->rq_rss_index = req->rq_rss_index[i]; 3415 mce->dest_type = req->dest_type[i]; 3416 mce->is_active = 1; 3417 hlist_add_head(&mce->node, &mce_list->head); 3418 mce_list->count++; 3419 } 3420 3421 mce_list->max += num_entry; 3422 3423 /* Dump the updated list to HW */ 3424 if (elem->dir == NIX_MCAST_INGRESS) 3425 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 3426 3427 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 3428 return 0; 3429 3430 free_mce: 3431 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 3432 hlist_del(&mce->node); 3433 kfree(mce); 3434 mce_list->count--; 3435 } 3436 3437 return -ENOMEM; 3438 } 3439 3440 static int nix_update_mce_list_entry(struct nix_mce_list *mce_list, 3441 u16 pcifunc, bool add) 3442 { 3443 struct mce *mce, *tail = NULL; 3444 bool delete = false; 3445 3446 /* Scan through the current list */ 3447 hlist_for_each_entry(mce, &mce_list->head, node) { 3448 /* If already exists, then delete */ 3449 if (mce->pcifunc == pcifunc && !add) { 3450 delete = true; 3451 break; 3452 } else if (mce->pcifunc == pcifunc && add) { 3453 /* entry already exists */ 3454 return 0; 3455 } 3456 tail = mce; 3457 } 3458 3459 if (delete) { 3460 hlist_del(&mce->node); 3461 kfree(mce); 3462 mce_list->count--; 3463 return 0; 3464 } 3465 3466 if (!add) 3467 return 0; 3468 3469 /* Add a new one to the list, at the tail */ 3470 mce = kzalloc_obj(*mce); 3471 if (!mce) 3472 return -ENOMEM; 3473 mce->pcifunc = pcifunc; 3474 if (!tail) 3475 hlist_add_head(&mce->node, &mce_list->head); 3476 else 3477 hlist_add_behind(&mce->node, &tail->node); 3478 mce_list->count++; 3479 return 0; 3480 } 3481 3482 int nix_update_mce_list(struct rvu *rvu, u16 pcifunc, 3483 struct nix_mce_list *mce_list, 3484 int mce_idx, int mcam_index, bool add) 3485 { 3486 int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr; 3487 struct npc_mcam *mcam = &rvu->hw->mcam; 3488 struct nix_mcast *mcast; 3489 struct nix_hw *nix_hw; 3490 struct mce *mce; 3491 3492 if (!mce_list) 3493 return -EINVAL; 3494 3495 /* Get this PF/VF func's MCE index */ 3496 idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK); 3497 3498 if (idx > (mce_idx + mce_list->max)) { 3499 dev_err(rvu->dev, 3500 "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n", 3501 __func__, idx, mce_list->max, 3502 rvu_get_pf(rvu->pdev, pcifunc)); 3503 return -EINVAL; 3504 } 3505 3506 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 3507 if (err) 3508 return err; 3509 3510 mcast = &nix_hw->mcast; 3511 mutex_lock(&mcast->mce_lock); 3512 3513 err = nix_update_mce_list_entry(mce_list, pcifunc, add); 3514 if (err) 3515 goto end; 3516 3517 /* Disable MCAM entry in NPC */ 3518 if (!mce_list->count) { 3519 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3520 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false); 3521 goto end; 3522 } 3523 3524 /* Dump the updated list to HW */ 3525 idx = mce_idx; 3526 last_idx = idx + mce_list->count - 1; 3527 hlist_for_each_entry(mce, &mce_list->head, node) { 3528 if (idx > last_idx) 3529 break; 3530 3531 next_idx = idx + 1; 3532 /* EOL should be set in last MCE */ 3533 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE, 3534 mce->pcifunc, next_idx, 3535 0, 1, 3536 (next_idx > last_idx) ? true : false); 3537 if (err) 3538 goto end; 3539 idx++; 3540 } 3541 3542 end: 3543 mutex_unlock(&mcast->mce_lock); 3544 return err; 3545 } 3546 3547 void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type, 3548 struct nix_mce_list **mce_list, int *mce_idx) 3549 { 3550 struct rvu_hwinfo *hw = rvu->hw; 3551 struct rvu_pfvf *pfvf; 3552 3553 if (!hw->cap.nix_rx_multicast || 3554 !is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, 3555 pcifunc & ~RVU_PFVF_FUNC_MASK))) { 3556 *mce_list = NULL; 3557 *mce_idx = 0; 3558 return; 3559 } 3560 3561 /* Get this PF/VF func's MCE index */ 3562 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 3563 3564 if (type == NIXLF_BCAST_ENTRY) { 3565 *mce_list = &pfvf->bcast_mce_list; 3566 *mce_idx = pfvf->bcast_mce_idx; 3567 } else if (type == NIXLF_ALLMULTI_ENTRY) { 3568 *mce_list = &pfvf->mcast_mce_list; 3569 *mce_idx = pfvf->mcast_mce_idx; 3570 } else if (type == NIXLF_PROMISC_ENTRY) { 3571 *mce_list = &pfvf->promisc_mce_list; 3572 *mce_idx = pfvf->promisc_mce_idx; 3573 } else { 3574 *mce_list = NULL; 3575 *mce_idx = 0; 3576 } 3577 } 3578 3579 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 3580 int type, bool add) 3581 { 3582 int err = 0, nixlf, blkaddr, mcam_index, mce_idx; 3583 struct npc_mcam *mcam = &rvu->hw->mcam; 3584 struct rvu_hwinfo *hw = rvu->hw; 3585 struct nix_mce_list *mce_list; 3586 int pf; 3587 3588 /* skip multicast pkt replication for AF's VFs & SDP links */ 3589 if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(rvu, pcifunc)) 3590 return 0; 3591 3592 if (!hw->cap.nix_rx_multicast) 3593 return 0; 3594 3595 pf = rvu_get_pf(rvu->pdev, pcifunc); 3596 if (!is_pf_cgxmapped(rvu, pf)) 3597 return 0; 3598 3599 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3600 if (blkaddr < 0) 3601 return -EINVAL; 3602 3603 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 3604 if (nixlf < 0) 3605 return -EINVAL; 3606 3607 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 3608 3609 mcam_index = npc_get_nixlf_mcam_index(mcam, 3610 pcifunc & ~RVU_PFVF_FUNC_MASK, 3611 nixlf, type); 3612 if (mcam_index < 0) 3613 return -EINVAL; 3614 3615 err = nix_update_mce_list(rvu, pcifunc, mce_list, 3616 mce_idx, mcam_index, add); 3617 return err; 3618 } 3619 3620 static void nix_setup_mcast_grp(struct nix_hw *nix_hw) 3621 { 3622 struct nix_mcast_grp *mcast_grp = &nix_hw->mcast_grp; 3623 3624 INIT_LIST_HEAD(&mcast_grp->mcast_grp_head); 3625 mutex_init(&mcast_grp->mcast_grp_lock); 3626 mcast_grp->next_grp_index = 1; 3627 mcast_grp->count = 0; 3628 } 3629 3630 static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw) 3631 { 3632 struct nix_mcast *mcast = &nix_hw->mcast; 3633 int err, pf, numvfs, idx; 3634 struct rvu_pfvf *pfvf; 3635 u16 pcifunc; 3636 u64 cfg; 3637 3638 /* Skip PF0 (i.e AF) */ 3639 for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) { 3640 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3641 /* If PF is not enabled, nothing to do */ 3642 if (!((cfg >> 20) & 0x01)) 3643 continue; 3644 /* Get numVFs attached to this PF */ 3645 numvfs = (cfg >> 12) & 0xFF; 3646 3647 pfvf = &rvu->pf[pf]; 3648 3649 /* This NIX0/1 block mapped to PF ? */ 3650 if (pfvf->nix_blkaddr != nix_hw->blkaddr) 3651 continue; 3652 3653 /* save start idx of broadcast mce list */ 3654 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3655 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1); 3656 3657 /* save start idx of multicast mce list */ 3658 pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3659 nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1); 3660 3661 /* save the start idx of promisc mce list */ 3662 pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS); 3663 nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1); 3664 3665 for (idx = 0; idx < (numvfs + 1); idx++) { 3666 /* idx-0 is for PF, followed by VFs */ 3667 pcifunc = rvu_make_pcifunc(rvu->pdev, pf, 0); 3668 pcifunc |= idx; 3669 /* Add dummy entries now, so that we don't have to check 3670 * for whether AQ_OP should be INIT/WRITE later on. 3671 * Will be updated when a NIXLF is attached/detached to 3672 * these PF/VFs. 3673 */ 3674 err = nix_blk_setup_mce(rvu, nix_hw, 3675 pfvf->bcast_mce_idx + idx, 3676 NIX_AQ_INSTOP_INIT, 3677 pcifunc, 0, 0, 1, true); 3678 if (err) 3679 return err; 3680 3681 /* add dummy entries to multicast mce list */ 3682 err = nix_blk_setup_mce(rvu, nix_hw, 3683 pfvf->mcast_mce_idx + idx, 3684 NIX_AQ_INSTOP_INIT, 3685 pcifunc, 0, 0, 1, true); 3686 if (err) 3687 return err; 3688 3689 /* add dummy entries to promisc mce list */ 3690 err = nix_blk_setup_mce(rvu, nix_hw, 3691 pfvf->promisc_mce_idx + idx, 3692 NIX_AQ_INSTOP_INIT, 3693 pcifunc, 0, 0, 1, true); 3694 if (err) 3695 return err; 3696 } 3697 } 3698 return 0; 3699 } 3700 3701 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3702 { 3703 struct nix_mcast *mcast = &nix_hw->mcast; 3704 struct rvu_hwinfo *hw = rvu->hw; 3705 int err, size; 3706 3707 size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F; 3708 size = BIT_ULL(size); 3709 3710 /* Allocate bitmap for rx mce entries */ 3711 mcast->mce_counter[NIX_MCAST_INGRESS].max = 256UL << MC_TBL_SIZE; 3712 err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3713 if (err) 3714 return -ENOMEM; 3715 3716 /* Allocate bitmap for tx mce entries */ 3717 mcast->mce_counter[NIX_MCAST_EGRESS].max = MC_TX_MAX; 3718 err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3719 if (err) { 3720 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3721 return -ENOMEM; 3722 } 3723 3724 /* Alloc memory for multicast/mirror replication entries */ 3725 err = qmem_alloc(rvu->dev, &mcast->mce_ctx, 3726 mcast->mce_counter[NIX_MCAST_INGRESS].max, size); 3727 if (err) { 3728 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3729 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3730 return -ENOMEM; 3731 } 3732 3733 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE, 3734 (u64)mcast->mce_ctx->iova); 3735 3736 /* Set max list length equal to max no of VFs per PF + PF itself */ 3737 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG, 3738 BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE); 3739 3740 /* Alloc memory for multicast replication buffers */ 3741 size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF; 3742 err = qmem_alloc(rvu->dev, &mcast->mcast_buf, 3743 (8UL << MC_BUF_CNT), size); 3744 if (err) { 3745 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]); 3746 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]); 3747 return -ENOMEM; 3748 } 3749 3750 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE, 3751 (u64)mcast->mcast_buf->iova); 3752 3753 /* Alloc pkind for NIX internal RX multicast/mirror replay */ 3754 mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc); 3755 3756 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG, 3757 BIT_ULL(63) | (mcast->replay_pkind << 24) | 3758 BIT_ULL(20) | MC_BUF_CNT); 3759 3760 mutex_init(&mcast->mce_lock); 3761 3762 nix_setup_mcast_grp(nix_hw); 3763 3764 return nix_setup_mce_tables(rvu, nix_hw); 3765 } 3766 3767 static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw) 3768 { 3769 struct nix_txvlan *vlan = &nix_hw->txvlan; 3770 int err; 3771 3772 /* Allocate resource bimap for tx vtag def registers*/ 3773 vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX; 3774 err = rvu_alloc_bitmap(&vlan->rsrc); 3775 if (err) 3776 return -ENOMEM; 3777 3778 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 3779 vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max, 3780 sizeof(u16), GFP_KERNEL); 3781 if (!vlan->entry2pfvf_map) 3782 goto free_mem; 3783 3784 mutex_init(&vlan->rsrc_lock); 3785 return 0; 3786 3787 free_mem: 3788 kfree(vlan->rsrc.bmap); 3789 return -ENOMEM; 3790 } 3791 3792 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3793 { 3794 struct nix_txsch *txsch; 3795 int err, lvl, schq; 3796 u64 cfg, reg; 3797 3798 /* Get scheduler queue count of each type and alloc 3799 * bitmap for each for alloc/free/attach operations. 3800 */ 3801 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 3802 txsch = &nix_hw->txsch[lvl]; 3803 txsch->lvl = lvl; 3804 switch (lvl) { 3805 case NIX_TXSCH_LVL_SMQ: 3806 reg = NIX_AF_MDQ_CONST; 3807 break; 3808 case NIX_TXSCH_LVL_TL4: 3809 reg = NIX_AF_TL4_CONST; 3810 break; 3811 case NIX_TXSCH_LVL_TL3: 3812 reg = NIX_AF_TL3_CONST; 3813 break; 3814 case NIX_TXSCH_LVL_TL2: 3815 reg = NIX_AF_TL2_CONST; 3816 break; 3817 case NIX_TXSCH_LVL_TL1: 3818 reg = NIX_AF_TL1_CONST; 3819 break; 3820 } 3821 cfg = rvu_read64(rvu, blkaddr, reg); 3822 txsch->schq.max = cfg & 0xFFFF; 3823 err = rvu_alloc_bitmap(&txsch->schq); 3824 if (err) 3825 return err; 3826 3827 /* Allocate memory for scheduler queues to 3828 * PF/VF pcifunc mapping info. 3829 */ 3830 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max, 3831 sizeof(u32), GFP_KERNEL); 3832 if (!txsch->pfvf_map) 3833 return -ENOMEM; 3834 for (schq = 0; schq < txsch->schq.max; schq++) 3835 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 3836 } 3837 3838 /* Setup a default value of 8192 as DWRR MTU */ 3839 if (rvu->hw->cap.nix_common_dwrr_mtu || 3840 rvu->hw->cap.nix_multiple_dwrr_mtu) { 3841 rvu_write64(rvu, blkaddr, 3842 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM), 3843 convert_bytes_to_dwrr_mtu(8192)); 3844 rvu_write64(rvu, blkaddr, 3845 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK), 3846 convert_bytes_to_dwrr_mtu(8192)); 3847 rvu_write64(rvu, blkaddr, 3848 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP), 3849 convert_bytes_to_dwrr_mtu(8192)); 3850 } 3851 3852 return 0; 3853 } 3854 3855 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw, 3856 int blkaddr, u32 cfg) 3857 { 3858 int fmt_idx; 3859 3860 for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) { 3861 if (nix_hw->mark_format.cfg[fmt_idx] == cfg) 3862 return fmt_idx; 3863 } 3864 if (fmt_idx >= nix_hw->mark_format.total) 3865 return -ERANGE; 3866 3867 rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg); 3868 nix_hw->mark_format.cfg[fmt_idx] = cfg; 3869 nix_hw->mark_format.in_use++; 3870 return fmt_idx; 3871 } 3872 3873 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw, 3874 int blkaddr) 3875 { 3876 u64 cfgs[] = { 3877 [NIX_MARK_CFG_IP_DSCP_RED] = 0x10003, 3878 [NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200, 3879 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203, 3880 [NIX_MARK_CFG_IP_ECN_RED] = 0x6000c, 3881 [NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00, 3882 [NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c, 3883 [NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008, 3884 [NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800, 3885 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808, 3886 }; 3887 int i, rc; 3888 u64 total; 3889 3890 total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8; 3891 nix_hw->mark_format.total = (u8)total; 3892 nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32), 3893 GFP_KERNEL); 3894 if (!nix_hw->mark_format.cfg) 3895 return -ENOMEM; 3896 for (i = 0; i < NIX_MARK_CFG_MAX; i++) { 3897 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]); 3898 if (rc < 0) 3899 dev_err(rvu->dev, "Err %d in setup mark format %d\n", 3900 i, rc); 3901 } 3902 3903 return 0; 3904 } 3905 3906 static void rvu_get_lbk_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3907 { 3908 /* CN10K supports LBK FIFO size 72 KB */ 3909 if (rvu->hw->lbk_bufsize == 0x12000) 3910 *max_mtu = CN10K_LBK_LINK_MAX_FRS; 3911 else 3912 *max_mtu = NIC_HW_MAX_FRS; 3913 } 3914 3915 static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3916 { 3917 int fifo_size = rvu_cgx_get_fifolen(rvu); 3918 3919 /* RPM supports FIFO len 128 KB and RPM2 supports double the 3920 * FIFO len to accommodate 8 LMACS 3921 */ 3922 if (fifo_size == 0x20000 || fifo_size == 0x40000) 3923 *max_mtu = CN10K_LMAC_LINK_MAX_FRS; 3924 else 3925 *max_mtu = NIC_HW_MAX_FRS; 3926 } 3927 3928 int rvu_mbox_handler_nix_get_hw_info(struct rvu *rvu, struct msg_req *req, 3929 struct nix_hw_info *rsp) 3930 { 3931 u16 pcifunc = req->hdr.pcifunc; 3932 u64 dwrr_mtu; 3933 int blkaddr; 3934 3935 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3936 if (blkaddr < 0) 3937 return NIX_AF_ERR_AF_LF_INVALID; 3938 3939 if (is_lbk_vf(rvu, pcifunc)) 3940 rvu_get_lbk_link_max_frs(rvu, &rsp->max_mtu); 3941 else 3942 rvu_get_lmac_link_max_frs(rvu, &rsp->max_mtu); 3943 3944 rsp->min_mtu = NIC_HW_MIN_FRS; 3945 3946 if (!rvu->hw->cap.nix_common_dwrr_mtu && 3947 !rvu->hw->cap.nix_multiple_dwrr_mtu) { 3948 /* Return '1' on OTx2 */ 3949 rsp->rpm_dwrr_mtu = 1; 3950 rsp->sdp_dwrr_mtu = 1; 3951 rsp->lbk_dwrr_mtu = 1; 3952 return 0; 3953 } 3954 3955 /* Return DWRR_MTU for TLx_SCHEDULE[RR_WEIGHT] config */ 3956 dwrr_mtu = rvu_read64(rvu, blkaddr, 3957 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM)); 3958 rsp->rpm_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3959 3960 dwrr_mtu = rvu_read64(rvu, blkaddr, 3961 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP)); 3962 rsp->sdp_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3963 3964 dwrr_mtu = rvu_read64(rvu, blkaddr, 3965 nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK)); 3966 rsp->lbk_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3967 3968 return 0; 3969 } 3970 3971 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req, 3972 struct msg_rsp *rsp) 3973 { 3974 u16 pcifunc = req->hdr.pcifunc; 3975 int i, nixlf, blkaddr, err; 3976 u64 stats; 3977 3978 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3979 if (err) 3980 return err; 3981 3982 /* Get stats count supported by HW */ 3983 stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 3984 3985 /* Reset tx stats */ 3986 for (i = 0; i < ((stats >> 24) & 0xFF); i++) 3987 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0); 3988 3989 /* Reset rx stats */ 3990 for (i = 0; i < ((stats >> 32) & 0xFF); i++) 3991 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0); 3992 3993 return 0; 3994 } 3995 3996 /* Returns the ALG index to be set into NPC_RX_ACTION */ 3997 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg) 3998 { 3999 int i; 4000 4001 /* Scan over exiting algo entries to find a match */ 4002 for (i = 0; i < nix_hw->flowkey.in_use; i++) 4003 if (nix_hw->flowkey.flowkey[i] == flow_cfg) 4004 return i; 4005 4006 return -ERANGE; 4007 } 4008 4009 /* Mask to match ipv6(NPC_LT_LC_IP6) and ipv6 ext(NPC_LT_LC_IP6_EXT) */ 4010 #define NPC_LT_LC_IP6_MATCH_MSK ((~(NPC_LT_LC_IP6 ^ NPC_LT_LC_IP6_EXT)) & 0xf) 4011 /* Mask to match both ipv4(NPC_LT_LC_IP) and ipv4 ext(NPC_LT_LC_IP_OPT) */ 4012 #define NPC_LT_LC_IP_MATCH_MSK ((~(NPC_LT_LC_IP ^ NPC_LT_LC_IP_OPT)) & 0xf) 4013 4014 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg) 4015 { 4016 int idx, nr_field, key_off, field_marker, keyoff_marker; 4017 int max_key_off, max_bit_pos, group_member; 4018 struct nix_rx_flowkey_alg *field; 4019 struct nix_rx_flowkey_alg tmp; 4020 u32 key_type, valid_key; 4021 u32 l3_l4_src_dst; 4022 int l4_key_offset = 0; 4023 4024 if (!alg) 4025 return -EINVAL; 4026 4027 #define FIELDS_PER_ALG 5 4028 #define MAX_KEY_OFF 40 4029 /* Clear all fields */ 4030 memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG); 4031 4032 /* Each of the 32 possible flow key algorithm definitions should 4033 * fall into above incremental config (except ALG0). Otherwise a 4034 * single NPC MCAM entry is not sufficient for supporting RSS. 4035 * 4036 * If a different definition or combination needed then NPC MCAM 4037 * has to be programmed to filter such pkts and it's action should 4038 * point to this definition to calculate flowtag or hash. 4039 * 4040 * The `for loop` goes over _all_ protocol field and the following 4041 * variables depicts the state machine forward progress logic. 4042 * 4043 * keyoff_marker - Enabled when hash byte length needs to be accounted 4044 * in field->key_offset update. 4045 * field_marker - Enabled when a new field needs to be selected. 4046 * group_member - Enabled when protocol is part of a group. 4047 */ 4048 4049 /* Last 4 bits (31:28) are reserved to specify SRC, DST 4050 * selection for L3, L4 i.e IPV[4,6]_SRC, IPV[4,6]_DST, 4051 * [TCP,UDP,SCTP]_SRC, [TCP,UDP,SCTP]_DST 4052 * 31 => L3_SRC, 30 => L3_DST, 29 => L4_SRC, 28 => L4_DST 4053 */ 4054 l3_l4_src_dst = flow_cfg; 4055 /* Reset these 4 bits, so that these won't be part of key */ 4056 flow_cfg &= NIX_FLOW_KEY_TYPE_L3_L4_MASK; 4057 4058 keyoff_marker = 0; max_key_off = 0; group_member = 0; 4059 nr_field = 0; key_off = 0; field_marker = 1; 4060 field = &tmp; max_bit_pos = fls(flow_cfg); 4061 for (idx = 0; 4062 idx < max_bit_pos && nr_field < FIELDS_PER_ALG && 4063 key_off < MAX_KEY_OFF; idx++) { 4064 key_type = BIT(idx); 4065 valid_key = flow_cfg & key_type; 4066 /* Found a field marker, reset the field values */ 4067 if (field_marker) 4068 memset(&tmp, 0, sizeof(tmp)); 4069 4070 field_marker = true; 4071 keyoff_marker = true; 4072 switch (key_type) { 4073 case NIX_FLOW_KEY_TYPE_PORT: 4074 field->sel_chan = true; 4075 /* This should be set to 1, when SEL_CHAN is set */ 4076 field->bytesm1 = 1; 4077 break; 4078 case NIX_FLOW_KEY_TYPE_IPV4_PROTO: 4079 field->lid = NPC_LID_LC; 4080 field->hdr_offset = 9; /* offset */ 4081 field->bytesm1 = 0; /* 1 byte */ 4082 field->ltype_match = NPC_LT_LC_IP; 4083 field->ltype_mask = NPC_LT_LC_IP_MATCH_MSK; 4084 break; 4085 case NIX_FLOW_KEY_TYPE_IPV4: 4086 case NIX_FLOW_KEY_TYPE_INNR_IPV4: 4087 field->lid = NPC_LID_LC; 4088 field->ltype_match = NPC_LT_LC_IP; 4089 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) { 4090 field->lid = NPC_LID_LG; 4091 field->ltype_match = NPC_LT_LG_TU_IP; 4092 } 4093 field->hdr_offset = 12; /* SIP offset */ 4094 field->bytesm1 = 7; /* SIP + DIP, 8 bytes */ 4095 4096 /* Only SIP */ 4097 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY) 4098 field->bytesm1 = 3; /* SIP, 4 bytes */ 4099 4100 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) { 4101 /* Both SIP + DIP */ 4102 if (field->bytesm1 == 3) { 4103 field->bytesm1 = 7; /* SIP + DIP, 8B */ 4104 } else { 4105 /* Only DIP */ 4106 field->hdr_offset = 16; /* DIP off */ 4107 field->bytesm1 = 3; /* DIP, 4 bytes */ 4108 } 4109 } 4110 field->ltype_mask = NPC_LT_LC_IP_MATCH_MSK; 4111 keyoff_marker = false; 4112 break; 4113 case NIX_FLOW_KEY_TYPE_IPV6: 4114 case NIX_FLOW_KEY_TYPE_INNR_IPV6: 4115 field->lid = NPC_LID_LC; 4116 field->ltype_match = NPC_LT_LC_IP6; 4117 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) { 4118 field->lid = NPC_LID_LG; 4119 field->ltype_match = NPC_LT_LG_TU_IP6; 4120 } 4121 field->hdr_offset = 8; /* SIP offset */ 4122 field->bytesm1 = 31; /* SIP + DIP, 32 bytes */ 4123 4124 /* Only SIP */ 4125 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY) 4126 field->bytesm1 = 15; /* SIP, 16 bytes */ 4127 4128 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) { 4129 /* Both SIP + DIP */ 4130 if (field->bytesm1 == 15) { 4131 /* SIP + DIP, 32 bytes */ 4132 field->bytesm1 = 31; 4133 } else { 4134 /* Only DIP */ 4135 field->hdr_offset = 24; /* DIP off */ 4136 field->bytesm1 = 15; /* DIP,16 bytes */ 4137 } 4138 } 4139 field->ltype_mask = NPC_LT_LC_IP6_MATCH_MSK; 4140 break; 4141 case NIX_FLOW_KEY_TYPE_TCP: 4142 case NIX_FLOW_KEY_TYPE_UDP: 4143 case NIX_FLOW_KEY_TYPE_SCTP: 4144 case NIX_FLOW_KEY_TYPE_INNR_TCP: 4145 case NIX_FLOW_KEY_TYPE_INNR_UDP: 4146 case NIX_FLOW_KEY_TYPE_INNR_SCTP: 4147 field->lid = NPC_LID_LD; 4148 if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP || 4149 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP || 4150 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) 4151 field->lid = NPC_LID_LH; 4152 field->bytesm1 = 3; /* Sport + Dport, 4 bytes */ 4153 4154 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_SRC_ONLY) 4155 field->bytesm1 = 1; /* SRC, 2 bytes */ 4156 4157 if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_DST_ONLY) { 4158 /* Both SRC + DST */ 4159 if (field->bytesm1 == 1) { 4160 /* SRC + DST, 4 bytes */ 4161 field->bytesm1 = 3; 4162 } else { 4163 /* Only DIP */ 4164 field->hdr_offset = 2; /* DST off */ 4165 field->bytesm1 = 1; /* DST, 2 bytes */ 4166 } 4167 } 4168 4169 /* Enum values for NPC_LID_LD and NPC_LID_LG are same, 4170 * so no need to change the ltype_match, just change 4171 * the lid for inner protocols 4172 */ 4173 BUILD_BUG_ON((int)NPC_LT_LD_TCP != 4174 (int)NPC_LT_LH_TU_TCP); 4175 BUILD_BUG_ON((int)NPC_LT_LD_UDP != 4176 (int)NPC_LT_LH_TU_UDP); 4177 BUILD_BUG_ON((int)NPC_LT_LD_SCTP != 4178 (int)NPC_LT_LH_TU_SCTP); 4179 4180 if ((key_type == NIX_FLOW_KEY_TYPE_TCP || 4181 key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) && 4182 valid_key) { 4183 field->ltype_match |= NPC_LT_LD_TCP; 4184 group_member = true; 4185 } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP || 4186 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) && 4187 valid_key) { 4188 field->ltype_match |= NPC_LT_LD_UDP; 4189 group_member = true; 4190 } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP || 4191 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) && 4192 valid_key) { 4193 field->ltype_match |= NPC_LT_LD_SCTP; 4194 group_member = true; 4195 } 4196 field->ltype_mask = ~field->ltype_match; 4197 if (key_type == NIX_FLOW_KEY_TYPE_SCTP || 4198 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) { 4199 /* Handle the case where any of the group item 4200 * is enabled in the group but not the final one 4201 */ 4202 if (group_member) { 4203 valid_key = true; 4204 group_member = false; 4205 } 4206 } else { 4207 field_marker = false; 4208 keyoff_marker = false; 4209 } 4210 4211 /* TCP/UDP/SCTP and ESP/AH falls at same offset so 4212 * remember the TCP key offset of 40 byte hash key. 4213 */ 4214 if (key_type == NIX_FLOW_KEY_TYPE_TCP) 4215 l4_key_offset = key_off; 4216 break; 4217 case NIX_FLOW_KEY_TYPE_NVGRE: 4218 field->lid = NPC_LID_LD; 4219 field->hdr_offset = 4; /* VSID offset */ 4220 field->bytesm1 = 2; 4221 field->ltype_match = NPC_LT_LD_NVGRE; 4222 field->ltype_mask = 0xF; 4223 break; 4224 case NIX_FLOW_KEY_TYPE_VXLAN: 4225 case NIX_FLOW_KEY_TYPE_GENEVE: 4226 field->lid = NPC_LID_LE; 4227 field->bytesm1 = 2; 4228 field->hdr_offset = 4; 4229 field->ltype_mask = 0xF; 4230 field_marker = false; 4231 keyoff_marker = false; 4232 4233 if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) { 4234 field->ltype_match |= NPC_LT_LE_VXLAN; 4235 group_member = true; 4236 } 4237 4238 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) { 4239 field->ltype_match |= NPC_LT_LE_GENEVE; 4240 group_member = true; 4241 } 4242 4243 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) { 4244 if (group_member) { 4245 field->ltype_mask = ~field->ltype_match; 4246 field_marker = true; 4247 keyoff_marker = true; 4248 valid_key = true; 4249 group_member = false; 4250 } 4251 } 4252 break; 4253 case NIX_FLOW_KEY_TYPE_ETH_DMAC: 4254 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC: 4255 field->lid = NPC_LID_LA; 4256 field->ltype_match = NPC_LT_LA_ETHER; 4257 if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) { 4258 field->lid = NPC_LID_LF; 4259 field->ltype_match = NPC_LT_LF_TU_ETHER; 4260 } 4261 field->hdr_offset = 0; 4262 field->bytesm1 = 5; /* DMAC 6 Byte */ 4263 field->ltype_mask = 0xF; 4264 break; 4265 case NIX_FLOW_KEY_TYPE_IPV6_EXT: 4266 field->lid = NPC_LID_LC; 4267 field->hdr_offset = 40; /* IPV6 hdr */ 4268 field->bytesm1 = 0; /* 1 Byte ext hdr*/ 4269 field->ltype_match = NPC_LT_LC_IP6_EXT; 4270 field->ltype_mask = 0xF; 4271 break; 4272 case NIX_FLOW_KEY_TYPE_GTPU: 4273 field->lid = NPC_LID_LE; 4274 field->hdr_offset = 4; 4275 field->bytesm1 = 3; /* 4 bytes TID*/ 4276 field->ltype_match = NPC_LT_LE_GTPU; 4277 field->ltype_mask = 0xF; 4278 break; 4279 case NIX_FLOW_KEY_TYPE_CUSTOM0: 4280 field->lid = NPC_LID_LC; 4281 field->hdr_offset = 6; 4282 field->bytesm1 = 1; /* 2 Bytes*/ 4283 field->ltype_match = NPC_LT_LC_CUSTOM0; 4284 field->ltype_mask = 0xF; 4285 break; 4286 case NIX_FLOW_KEY_TYPE_VLAN: 4287 field->lid = NPC_LID_LB; 4288 field->hdr_offset = 2; /* Skip TPID (2-bytes) */ 4289 field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */ 4290 field->ltype_match = NPC_LT_LB_CTAG; 4291 field->ltype_mask = 0xF; 4292 field->fn_mask = 1; /* Mask out the first nibble */ 4293 break; 4294 case NIX_FLOW_KEY_TYPE_AH: 4295 case NIX_FLOW_KEY_TYPE_ESP: 4296 field->hdr_offset = 0; 4297 field->bytesm1 = 7; /* SPI + sequence number */ 4298 field->ltype_mask = 0xF; 4299 field->lid = NPC_LID_LE; 4300 field->ltype_match = NPC_LT_LE_ESP; 4301 if (key_type == NIX_FLOW_KEY_TYPE_AH) { 4302 field->lid = NPC_LID_LD; 4303 field->ltype_match = NPC_LT_LD_AH; 4304 field->hdr_offset = 4; 4305 keyoff_marker = false; 4306 } 4307 break; 4308 } 4309 field->ena = 1; 4310 4311 /* Found a valid flow key type */ 4312 if (valid_key) { 4313 /* Use the key offset of TCP/UDP/SCTP fields 4314 * for ESP/AH fields. 4315 */ 4316 if (key_type == NIX_FLOW_KEY_TYPE_ESP || 4317 key_type == NIX_FLOW_KEY_TYPE_AH) 4318 key_off = l4_key_offset; 4319 field->key_offset = key_off; 4320 memcpy(&alg[nr_field], field, sizeof(*field)); 4321 max_key_off = max(max_key_off, field->bytesm1 + 1); 4322 4323 /* Found a field marker, get the next field */ 4324 if (field_marker) 4325 nr_field++; 4326 } 4327 4328 /* Found a keyoff marker, update the new key_off */ 4329 if (keyoff_marker) { 4330 key_off += max_key_off; 4331 max_key_off = 0; 4332 } 4333 } 4334 /* Processed all the flow key types */ 4335 if (idx == max_bit_pos && key_off <= MAX_KEY_OFF) 4336 return 0; 4337 else 4338 return NIX_AF_ERR_RSS_NOSPC_FIELD; 4339 } 4340 4341 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg) 4342 { 4343 u64 field[FIELDS_PER_ALG]; 4344 struct nix_hw *hw; 4345 int fid, rc; 4346 4347 hw = get_nix_hw(rvu->hw, blkaddr); 4348 if (!hw) 4349 return NIX_AF_ERR_INVALID_NIXBLK; 4350 4351 /* No room to add new flow hash algoritham */ 4352 if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX) 4353 return NIX_AF_ERR_RSS_NOSPC_ALGO; 4354 4355 /* Generate algo fields for the given flow_cfg */ 4356 rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg); 4357 if (rc) 4358 return rc; 4359 4360 /* Update ALGX_FIELDX register with generated fields */ 4361 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 4362 rvu_write64(rvu, blkaddr, 4363 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use, 4364 fid), field[fid]); 4365 4366 /* Store the flow_cfg for futher lookup */ 4367 rc = hw->flowkey.in_use; 4368 hw->flowkey.flowkey[rc] = flow_cfg; 4369 hw->flowkey.in_use++; 4370 4371 return rc; 4372 } 4373 4374 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu, 4375 struct nix_rss_flowkey_cfg *req, 4376 struct nix_rss_flowkey_cfg_rsp *rsp) 4377 { 4378 u16 pcifunc = req->hdr.pcifunc; 4379 int alg_idx, nixlf, blkaddr; 4380 struct nix_hw *nix_hw; 4381 int err; 4382 4383 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4384 if (err) 4385 return err; 4386 4387 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4388 if (!nix_hw) 4389 return NIX_AF_ERR_INVALID_NIXBLK; 4390 4391 alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg); 4392 /* Failed to get algo index from the exiting list, reserve new */ 4393 if (alg_idx < 0) { 4394 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr, 4395 req->flowkey_cfg); 4396 if (alg_idx < 0) 4397 return alg_idx; 4398 } 4399 rsp->alg_idx = alg_idx; 4400 rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group, 4401 alg_idx, req->mcam_index); 4402 return 0; 4403 } 4404 4405 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr) 4406 { 4407 u32 flowkey_cfg, minkey_cfg; 4408 int alg, fid, rc; 4409 4410 /* Disable all flow key algx fieldx */ 4411 for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) { 4412 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 4413 rvu_write64(rvu, blkaddr, 4414 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid), 4415 0); 4416 } 4417 4418 /* IPv4/IPv6 SIP/DIPs */ 4419 flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6; 4420 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4421 if (rc < 0) 4422 return rc; 4423 4424 /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4425 minkey_cfg = flowkey_cfg; 4426 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP; 4427 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4428 if (rc < 0) 4429 return rc; 4430 4431 /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4432 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP; 4433 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4434 if (rc < 0) 4435 return rc; 4436 4437 /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 4438 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP; 4439 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4440 if (rc < 0) 4441 return rc; 4442 4443 /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4444 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4445 NIX_FLOW_KEY_TYPE_UDP; 4446 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4447 if (rc < 0) 4448 return rc; 4449 4450 /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4451 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4452 NIX_FLOW_KEY_TYPE_SCTP; 4453 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4454 if (rc < 0) 4455 return rc; 4456 4457 /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4458 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP | 4459 NIX_FLOW_KEY_TYPE_SCTP; 4460 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4461 if (rc < 0) 4462 return rc; 4463 4464 /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 4465 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 4466 NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP; 4467 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 4468 if (rc < 0) 4469 return rc; 4470 4471 return 0; 4472 } 4473 4474 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu, 4475 struct nix_set_mac_addr *req, 4476 struct msg_rsp *rsp) 4477 { 4478 bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK; 4479 u16 pcifunc = req->hdr.pcifunc; 4480 int blkaddr, nixlf, err; 4481 struct rvu_pfvf *pfvf; 4482 4483 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 4484 if (err) 4485 return err; 4486 4487 pfvf = rvu_get_pfvf(rvu, pcifunc); 4488 4489 /* untrusted VF can't overwrite admin(PF) changes */ 4490 if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 4491 (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) { 4492 dev_warn(rvu->dev, 4493 "MAC address set by admin(PF) cannot be overwritten by untrusted VF"); 4494 return -EPERM; 4495 } 4496 4497 ether_addr_copy(pfvf->mac_addr, req->mac_addr); 4498 4499 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 4500 pfvf->rx_chan_base, req->mac_addr); 4501 4502 if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf) 4503 ether_addr_copy(pfvf->default_mac, req->mac_addr); 4504 4505 return 0; 4506 } 4507 4508 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu, 4509 struct msg_req *req, 4510 struct nix_get_mac_addr_rsp *rsp) 4511 { 4512 u16 pcifunc = req->hdr.pcifunc; 4513 struct rvu_pfvf *pfvf; 4514 4515 if (!is_nixlf_attached(rvu, pcifunc)) 4516 return NIX_AF_ERR_AF_LF_INVALID; 4517 4518 pfvf = rvu_get_pfvf(rvu, pcifunc); 4519 4520 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 4521 4522 return 0; 4523 } 4524 4525 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, 4526 struct msg_rsp *rsp) 4527 { 4528 bool allmulti, promisc, nix_rx_multicast; 4529 u16 pcifunc = req->hdr.pcifunc; 4530 struct rvu_pfvf *pfvf; 4531 int nixlf, err; 4532 4533 pfvf = rvu_get_pfvf(rvu, pcifunc); 4534 promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false; 4535 allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false; 4536 pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false; 4537 4538 nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list; 4539 4540 if (is_vf(pcifunc) && !nix_rx_multicast && 4541 (promisc || allmulti)) { 4542 dev_warn_ratelimited(rvu->dev, 4543 "VF promisc/multicast not supported\n"); 4544 return 0; 4545 } 4546 4547 /* untrusted VF can't configure promisc/allmulti */ 4548 if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 4549 (promisc || allmulti)) 4550 return 0; 4551 4552 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4553 if (err) 4554 return err; 4555 4556 if (nix_rx_multicast) { 4557 /* add/del this PF_FUNC to/from mcast pkt replication list */ 4558 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY, 4559 allmulti); 4560 if (err) { 4561 dev_err(rvu->dev, 4562 "Failed to update pcifunc 0x%x to multicast list\n", 4563 pcifunc); 4564 return err; 4565 } 4566 4567 /* add/del this PF_FUNC to/from promisc pkt replication list */ 4568 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY, 4569 promisc); 4570 if (err) { 4571 dev_err(rvu->dev, 4572 "Failed to update pcifunc 0x%x to promisc list\n", 4573 pcifunc); 4574 return err; 4575 } 4576 } 4577 4578 /* install/uninstall allmulti entry */ 4579 if (allmulti) { 4580 rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf, 4581 pfvf->rx_chan_base); 4582 } else { 4583 if (!nix_rx_multicast) 4584 rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false); 4585 } 4586 4587 /* install/uninstall promisc entry */ 4588 if (promisc) 4589 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 4590 pfvf->rx_chan_base, 4591 pfvf->rx_chan_cnt); 4592 else 4593 if (!nix_rx_multicast) 4594 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); 4595 4596 return 0; 4597 } 4598 4599 static void nix_find_link_frs(struct rvu *rvu, 4600 struct nix_frs_cfg *req, u16 pcifunc) 4601 { 4602 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4603 struct rvu_pfvf *pfvf; 4604 int maxlen, minlen; 4605 int numvfs, hwvf; 4606 int vf; 4607 4608 /* Update with requester's min/max lengths */ 4609 pfvf = rvu_get_pfvf(rvu, pcifunc); 4610 pfvf->maxlen = req->maxlen; 4611 if (req->update_minlen) 4612 pfvf->minlen = req->minlen; 4613 4614 maxlen = req->maxlen; 4615 minlen = req->update_minlen ? req->minlen : 0; 4616 4617 /* Get this PF's numVFs and starting hwvf */ 4618 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf); 4619 4620 /* For each VF, compare requested max/minlen */ 4621 for (vf = 0; vf < numvfs; vf++) { 4622 pfvf = &rvu->hwvf[hwvf + vf]; 4623 if (pfvf->maxlen > maxlen) 4624 maxlen = pfvf->maxlen; 4625 if (req->update_minlen && 4626 pfvf->minlen && pfvf->minlen < minlen) 4627 minlen = pfvf->minlen; 4628 } 4629 4630 /* Compare requested max/minlen with PF's max/minlen */ 4631 pfvf = &rvu->pf[pf]; 4632 if (pfvf->maxlen > maxlen) 4633 maxlen = pfvf->maxlen; 4634 if (req->update_minlen && 4635 pfvf->minlen && pfvf->minlen < minlen) 4636 minlen = pfvf->minlen; 4637 4638 /* Update the request with max/min PF's and it's VF's max/min */ 4639 req->maxlen = maxlen; 4640 if (req->update_minlen) 4641 req->minlen = minlen; 4642 } 4643 4644 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req, 4645 struct msg_rsp *rsp) 4646 { 4647 struct rvu_hwinfo *hw = rvu->hw; 4648 u16 pcifunc = req->hdr.pcifunc; 4649 int pf = rvu_get_pf(rvu->pdev, pcifunc); 4650 int blkaddr, link = -1; 4651 struct nix_hw *nix_hw; 4652 struct rvu_pfvf *pfvf; 4653 u8 cgx = 0, lmac = 0; 4654 u16 max_mtu; 4655 u64 cfg; 4656 4657 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 4658 if (blkaddr < 0) 4659 return NIX_AF_ERR_AF_LF_INVALID; 4660 4661 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4662 if (!nix_hw) 4663 return NIX_AF_ERR_INVALID_NIXBLK; 4664 4665 if (is_lbk_vf(rvu, pcifunc) || is_rep_dev(rvu, pcifunc)) 4666 rvu_get_lbk_link_max_frs(rvu, &max_mtu); 4667 else 4668 rvu_get_lmac_link_max_frs(rvu, &max_mtu); 4669 4670 if (!req->sdp_link && req->maxlen > max_mtu) 4671 return NIX_AF_ERR_FRS_INVALID; 4672 4673 if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS) 4674 return NIX_AF_ERR_FRS_INVALID; 4675 4676 /* Check if config is for SDP link */ 4677 if (req->sdp_link) { 4678 if (!hw->sdp_links) 4679 return NIX_AF_ERR_RX_LINK_INVALID; 4680 link = hw->cgx_links + hw->lbk_links; 4681 goto linkcfg; 4682 } 4683 4684 /* Check if the request is from CGX mapped RVU PF */ 4685 if (is_pf_cgxmapped(rvu, pf)) { 4686 /* Get CGX and LMAC to which this PF is mapped and find link */ 4687 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac); 4688 link = (cgx * hw->lmac_per_cgx) + lmac; 4689 } else if (pf == 0) { 4690 /* For VFs of PF0 ingress is LBK port, so config LBK link */ 4691 pfvf = rvu_get_pfvf(rvu, pcifunc); 4692 link = hw->cgx_links + pfvf->lbkid; 4693 } else if (is_rep_dev(rvu, pcifunc)) { 4694 link = hw->cgx_links + 0; 4695 } 4696 4697 if (link < 0) 4698 return NIX_AF_ERR_RX_LINK_INVALID; 4699 4700 linkcfg: 4701 nix_find_link_frs(rvu, req, pcifunc); 4702 4703 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link)); 4704 cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16); 4705 if (req->update_minlen) 4706 cfg = (cfg & ~0xFFFFULL) | req->minlen; 4707 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg); 4708 4709 return 0; 4710 } 4711 4712 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req, 4713 struct msg_rsp *rsp) 4714 { 4715 int nixlf, blkaddr, err; 4716 u64 cfg; 4717 4718 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr); 4719 if (err) 4720 return err; 4721 4722 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf)); 4723 /* Set the interface configuration */ 4724 if (req->len_verify & BIT(0)) 4725 cfg |= BIT_ULL(41); 4726 else 4727 cfg &= ~BIT_ULL(41); 4728 4729 if (req->len_verify & BIT(1)) 4730 cfg |= BIT_ULL(40); 4731 else 4732 cfg &= ~BIT_ULL(40); 4733 4734 if (req->len_verify & NIX_RX_DROP_RE) 4735 cfg |= BIT_ULL(32); 4736 else 4737 cfg &= ~BIT_ULL(32); 4738 4739 if (req->csum_verify & BIT(0)) 4740 cfg |= BIT_ULL(37); 4741 else 4742 cfg &= ~BIT_ULL(37); 4743 4744 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg); 4745 4746 return 0; 4747 } 4748 4749 static u64 rvu_get_lbk_link_credits(struct rvu *rvu, u16 lbk_max_frs) 4750 { 4751 return 1600; /* 16 * max LBK datarate = 16 * 100Gbps */ 4752 } 4753 4754 static void nix_link_config(struct rvu *rvu, int blkaddr, 4755 struct nix_hw *nix_hw) 4756 { 4757 struct rvu_hwinfo *hw = rvu->hw; 4758 int cgx, lmac_cnt, slink, link; 4759 u16 lbk_max_frs, lmac_max_frs; 4760 unsigned long lmac_bmap; 4761 u64 tx_credits, cfg; 4762 u64 lmac_fifo_len; 4763 int iter; 4764 4765 rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs); 4766 rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs); 4767 4768 /* Set SDP link credit */ 4769 rvu_write64(rvu, blkaddr, NIX_AF_SDP_LINK_CREDIT, SDP_LINK_CREDIT); 4770 4771 /* Set default min/max packet lengths allowed on NIX Rx links. 4772 * 4773 * With HW reset minlen value of 60byte, HW will treat ARP pkts 4774 * as undersize and report them to SW as error pkts, hence 4775 * setting it to 40 bytes. 4776 */ 4777 for (link = 0; link < hw->cgx_links; link++) { 4778 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4779 ((u64)lmac_max_frs << 16) | NIC_HW_MIN_FRS); 4780 } 4781 4782 for (link = hw->cgx_links; link < hw->lbk_links; link++) { 4783 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4784 ((u64)lbk_max_frs << 16) | NIC_HW_MIN_FRS); 4785 } 4786 if (hw->sdp_links) { 4787 link = hw->cgx_links + hw->lbk_links; 4788 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4789 SDP_HW_MAX_FRS << 16 | SDP_HW_MIN_FRS); 4790 } 4791 4792 /* Get MCS external bypass status for CN10K-B */ 4793 if (mcs_get_blkcnt() == 1) { 4794 /* Adjust for 2 credits when external bypass is disabled */ 4795 nix_hw->cc_mcs_cnt = is_mcs_bypass(0) ? 0 : 2; 4796 } 4797 4798 /* Set credits for Tx links assuming max packet length allowed. 4799 * This will be reconfigured based on MTU set for PF/VF. 4800 */ 4801 for (cgx = 0; cgx < hw->cgx; cgx++) { 4802 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu)); 4803 /* Skip when cgx is not available or lmac cnt is zero */ 4804 if (lmac_cnt <= 0) 4805 continue; 4806 slink = cgx * hw->lmac_per_cgx; 4807 4808 /* Get LMAC id's from bitmap */ 4809 lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); 4810 for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) { 4811 lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); 4812 if (!lmac_fifo_len) { 4813 dev_err(rvu->dev, 4814 "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", 4815 __func__, cgx, iter); 4816 continue; 4817 } 4818 tx_credits = (lmac_fifo_len - lmac_max_frs) / 16; 4819 /* Enable credits and set credit pkt count to max allowed */ 4820 cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4821 cfg |= FIELD_PREP(NIX_AF_LINKX_MCS_CNT_MASK, nix_hw->cc_mcs_cnt); 4822 4823 link = iter + slink; 4824 nix_hw->tx_credits[link] = tx_credits; 4825 rvu_write64(rvu, blkaddr, 4826 NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); 4827 } 4828 } 4829 4830 /* Set Tx credits for LBK link */ 4831 slink = hw->cgx_links; 4832 for (link = slink; link < (slink + hw->lbk_links); link++) { 4833 tx_credits = rvu_get_lbk_link_credits(rvu, lbk_max_frs); 4834 nix_hw->tx_credits[link] = tx_credits; 4835 /* Enable credits and set credit pkt count to max allowed */ 4836 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4837 rvu_write64(rvu, blkaddr, 4838 NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits); 4839 } 4840 } 4841 4842 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr) 4843 { 4844 int idx, err; 4845 u64 status; 4846 4847 /* Start X2P bus calibration */ 4848 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4849 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9)); 4850 /* Wait for calibration to complete */ 4851 err = rvu_poll_reg(rvu, blkaddr, 4852 NIX_AF_STATUS, BIT_ULL(10), false); 4853 if (err) { 4854 dev_err(rvu->dev, "NIX X2P bus calibration failed\n"); 4855 return err; 4856 } 4857 4858 status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS); 4859 /* Check if CGX devices are ready */ 4860 for (idx = 0; idx < rvu->cgx_cnt_max; idx++) { 4861 /* Skip when cgx port is not available */ 4862 if (!rvu_cgx_pdata(idx, rvu) || 4863 (status & (BIT_ULL(16 + idx)))) 4864 continue; 4865 dev_err(rvu->dev, 4866 "CGX%d didn't respond to NIX X2P calibration\n", idx); 4867 err = -EBUSY; 4868 } 4869 4870 /* Check if LBK is ready */ 4871 if (!(status & BIT_ULL(19))) { 4872 dev_err(rvu->dev, 4873 "LBK didn't respond to NIX X2P calibration\n"); 4874 err = -EBUSY; 4875 } 4876 4877 /* Clear 'calibrate_x2p' bit */ 4878 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4879 rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9)); 4880 if (err || (status & 0x3FFULL)) 4881 dev_err(rvu->dev, 4882 "NIX X2P calibration failed, status 0x%llx\n", status); 4883 if (err) 4884 return err; 4885 return 0; 4886 } 4887 4888 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block) 4889 { 4890 u64 cfg; 4891 int err; 4892 4893 /* Set admin queue endianness */ 4894 cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG); 4895 #ifdef __BIG_ENDIAN 4896 cfg |= BIT_ULL(8); 4897 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4898 #else 4899 cfg &= ~BIT_ULL(8); 4900 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4901 #endif 4902 4903 /* Do not bypass NDC cache */ 4904 cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG); 4905 cfg &= ~0x3FFEULL; 4906 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 4907 /* Disable caching of SQB aka SQEs */ 4908 cfg |= 0x04ULL; 4909 #endif 4910 rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg); 4911 4912 /* Result structure can be followed by RQ/SQ/CQ context at 4913 * RES + 128bytes and a write mask at RES + 256 bytes, depending on 4914 * operation type. Alloc sufficient result memory for all operations. 4915 */ 4916 err = rvu_aq_alloc(rvu, &block->aq, 4917 Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s), 4918 ALIGN(sizeof(struct nix_aq_res_s), 128) + 256); 4919 if (err) 4920 return err; 4921 4922 rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE); 4923 rvu_write64(rvu, block->addr, 4924 NIX_AF_AQ_BASE, (u64)block->aq->inst->iova); 4925 return 0; 4926 } 4927 4928 static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr) 4929 { 4930 struct rvu_hwinfo *hw = rvu->hw; 4931 u64 hw_const; 4932 4933 hw_const = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 4934 4935 /* On OcteonTx2 DWRR quantum is directly configured into each of 4936 * the transmit scheduler queues. And PF/VF drivers were free to 4937 * config any value upto 2^24. 4938 * On CN10K, HW is modified, the quantum configuration at scheduler 4939 * queues is in terms of weight. And SW needs to setup a base DWRR MTU 4940 * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do 4941 * 'DWRR MTU * weight' to get the quantum. 4942 * 4943 * Check if HW uses a common MTU for all DWRR quantum configs. 4944 * On OcteonTx2 this register field is '0'. 4945 */ 4946 if ((((hw_const >> 56) & 0x10) == 0x10) && !(hw_const & BIT_ULL(61))) 4947 hw->cap.nix_common_dwrr_mtu = true; 4948 4949 if (hw_const & BIT_ULL(61)) 4950 hw->cap.nix_multiple_dwrr_mtu = true; 4951 } 4952 4953 static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) 4954 { 4955 const struct npc_lt_def_cfg *ltdefs; 4956 struct rvu_hwinfo *hw = rvu->hw; 4957 int blkaddr = nix_hw->blkaddr; 4958 struct rvu_block *block; 4959 int err; 4960 u64 cfg; 4961 4962 block = &hw->block[blkaddr]; 4963 4964 if (is_rvu_96xx_B0(rvu)) { 4965 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt 4966 * internal state when conditional clocks are turned off. 4967 * Hence enable them. 4968 */ 4969 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4970 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL); 4971 } 4972 4973 /* Set chan/link to backpressure TL3 instead of TL2 */ 4974 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); 4975 4976 /* Disable SQ manager's sticky mode operation (set TM6 = 0, TM11 = 0) 4977 * This sticky mode is known to cause SQ stalls when multiple 4978 * SQs are mapped to same SMQ and transmitting pkts simultaneously. 4979 * NIX PSE may deadlock when there are any sticky to non-sticky 4980 * transmission. Hence disable it (TM5 = 0). 4981 */ 4982 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); 4983 cfg &= ~(BIT_ULL(15) | BIT_ULL(14) | BIT_ULL(23)); 4984 /* NIX may drop credits when condition clocks are turned off. 4985 * Hence enable control flow clk (set TM9 = 1). 4986 */ 4987 cfg |= BIT_ULL(21); 4988 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); 4989 4990 ltdefs = rvu->kpu.lt_def; 4991 /* Calibrate X2P bus to check if CGX/LBK links are fine */ 4992 err = nix_calibrate_x2p(rvu, blkaddr); 4993 if (err) 4994 return err; 4995 4996 /* Setup capabilities of the NIX block */ 4997 rvu_nix_setup_capabilities(rvu, blkaddr); 4998 4999 /* Initialize admin queue */ 5000 err = nix_aq_init(rvu, block); 5001 if (err) 5002 return err; 5003 5004 /* Restore CINT timer delay to HW reset values */ 5005 rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); 5006 5007 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); 5008 5009 /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ 5010 cfg |= 1ULL; 5011 if (!is_rvu_otx2(rvu)) 5012 cfg |= NIX_PTP_1STEP_EN; 5013 5014 rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); 5015 5016 if (!is_rvu_otx2(rvu)) 5017 rvu_nix_block_cn10k_init(rvu, nix_hw); 5018 5019 if (is_block_implemented(hw, blkaddr)) { 5020 err = nix_setup_txschq(rvu, nix_hw, blkaddr); 5021 if (err) 5022 return err; 5023 5024 err = nix_setup_ipolicers(rvu, nix_hw, blkaddr); 5025 if (err) 5026 return err; 5027 5028 err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr); 5029 if (err) 5030 return err; 5031 5032 err = nix_setup_mcast(rvu, nix_hw, blkaddr); 5033 if (err) 5034 return err; 5035 5036 err = nix_setup_txvlan(rvu, nix_hw); 5037 if (err) 5038 return err; 5039 5040 err = nix_setup_bpids(rvu, nix_hw, blkaddr); 5041 if (err) 5042 return err; 5043 5044 /* Configure segmentation offload formats */ 5045 nix_setup_lso(rvu, nix_hw, blkaddr); 5046 5047 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info. 5048 * This helps HW protocol checker to identify headers 5049 * and validate length and checksums. 5050 */ 5051 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2, 5052 (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) | 5053 ltdefs->rx_ol2.ltype_mask); 5054 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4, 5055 (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) | 5056 ltdefs->rx_oip4.ltype_mask); 5057 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4, 5058 (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) | 5059 ltdefs->rx_iip4.ltype_mask); 5060 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6, 5061 (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) | 5062 ltdefs->rx_oip6.ltype_mask); 5063 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6, 5064 (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) | 5065 ltdefs->rx_iip6.ltype_mask); 5066 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP, 5067 (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) | 5068 ltdefs->rx_otcp.ltype_mask); 5069 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP, 5070 (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) | 5071 ltdefs->rx_itcp.ltype_mask); 5072 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP, 5073 (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) | 5074 ltdefs->rx_oudp.ltype_mask); 5075 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP, 5076 (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) | 5077 ltdefs->rx_iudp.ltype_mask); 5078 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP, 5079 (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) | 5080 ltdefs->rx_osctp.ltype_mask); 5081 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP, 5082 (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) | 5083 ltdefs->rx_isctp.ltype_mask); 5084 5085 if (!is_rvu_otx2(rvu)) { 5086 /* Enable APAD calculation for other protocols 5087 * matching APAD0 and APAD1 lt def registers. 5088 */ 5089 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0, 5090 (ltdefs->rx_apad0.valid << 11) | 5091 (ltdefs->rx_apad0.lid << 8) | 5092 (ltdefs->rx_apad0.ltype_match << 4) | 5093 ltdefs->rx_apad0.ltype_mask); 5094 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1, 5095 (ltdefs->rx_apad1.valid << 11) | 5096 (ltdefs->rx_apad1.lid << 8) | 5097 (ltdefs->rx_apad1.ltype_match << 4) | 5098 ltdefs->rx_apad1.ltype_mask); 5099 5100 /* Receive ethertype definition register defines layer 5101 * information in NPC_RESULT_S to identify the Ethertype 5102 * location in L2 header. Used for Ethertype overwriting 5103 * in inline IPsec flow. 5104 */ 5105 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0), 5106 (ltdefs->rx_et[0].offset << 12) | 5107 (ltdefs->rx_et[0].valid << 11) | 5108 (ltdefs->rx_et[0].lid << 8) | 5109 (ltdefs->rx_et[0].ltype_match << 4) | 5110 ltdefs->rx_et[0].ltype_mask); 5111 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1), 5112 (ltdefs->rx_et[1].offset << 12) | 5113 (ltdefs->rx_et[1].valid << 11) | 5114 (ltdefs->rx_et[1].lid << 8) | 5115 (ltdefs->rx_et[1].ltype_match << 4) | 5116 ltdefs->rx_et[1].ltype_mask); 5117 } 5118 5119 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr); 5120 if (err) 5121 return err; 5122 5123 nix_hw->tx_credits = kcalloc(hw->cgx_links + hw->lbk_links, 5124 sizeof(u64), GFP_KERNEL); 5125 if (!nix_hw->tx_credits) 5126 return -ENOMEM; 5127 5128 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */ 5129 nix_link_config(rvu, blkaddr, nix_hw); 5130 5131 /* Enable Channel backpressure */ 5132 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0)); 5133 } 5134 return 0; 5135 } 5136 5137 int rvu_nix_init(struct rvu *rvu) 5138 { 5139 struct rvu_hwinfo *hw = rvu->hw; 5140 struct nix_hw *nix_hw; 5141 int blkaddr = 0, err; 5142 int i = 0; 5143 5144 hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw), 5145 GFP_KERNEL); 5146 if (!hw->nix) 5147 return -ENOMEM; 5148 5149 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5150 while (blkaddr) { 5151 nix_hw = &hw->nix[i]; 5152 nix_hw->rvu = rvu; 5153 nix_hw->blkaddr = blkaddr; 5154 err = rvu_nix_block_init(rvu, nix_hw); 5155 if (err) 5156 return err; 5157 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5158 i++; 5159 } 5160 5161 return 0; 5162 } 5163 5164 static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr, 5165 struct rvu_block *block) 5166 { 5167 struct nix_txsch *txsch; 5168 struct nix_mcast *mcast; 5169 struct nix_txvlan *vlan; 5170 struct nix_hw *nix_hw; 5171 int lvl; 5172 5173 rvu_aq_free(rvu, block->aq); 5174 5175 if (is_block_implemented(rvu->hw, blkaddr)) { 5176 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5177 if (!nix_hw) 5178 return; 5179 5180 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 5181 txsch = &nix_hw->txsch[lvl]; 5182 kfree(txsch->schq.bmap); 5183 } 5184 5185 kfree(nix_hw->tx_credits); 5186 5187 nix_ipolicer_freemem(rvu, nix_hw); 5188 5189 vlan = &nix_hw->txvlan; 5190 kfree(vlan->rsrc.bmap); 5191 mutex_destroy(&vlan->rsrc_lock); 5192 5193 mcast = &nix_hw->mcast; 5194 qmem_free(rvu->dev, mcast->mce_ctx); 5195 qmem_free(rvu->dev, mcast->mcast_buf); 5196 mutex_destroy(&mcast->mce_lock); 5197 } 5198 } 5199 5200 void rvu_nix_freemem(struct rvu *rvu) 5201 { 5202 struct rvu_hwinfo *hw = rvu->hw; 5203 struct rvu_block *block; 5204 int blkaddr = 0; 5205 5206 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5207 while (blkaddr) { 5208 block = &hw->block[blkaddr]; 5209 rvu_nix_block_freemem(rvu, blkaddr, block); 5210 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 5211 } 5212 } 5213 5214 static void nix_mcast_update_action(struct rvu *rvu, 5215 struct nix_mcast_grp_elem *elem) 5216 { 5217 struct npc_mcam *mcam = &rvu->hw->mcam; 5218 struct nix_rx_action rx_action = { 0 }; 5219 struct nix_tx_action tx_action = { 0 }; 5220 int npc_blkaddr; 5221 5222 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 5223 if (elem->dir == NIX_MCAST_INGRESS) { 5224 *(u64 *)&rx_action = npc_get_mcam_action(rvu, mcam, 5225 npc_blkaddr, 5226 elem->mcam_index); 5227 rx_action.index = elem->mce_start_index; 5228 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index, 5229 *(u64 *)&rx_action); 5230 } else { 5231 *(u64 *)&tx_action = npc_get_mcam_action(rvu, mcam, 5232 npc_blkaddr, 5233 elem->mcam_index); 5234 tx_action.index = elem->mce_start_index; 5235 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index, 5236 *(u64 *)&tx_action); 5237 } 5238 } 5239 5240 static void nix_mcast_update_mce_entry(struct rvu *rvu, u16 pcifunc, u8 is_active) 5241 { 5242 struct nix_mcast_grp_elem *elem; 5243 struct nix_mcast_grp *mcast_grp; 5244 struct nix_hw *nix_hw; 5245 int blkaddr; 5246 5247 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5248 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5249 if (!nix_hw) 5250 return; 5251 5252 mcast_grp = &nix_hw->mcast_grp; 5253 5254 mutex_lock(&mcast_grp->mcast_grp_lock); 5255 list_for_each_entry(elem, &mcast_grp->mcast_grp_head, list) { 5256 struct nix_mce_list *mce_list; 5257 struct mce *mce; 5258 5259 /* Iterate the group elements and disable the element which 5260 * received the disable request. 5261 */ 5262 mce_list = &elem->mcast_mce_list; 5263 hlist_for_each_entry(mce, &mce_list->head, node) { 5264 if (mce->pcifunc == pcifunc) { 5265 mce->is_active = is_active; 5266 break; 5267 } 5268 } 5269 5270 /* Dump the updated list to HW */ 5271 if (elem->dir == NIX_MCAST_INGRESS) 5272 nix_update_ingress_mce_list_hw(rvu, nix_hw, elem); 5273 else 5274 nix_update_egress_mce_list_hw(rvu, nix_hw, elem); 5275 5276 /* Update the multicast index in NPC rule */ 5277 nix_mcast_update_action(rvu, elem); 5278 } 5279 mutex_unlock(&mcast_grp->mcast_grp_lock); 5280 } 5281 5282 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req, 5283 struct msg_rsp *rsp) 5284 { 5285 u16 pcifunc = req->hdr.pcifunc; 5286 struct rvu_pfvf *pfvf; 5287 int nixlf, err, pf; 5288 5289 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 5290 if (err) 5291 return err; 5292 5293 /* Enable the interface if it is in any multicast list */ 5294 nix_mcast_update_mce_entry(rvu, pcifunc, 1); 5295 5296 rvu_npc_enable_default_entries(rvu, pcifunc, nixlf); 5297 5298 npc_mcam_enable_flows(rvu, pcifunc); 5299 5300 pfvf = rvu_get_pfvf(rvu, pcifunc); 5301 set_bit(NIXLF_INITIALIZED, &pfvf->flags); 5302 5303 rvu_switch_update_rules(rvu, pcifunc, true); 5304 5305 pf = rvu_get_pf(rvu->pdev, pcifunc); 5306 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5307 rvu_rep_notify_pfvf_state(rvu, pcifunc, true); 5308 5309 return rvu_cgx_start_stop_io(rvu, pcifunc, true); 5310 } 5311 5312 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, 5313 struct msg_rsp *rsp) 5314 { 5315 u16 pcifunc = req->hdr.pcifunc; 5316 struct rvu_pfvf *pfvf; 5317 int nixlf, err, pf; 5318 5319 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 5320 if (err) 5321 return err; 5322 5323 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 5324 /* Disable the interface if it is in any multicast list */ 5325 nix_mcast_update_mce_entry(rvu, pcifunc, 0); 5326 5327 pfvf = rvu_get_pfvf(rvu, pcifunc); 5328 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 5329 5330 err = rvu_cgx_start_stop_io(rvu, pcifunc, false); 5331 if (err) 5332 return err; 5333 5334 rvu_switch_update_rules(rvu, pcifunc, false); 5335 rvu_cgx_tx_enable(rvu, pcifunc, true); 5336 5337 pf = rvu_get_pf(rvu->pdev, pcifunc); 5338 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5339 rvu_rep_notify_pfvf_state(rvu, pcifunc, false); 5340 return 0; 5341 } 5342 5343 #define RX_SA_BASE GENMASK_ULL(52, 7) 5344 5345 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) 5346 { 5347 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 5348 struct hwctx_disable_req ctx_req; 5349 int pf = rvu_get_pf(rvu->pdev, pcifunc); 5350 struct mac_ops *mac_ops; 5351 u8 cgx_id, lmac_id; 5352 u64 sa_base; 5353 void *cgxd; 5354 int err; 5355 5356 ctx_req.hdr.pcifunc = pcifunc; 5357 5358 /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */ 5359 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 5360 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 5361 nix_interface_deinit(rvu, pcifunc, nixlf); 5362 nix_rx_sync(rvu, blkaddr); 5363 nix_txschq_free(rvu, pcifunc); 5364 5365 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 5366 5367 if (is_pf_cgxmapped(rvu, pf) && rvu->rep_mode) 5368 rvu_rep_notify_pfvf_state(rvu, pcifunc, false); 5369 5370 rvu_cgx_start_stop_io(rvu, pcifunc, false); 5371 5372 if (pfvf->sq_ctx) { 5373 ctx_req.ctype = NIX_AQ_CTYPE_SQ; 5374 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5375 if (err) 5376 dev_err(rvu->dev, "SQ ctx disable failed\n"); 5377 } 5378 5379 if (pfvf->rq_ctx) { 5380 ctx_req.ctype = NIX_AQ_CTYPE_RQ; 5381 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5382 if (err) 5383 dev_err(rvu->dev, "RQ ctx disable failed\n"); 5384 } 5385 5386 if (pfvf->cq_ctx) { 5387 ctx_req.ctype = NIX_AQ_CTYPE_CQ; 5388 err = nix_lf_hwctx_disable(rvu, &ctx_req); 5389 if (err) 5390 dev_err(rvu->dev, "CQ ctx disable failed\n"); 5391 } 5392 5393 /* reset HW config done for Switch headers */ 5394 rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT, 5395 (PKIND_TX | PKIND_RX), 0, 0, 0, 0); 5396 5397 /* Disabling CGX and NPC config done for PTP */ 5398 if (pfvf->hw_rx_tstamp_en) { 5399 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 5400 cgxd = rvu_cgx_pdata(cgx_id, rvu); 5401 mac_ops = get_mac_ops(cgxd); 5402 mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, false); 5403 /* Undo NPC config done for PTP */ 5404 if (npc_config_ts_kpuaction(rvu, pf, pcifunc, false)) 5405 dev_err(rvu->dev, "NPC config for PTP failed\n"); 5406 pfvf->hw_rx_tstamp_en = false; 5407 } 5408 5409 /* reset priority flow control config */ 5410 rvu_cgx_prio_flow_ctrl_cfg(rvu, pcifunc, 0, 0, 0); 5411 5412 /* reset 802.3x flow control config */ 5413 rvu_cgx_cfg_pause_frm(rvu, pcifunc, 0, 0); 5414 5415 nix_ctx_free(rvu, pfvf); 5416 5417 nix_free_all_bandprof(rvu, pcifunc); 5418 5419 sa_base = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(nixlf)); 5420 if (FIELD_GET(RX_SA_BASE, sa_base)) { 5421 err = rvu_cpt_ctx_flush(rvu, pcifunc); 5422 if (err) 5423 dev_err(rvu->dev, 5424 "CPT ctx flush failed with error: %d\n", err); 5425 } 5426 } 5427 5428 #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32) 5429 5430 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable) 5431 { 5432 struct rvu_hwinfo *hw = rvu->hw; 5433 struct rvu_block *block; 5434 int blkaddr, pf; 5435 int nixlf; 5436 u64 cfg; 5437 5438 pf = rvu_get_pf(rvu->pdev, pcifunc); 5439 if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_PTP)) 5440 return 0; 5441 5442 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5443 if (blkaddr < 0) 5444 return NIX_AF_ERR_AF_LF_INVALID; 5445 5446 block = &hw->block[blkaddr]; 5447 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 5448 if (nixlf < 0) 5449 return NIX_AF_ERR_AF_LF_INVALID; 5450 5451 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf)); 5452 5453 if (enable) 5454 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN; 5455 else 5456 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN; 5457 5458 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 5459 5460 return 0; 5461 } 5462 5463 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req, 5464 struct msg_rsp *rsp) 5465 { 5466 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true); 5467 } 5468 5469 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req, 5470 struct msg_rsp *rsp) 5471 { 5472 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false); 5473 } 5474 5475 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu, 5476 struct nix_lso_format_cfg *req, 5477 struct nix_lso_format_cfg_rsp *rsp) 5478 { 5479 u16 pcifunc = req->hdr.pcifunc; 5480 struct nix_hw *nix_hw; 5481 struct rvu_pfvf *pfvf; 5482 int blkaddr, idx, f; 5483 u64 reg; 5484 5485 pfvf = rvu_get_pfvf(rvu, pcifunc); 5486 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 5487 if (!pfvf->nixlf || blkaddr < 0) 5488 return NIX_AF_ERR_AF_LF_INVALID; 5489 5490 nix_hw = get_nix_hw(rvu->hw, blkaddr); 5491 if (!nix_hw) 5492 return NIX_AF_ERR_INVALID_NIXBLK; 5493 5494 /* Find existing matching LSO format, if any */ 5495 for (idx = 0; idx < nix_hw->lso.in_use; idx++) { 5496 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) { 5497 reg = rvu_read64(rvu, blkaddr, 5498 NIX_AF_LSO_FORMATX_FIELDX(idx, f)); 5499 if (req->fields[f] != (reg & req->field_mask)) 5500 break; 5501 } 5502 5503 if (f == NIX_LSO_FIELD_MAX) 5504 break; 5505 } 5506 5507 if (idx < nix_hw->lso.in_use) { 5508 /* Match found */ 5509 rsp->lso_format_idx = idx; 5510 return 0; 5511 } 5512 5513 if (nix_hw->lso.in_use == nix_hw->lso.total) 5514 return NIX_AF_ERR_LSO_CFG_FAIL; 5515 5516 rsp->lso_format_idx = nix_hw->lso.in_use++; 5517 5518 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) 5519 rvu_write64(rvu, blkaddr, 5520 NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f), 5521 req->fields[f]); 5522 5523 return 0; 5524 } 5525 5526 #define IPSEC_GEN_CFG_EGRP GENMASK_ULL(50, 48) 5527 #define IPSEC_GEN_CFG_OPCODE GENMASK_ULL(47, 32) 5528 #define IPSEC_GEN_CFG_PARAM1 GENMASK_ULL(31, 16) 5529 #define IPSEC_GEN_CFG_PARAM2 GENMASK_ULL(15, 0) 5530 5531 #define CPT_INST_QSEL_BLOCK GENMASK_ULL(28, 24) 5532 #define CPT_INST_QSEL_PF_FUNC GENMASK_ULL(23, 8) 5533 #define CPT_INST_QSEL_SLOT GENMASK_ULL(7, 0) 5534 5535 #define CPT_INST_CREDIT_TH GENMASK_ULL(53, 32) 5536 #define CPT_INST_CREDIT_BPID GENMASK_ULL(30, 22) 5537 #define CPT_INST_CREDIT_CNT GENMASK_ULL(21, 0) 5538 5539 static void nix_inline_ipsec_cfg(struct rvu *rvu, struct nix_inline_ipsec_cfg *req, 5540 int blkaddr) 5541 { 5542 u8 cpt_idx, cpt_blkaddr; 5543 u64 val; 5544 5545 cpt_idx = (blkaddr == BLKADDR_NIX0) ? 0 : 1; 5546 if (req->enable) { 5547 val = 0; 5548 /* Enable context prefetching */ 5549 if (!is_rvu_otx2(rvu)) 5550 val |= BIT_ULL(51); 5551 5552 /* Set OPCODE and EGRP */ 5553 val |= FIELD_PREP(IPSEC_GEN_CFG_EGRP, req->gen_cfg.egrp); 5554 val |= FIELD_PREP(IPSEC_GEN_CFG_OPCODE, req->gen_cfg.opcode); 5555 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM1, req->gen_cfg.param1); 5556 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM2, req->gen_cfg.param2); 5557 5558 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, val); 5559 5560 /* Set CPT queue for inline IPSec */ 5561 val = FIELD_PREP(CPT_INST_QSEL_SLOT, req->inst_qsel.cpt_slot); 5562 val |= FIELD_PREP(CPT_INST_QSEL_PF_FUNC, 5563 req->inst_qsel.cpt_pf_func); 5564 5565 if (!is_rvu_otx2(rvu)) { 5566 cpt_blkaddr = (cpt_idx == 0) ? BLKADDR_CPT0 : 5567 BLKADDR_CPT1; 5568 val |= FIELD_PREP(CPT_INST_QSEL_BLOCK, cpt_blkaddr); 5569 } 5570 5571 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 5572 val); 5573 5574 /* Set CPT credit */ 5575 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 5576 if ((val & 0x3FFFFF) != 0x3FFFFF) 5577 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 5578 0x3FFFFF - val); 5579 5580 val = FIELD_PREP(CPT_INST_CREDIT_CNT, req->cpt_credit); 5581 val |= FIELD_PREP(CPT_INST_CREDIT_BPID, req->bpid); 5582 val |= FIELD_PREP(CPT_INST_CREDIT_TH, req->credit_th); 5583 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), val); 5584 } else { 5585 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, 0x0); 5586 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 5587 0x0); 5588 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 5589 if ((val & 0x3FFFFF) != 0x3FFFFF) 5590 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 5591 0x3FFFFF - val); 5592 } 5593 } 5594 5595 int rvu_mbox_handler_nix_inline_ipsec_cfg(struct rvu *rvu, 5596 struct nix_inline_ipsec_cfg *req, 5597 struct msg_rsp *rsp) 5598 { 5599 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5600 return 0; 5601 5602 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX0); 5603 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 5604 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX1); 5605 5606 return 0; 5607 } 5608 5609 int rvu_mbox_handler_nix_read_inline_ipsec_cfg(struct rvu *rvu, 5610 struct msg_req *req, 5611 struct nix_inline_ipsec_cfg *rsp) 5612 5613 { 5614 u64 val; 5615 5616 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5617 return 0; 5618 5619 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_IPSEC_GEN_CFG); 5620 rsp->gen_cfg.egrp = FIELD_GET(IPSEC_GEN_CFG_EGRP, val); 5621 rsp->gen_cfg.opcode = FIELD_GET(IPSEC_GEN_CFG_OPCODE, val); 5622 rsp->gen_cfg.param1 = FIELD_GET(IPSEC_GEN_CFG_PARAM1, val); 5623 rsp->gen_cfg.param2 = FIELD_GET(IPSEC_GEN_CFG_PARAM2, val); 5624 5625 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_CPTX_CREDIT(0)); 5626 rsp->cpt_credit = FIELD_GET(CPT_INST_CREDIT_CNT, val); 5627 rsp->credit_th = FIELD_GET(CPT_INST_CREDIT_TH, val); 5628 rsp->bpid = FIELD_GET(CPT_INST_CREDIT_BPID, val); 5629 5630 return 0; 5631 } 5632 5633 int rvu_mbox_handler_nix_inline_ipsec_lf_cfg(struct rvu *rvu, 5634 struct nix_inline_ipsec_lf_cfg *req, 5635 struct msg_rsp *rsp) 5636 { 5637 int lf, blkaddr, err; 5638 u64 val; 5639 5640 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 5641 return 0; 5642 5643 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &lf, &blkaddr); 5644 if (err) 5645 return err; 5646 5647 if (req->enable) { 5648 /* Set TT, TAG_CONST, SA_POW2_SIZE and LENM1_MAX */ 5649 val = (u64)req->ipsec_cfg0.tt << 44 | 5650 (u64)req->ipsec_cfg0.tag_const << 20 | 5651 (u64)req->ipsec_cfg0.sa_pow2_size << 16 | 5652 req->ipsec_cfg0.lenm1_max; 5653 5654 if (blkaddr == BLKADDR_NIX1) 5655 val |= BIT_ULL(46); 5656 5657 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), val); 5658 5659 /* Set SA_IDX_W and SA_IDX_MAX */ 5660 val = (u64)req->ipsec_cfg1.sa_idx_w << 32 | 5661 req->ipsec_cfg1.sa_idx_max; 5662 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), val); 5663 5664 /* Set SA base address */ 5665 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 5666 req->sa_base_addr); 5667 } else { 5668 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), 0x0); 5669 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), 0x0); 5670 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 5671 0x0); 5672 } 5673 5674 return 0; 5675 } 5676 5677 void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc) 5678 { 5679 bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 5680 5681 /* overwrite vf mac address with default_mac */ 5682 if (from_vf) 5683 ether_addr_copy(pfvf->mac_addr, pfvf->default_mac); 5684 } 5685 5686 /* NIX ingress policers or bandwidth profiles APIs */ 5687 static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr) 5688 { 5689 struct npc_lt_def_cfg defs, *ltdefs; 5690 5691 ltdefs = &defs; 5692 memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg)); 5693 5694 /* Extract PCP and DEI fields from outer VLAN from byte offset 5695 * 2 from the start of LB_PTR (ie TAG). 5696 * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN 5697 * fields are considered when 'Tunnel enable' is set in profile. 5698 */ 5699 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI, 5700 (2UL << 12) | (ltdefs->ovlan.lid << 8) | 5701 (ltdefs->ovlan.ltype_match << 4) | 5702 ltdefs->ovlan.ltype_mask); 5703 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI, 5704 (2UL << 12) | (ltdefs->ivlan.lid << 8) | 5705 (ltdefs->ivlan.ltype_match << 4) | 5706 ltdefs->ivlan.ltype_mask); 5707 5708 /* DSCP field in outer and tunneled IPv4 packets */ 5709 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP, 5710 (1UL << 12) | (ltdefs->rx_oip4.lid << 8) | 5711 (ltdefs->rx_oip4.ltype_match << 4) | 5712 ltdefs->rx_oip4.ltype_mask); 5713 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP, 5714 (1UL << 12) | (ltdefs->rx_iip4.lid << 8) | 5715 (ltdefs->rx_iip4.ltype_match << 4) | 5716 ltdefs->rx_iip4.ltype_mask); 5717 5718 /* DSCP field (traffic class) in outer and tunneled IPv6 packets */ 5719 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP, 5720 (1UL << 11) | (ltdefs->rx_oip6.lid << 8) | 5721 (ltdefs->rx_oip6.ltype_match << 4) | 5722 ltdefs->rx_oip6.ltype_mask); 5723 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP, 5724 (1UL << 11) | (ltdefs->rx_iip6.lid << 8) | 5725 (ltdefs->rx_iip6.ltype_match << 4) | 5726 ltdefs->rx_iip6.ltype_mask); 5727 } 5728 5729 static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw, 5730 int layer, int prof_idx) 5731 { 5732 struct nix_cn10k_aq_enq_req aq_req; 5733 int rc; 5734 5735 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5736 5737 aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14); 5738 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 5739 aq_req.op = NIX_AQ_INSTOP_INIT; 5740 5741 /* Context is all zeros, submit to AQ */ 5742 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5743 (struct nix_aq_enq_req *)&aq_req, NULL); 5744 if (rc) 5745 dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n", 5746 layer, prof_idx); 5747 return rc; 5748 } 5749 5750 static int nix_setup_ipolicers(struct rvu *rvu, 5751 struct nix_hw *nix_hw, int blkaddr) 5752 { 5753 struct rvu_hwinfo *hw = rvu->hw; 5754 struct nix_ipolicer *ipolicer; 5755 int err, layer, prof_idx; 5756 u64 cfg; 5757 5758 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); 5759 if (!(cfg & BIT_ULL(61))) { 5760 hw->cap.ipolicer = false; 5761 return 0; 5762 } 5763 5764 hw->cap.ipolicer = true; 5765 nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS, 5766 sizeof(*ipolicer), GFP_KERNEL); 5767 if (!nix_hw->ipolicer) 5768 return -ENOMEM; 5769 5770 cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST); 5771 5772 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5773 ipolicer = &nix_hw->ipolicer[layer]; 5774 switch (layer) { 5775 case BAND_PROF_LEAF_LAYER: 5776 ipolicer->band_prof.max = cfg & 0XFFFF; 5777 break; 5778 case BAND_PROF_MID_LAYER: 5779 ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF; 5780 break; 5781 case BAND_PROF_TOP_LAYER: 5782 ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF; 5783 break; 5784 } 5785 5786 if (!ipolicer->band_prof.max) 5787 continue; 5788 5789 err = rvu_alloc_bitmap(&ipolicer->band_prof); 5790 if (err) 5791 return err; 5792 5793 ipolicer->pfvf_map = devm_kcalloc(rvu->dev, 5794 ipolicer->band_prof.max, 5795 sizeof(u16), GFP_KERNEL); 5796 if (!ipolicer->pfvf_map) 5797 return -ENOMEM; 5798 5799 ipolicer->match_id = devm_kcalloc(rvu->dev, 5800 ipolicer->band_prof.max, 5801 sizeof(u16), GFP_KERNEL); 5802 if (!ipolicer->match_id) 5803 return -ENOMEM; 5804 5805 for (prof_idx = 0; 5806 prof_idx < ipolicer->band_prof.max; prof_idx++) { 5807 /* Set AF as current owner for INIT ops to succeed */ 5808 ipolicer->pfvf_map[prof_idx] = 0x00; 5809 5810 /* There is no enable bit in the profile context, 5811 * so no context disable. So let's INIT them here 5812 * so that PF/VF later on have to just do WRITE to 5813 * setup policer rates and config. 5814 */ 5815 err = nix_init_policer_context(rvu, nix_hw, 5816 layer, prof_idx); 5817 if (err) 5818 return err; 5819 } 5820 5821 /* Allocate memory for maintaining ref_counts for MID level 5822 * profiles, this will be needed for leaf layer profiles' 5823 * aggregation. 5824 */ 5825 if (layer != BAND_PROF_MID_LAYER) 5826 continue; 5827 5828 ipolicer->ref_count = devm_kcalloc(rvu->dev, 5829 ipolicer->band_prof.max, 5830 sizeof(u16), GFP_KERNEL); 5831 if (!ipolicer->ref_count) 5832 return -ENOMEM; 5833 } 5834 5835 /* Set policer timeunit to 2us ie (19 + 1) * 100 nsec = 2us */ 5836 rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19); 5837 5838 nix_config_rx_pkt_policer_precolor(rvu, blkaddr); 5839 5840 return 0; 5841 } 5842 5843 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw) 5844 { 5845 struct nix_ipolicer *ipolicer; 5846 int layer; 5847 5848 if (!rvu->hw->cap.ipolicer) 5849 return; 5850 5851 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5852 ipolicer = &nix_hw->ipolicer[layer]; 5853 5854 if (!ipolicer->band_prof.max) 5855 continue; 5856 5857 kfree(ipolicer->band_prof.bmap); 5858 } 5859 } 5860 5861 #define NIX_BW_PROF_HI_MASK GENMASK(10, 7) 5862 5863 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 5864 struct nix_hw *nix_hw, u16 pcifunc) 5865 { 5866 struct nix_ipolicer *ipolicer; 5867 int layer, hi_layer, prof_idx; 5868 5869 /* Bits [15:14] in profile index represent layer */ 5870 layer = (req->qidx >> 14) & 0x03; 5871 prof_idx = req->qidx & 0x3FFF; 5872 5873 ipolicer = &nix_hw->ipolicer[layer]; 5874 if (prof_idx >= ipolicer->band_prof.max) 5875 return -EINVAL; 5876 5877 /* Check if the profile is allocated to the requesting PCIFUNC or not 5878 * with the exception of AF. AF is allowed to read and update contexts. 5879 */ 5880 if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc) 5881 return -EINVAL; 5882 5883 /* If this profile is linked to higher layer profile then check 5884 * if that profile is also allocated to the requesting PCIFUNC 5885 * or not. 5886 */ 5887 if (!req->prof.hl_en) 5888 return 0; 5889 5890 /* Leaf layer profile can link only to mid layer and 5891 * mid layer to top layer. 5892 */ 5893 if (layer == BAND_PROF_LEAF_LAYER) 5894 hi_layer = BAND_PROF_MID_LAYER; 5895 else if (layer == BAND_PROF_MID_LAYER) 5896 hi_layer = BAND_PROF_TOP_LAYER; 5897 else 5898 return -EINVAL; 5899 5900 ipolicer = &nix_hw->ipolicer[hi_layer]; 5901 prof_idx = FIELD_PREP(NIX_BW_PROF_HI_MASK, req->prof.band_prof_id_h); 5902 prof_idx |= req->prof.band_prof_id; 5903 if (prof_idx >= ipolicer->band_prof.max || 5904 ipolicer->pfvf_map[prof_idx] != pcifunc) 5905 return -EINVAL; 5906 5907 return 0; 5908 } 5909 5910 int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu, 5911 struct nix_bandprof_alloc_req *req, 5912 struct nix_bandprof_alloc_rsp *rsp) 5913 { 5914 int blkaddr, layer, prof, idx, err; 5915 u16 pcifunc = req->hdr.pcifunc; 5916 struct nix_ipolicer *ipolicer; 5917 struct nix_hw *nix_hw; 5918 5919 if (!rvu->hw->cap.ipolicer) 5920 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5921 5922 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5923 if (err) 5924 return err; 5925 5926 mutex_lock(&rvu->rsrc_lock); 5927 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5928 if (layer == BAND_PROF_INVAL_LAYER) 5929 continue; 5930 if (!req->prof_count[layer]) 5931 continue; 5932 5933 ipolicer = &nix_hw->ipolicer[layer]; 5934 for (idx = 0; idx < req->prof_count[layer]; idx++) { 5935 /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */ 5936 if (idx == MAX_BANDPROF_PER_PFFUNC) 5937 break; 5938 5939 prof = rvu_alloc_rsrc(&ipolicer->band_prof); 5940 if (prof < 0) 5941 break; 5942 rsp->prof_count[layer]++; 5943 rsp->prof_idx[layer][idx] = prof; 5944 ipolicer->pfvf_map[prof] = pcifunc; 5945 } 5946 } 5947 mutex_unlock(&rvu->rsrc_lock); 5948 return 0; 5949 } 5950 5951 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc) 5952 { 5953 int blkaddr, layer, prof_idx, err; 5954 struct nix_ipolicer *ipolicer; 5955 struct nix_hw *nix_hw; 5956 5957 if (!rvu->hw->cap.ipolicer) 5958 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5959 5960 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5961 if (err) 5962 return err; 5963 5964 mutex_lock(&rvu->rsrc_lock); 5965 /* Free all the profiles allocated to the PCIFUNC */ 5966 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5967 if (layer == BAND_PROF_INVAL_LAYER) 5968 continue; 5969 ipolicer = &nix_hw->ipolicer[layer]; 5970 5971 for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) { 5972 if (ipolicer->pfvf_map[prof_idx] != pcifunc) 5973 continue; 5974 5975 /* Clear ratelimit aggregation, if any */ 5976 if (layer == BAND_PROF_LEAF_LAYER && 5977 ipolicer->match_id[prof_idx]) 5978 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 5979 5980 ipolicer->pfvf_map[prof_idx] = 0x00; 5981 ipolicer->match_id[prof_idx] = 0; 5982 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 5983 } 5984 } 5985 mutex_unlock(&rvu->rsrc_lock); 5986 return 0; 5987 } 5988 5989 int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu, 5990 struct nix_bandprof_free_req *req, 5991 struct msg_rsp *rsp) 5992 { 5993 int blkaddr, layer, prof_idx, idx, err; 5994 u16 pcifunc = req->hdr.pcifunc; 5995 struct nix_ipolicer *ipolicer; 5996 struct nix_hw *nix_hw; 5997 5998 if (req->free_all) 5999 return nix_free_all_bandprof(rvu, pcifunc); 6000 6001 if (!rvu->hw->cap.ipolicer) 6002 return NIX_AF_ERR_IPOLICER_NOTSUPP; 6003 6004 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 6005 if (err) 6006 return err; 6007 6008 mutex_lock(&rvu->rsrc_lock); 6009 /* Free the requested profile indices */ 6010 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 6011 if (layer == BAND_PROF_INVAL_LAYER) 6012 continue; 6013 if (!req->prof_count[layer]) 6014 continue; 6015 6016 ipolicer = &nix_hw->ipolicer[layer]; 6017 for (idx = 0; idx < req->prof_count[layer]; idx++) { 6018 if (idx == MAX_BANDPROF_PER_PFFUNC) 6019 break; 6020 prof_idx = req->prof_idx[layer][idx]; 6021 if (prof_idx >= ipolicer->band_prof.max || 6022 ipolicer->pfvf_map[prof_idx] != pcifunc) 6023 continue; 6024 6025 /* Clear ratelimit aggregation, if any */ 6026 if (layer == BAND_PROF_LEAF_LAYER && 6027 ipolicer->match_id[prof_idx]) 6028 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 6029 6030 ipolicer->pfvf_map[prof_idx] = 0x00; 6031 ipolicer->match_id[prof_idx] = 0; 6032 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 6033 } 6034 } 6035 mutex_unlock(&rvu->rsrc_lock); 6036 return 0; 6037 } 6038 6039 int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw, 6040 struct nix_cn10k_aq_enq_req *aq_req, 6041 struct nix_cn10k_aq_enq_rsp *aq_rsp, 6042 u16 pcifunc, u8 ctype, u32 qidx) 6043 { 6044 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6045 aq_req->hdr.pcifunc = pcifunc; 6046 aq_req->ctype = ctype; 6047 aq_req->op = NIX_AQ_INSTOP_READ; 6048 aq_req->qidx = qidx; 6049 6050 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6051 (struct nix_aq_enq_req *)aq_req, 6052 (struct nix_aq_enq_rsp *)aq_rsp); 6053 } 6054 6055 static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu, 6056 struct nix_hw *nix_hw, 6057 struct nix_cn10k_aq_enq_req *aq_req, 6058 struct nix_cn10k_aq_enq_rsp *aq_rsp, 6059 u32 leaf_prof, u16 mid_prof) 6060 { 6061 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6062 aq_req->hdr.pcifunc = 0x00; 6063 aq_req->ctype = NIX_AQ_CTYPE_BANDPROF; 6064 aq_req->op = NIX_AQ_INSTOP_WRITE; 6065 aq_req->qidx = leaf_prof; 6066 6067 aq_req->prof.band_prof_id = mid_prof & 0x7F; 6068 aq_req->prof_mask.band_prof_id = GENMASK(6, 0); 6069 aq_req->prof.band_prof_id_h = FIELD_GET(NIX_BW_PROF_HI_MASK, mid_prof); 6070 aq_req->prof_mask.band_prof_id_h = GENMASK(3, 0); 6071 aq_req->prof.hl_en = 1; 6072 aq_req->prof_mask.hl_en = 1; 6073 6074 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6075 (struct nix_aq_enq_req *)aq_req, 6076 (struct nix_aq_enq_rsp *)aq_rsp); 6077 } 6078 6079 #define NIX_RQ_PROF_HI_MASK GENMASK(13, 10) 6080 6081 int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, 6082 u16 rq_idx, u16 match_id) 6083 { 6084 int leaf_prof, mid_prof, leaf_match; 6085 struct nix_cn10k_aq_enq_req aq_req; 6086 struct nix_cn10k_aq_enq_rsp aq_rsp; 6087 struct nix_ipolicer *ipolicer; 6088 struct nix_hw *nix_hw; 6089 int blkaddr, idx, rc; 6090 6091 if (!rvu->hw->cap.ipolicer) 6092 return 0; 6093 6094 rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 6095 if (rc) 6096 return rc; 6097 6098 /* Fetch the RQ's context to see if policing is enabled */ 6099 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc, 6100 NIX_AQ_CTYPE_RQ, rq_idx); 6101 if (rc) { 6102 dev_err(rvu->dev, 6103 "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n", 6104 __func__, rq_idx, pcifunc); 6105 return rc; 6106 } 6107 6108 if (!aq_rsp.rq.policer_ena) 6109 return 0; 6110 6111 /* Get the bandwidth profile ID mapped to this RQ */ 6112 leaf_prof = FIELD_PREP(NIX_RQ_PROF_HI_MASK, aq_rsp.rq.band_prof_id_h); 6113 leaf_prof |= aq_rsp.rq.band_prof_id; 6114 6115 ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER]; 6116 ipolicer->match_id[leaf_prof] = match_id; 6117 6118 /* Check if any other leaf profile is marked with same match_id */ 6119 for (idx = 0; idx < ipolicer->band_prof.max; idx++) { 6120 if (idx == leaf_prof) 6121 continue; 6122 if (ipolicer->match_id[idx] != match_id) 6123 continue; 6124 6125 leaf_match = idx; 6126 break; 6127 } 6128 6129 if (idx == ipolicer->band_prof.max) 6130 return 0; 6131 6132 /* Fetch the matching profile's context to check if it's already 6133 * mapped to a mid level profile. 6134 */ 6135 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6136 NIX_AQ_CTYPE_BANDPROF, leaf_match); 6137 if (rc) { 6138 dev_err(rvu->dev, 6139 "%s: Failed to fetch context of leaf profile %d\n", 6140 __func__, leaf_match); 6141 return rc; 6142 } 6143 6144 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 6145 if (aq_rsp.prof.hl_en) { 6146 /* Get Mid layer prof index and map leaf_prof index 6147 * also such that flows that are being steered 6148 * to different RQs and marked with same match_id 6149 * are rate limited in a aggregate fashion 6150 */ 6151 mid_prof = FIELD_PREP(NIX_BW_PROF_HI_MASK, 6152 aq_rsp.prof.band_prof_id_h); 6153 mid_prof |= aq_rsp.prof.band_prof_id; 6154 6155 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6156 &aq_req, &aq_rsp, 6157 leaf_prof, mid_prof); 6158 if (rc) { 6159 dev_err(rvu->dev, 6160 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6161 __func__, leaf_prof, mid_prof); 6162 goto exit; 6163 } 6164 6165 mutex_lock(&rvu->rsrc_lock); 6166 ipolicer->ref_count[mid_prof]++; 6167 mutex_unlock(&rvu->rsrc_lock); 6168 goto exit; 6169 } 6170 6171 /* Allocate a mid layer profile and 6172 * map both 'leaf_prof' and 'leaf_match' profiles to it. 6173 */ 6174 mutex_lock(&rvu->rsrc_lock); 6175 mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof); 6176 if (mid_prof < 0) { 6177 dev_err(rvu->dev, 6178 "%s: Unable to allocate mid layer profile\n", __func__); 6179 mutex_unlock(&rvu->rsrc_lock); 6180 goto exit; 6181 } 6182 mutex_unlock(&rvu->rsrc_lock); 6183 ipolicer->pfvf_map[mid_prof] = 0x00; 6184 ipolicer->ref_count[mid_prof] = 0; 6185 6186 /* Initialize mid layer profile same as 'leaf_prof' */ 6187 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6188 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 6189 if (rc) { 6190 dev_err(rvu->dev, 6191 "%s: Failed to fetch context of leaf profile %d\n", 6192 __func__, leaf_prof); 6193 goto exit; 6194 } 6195 6196 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 6197 aq_req.hdr.pcifunc = 0x00; 6198 aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14); 6199 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 6200 aq_req.op = NIX_AQ_INSTOP_WRITE; 6201 memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s)); 6202 memset((char *)&aq_req.prof_mask, 0xff, sizeof(struct nix_bandprof_s)); 6203 /* Clear higher layer enable bit in the mid profile, just in case */ 6204 aq_req.prof.hl_en = 0; 6205 aq_req.prof_mask.hl_en = 1; 6206 6207 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 6208 (struct nix_aq_enq_req *)&aq_req, NULL); 6209 if (rc) { 6210 dev_err(rvu->dev, 6211 "%s: Failed to INIT context of mid layer profile %d\n", 6212 __func__, mid_prof); 6213 goto exit; 6214 } 6215 6216 /* Map both leaf profiles to this mid layer profile */ 6217 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6218 &aq_req, &aq_rsp, 6219 leaf_prof, mid_prof); 6220 if (rc) { 6221 dev_err(rvu->dev, 6222 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6223 __func__, leaf_prof, mid_prof); 6224 goto exit; 6225 } 6226 6227 mutex_lock(&rvu->rsrc_lock); 6228 ipolicer->ref_count[mid_prof]++; 6229 mutex_unlock(&rvu->rsrc_lock); 6230 6231 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 6232 &aq_req, &aq_rsp, 6233 leaf_match, mid_prof); 6234 if (rc) { 6235 dev_err(rvu->dev, 6236 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 6237 __func__, leaf_match, mid_prof); 6238 ipolicer->ref_count[mid_prof]--; 6239 goto exit; 6240 } 6241 6242 mutex_lock(&rvu->rsrc_lock); 6243 ipolicer->ref_count[mid_prof]++; 6244 mutex_unlock(&rvu->rsrc_lock); 6245 6246 exit: 6247 return rc; 6248 } 6249 6250 /* Called with mutex rsrc_lock */ 6251 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 6252 u32 leaf_prof) 6253 { 6254 struct nix_cn10k_aq_enq_req aq_req; 6255 struct nix_cn10k_aq_enq_rsp aq_rsp; 6256 struct nix_ipolicer *ipolicer; 6257 u16 mid_prof; 6258 int rc; 6259 6260 mutex_unlock(&rvu->rsrc_lock); 6261 6262 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 6263 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 6264 6265 mutex_lock(&rvu->rsrc_lock); 6266 if (rc) { 6267 dev_err(rvu->dev, 6268 "%s: Failed to fetch context of leaf profile %d\n", 6269 __func__, leaf_prof); 6270 return; 6271 } 6272 6273 if (!aq_rsp.prof.hl_en) 6274 return; 6275 6276 mid_prof = FIELD_PREP(NIX_BW_PROF_HI_MASK, aq_rsp.prof.band_prof_id_h); 6277 mid_prof |= aq_rsp.prof.band_prof_id; 6278 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 6279 ipolicer->ref_count[mid_prof]--; 6280 /* If ref_count is zero, free mid layer profile */ 6281 if (!ipolicer->ref_count[mid_prof]) { 6282 ipolicer->pfvf_map[mid_prof] = 0x00; 6283 rvu_free_rsrc(&ipolicer->band_prof, mid_prof); 6284 } 6285 } 6286 6287 int rvu_mbox_handler_nix_bandprof_get_hwinfo(struct rvu *rvu, struct msg_req *req, 6288 struct nix_bandprof_get_hwinfo_rsp *rsp) 6289 { 6290 struct nix_ipolicer *ipolicer; 6291 int blkaddr, layer, err; 6292 struct nix_hw *nix_hw; 6293 u64 tu; 6294 6295 if (!rvu->hw->cap.ipolicer) 6296 return NIX_AF_ERR_IPOLICER_NOTSUPP; 6297 6298 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6299 if (err) 6300 return err; 6301 6302 /* Return number of bandwidth profiles free at each layer */ 6303 mutex_lock(&rvu->rsrc_lock); 6304 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 6305 if (layer == BAND_PROF_INVAL_LAYER) 6306 continue; 6307 6308 ipolicer = &nix_hw->ipolicer[layer]; 6309 rsp->prof_count[layer] = rvu_rsrc_free_count(&ipolicer->band_prof); 6310 } 6311 mutex_unlock(&rvu->rsrc_lock); 6312 6313 /* Set the policer timeunit in nanosec */ 6314 tu = rvu_read64(rvu, blkaddr, NIX_AF_PL_TS) & GENMASK_ULL(9, 0); 6315 rsp->policer_timeunit = (tu + 1) * 100; 6316 6317 return 0; 6318 } 6319 6320 static struct nix_mcast_grp_elem *rvu_nix_mcast_find_grp_elem(struct nix_mcast_grp *mcast_grp, 6321 u32 mcast_grp_idx) 6322 { 6323 struct nix_mcast_grp_elem *iter; 6324 bool is_found = false; 6325 6326 list_for_each_entry(iter, &mcast_grp->mcast_grp_head, list) { 6327 if (iter->mcast_grp_idx == mcast_grp_idx) { 6328 is_found = true; 6329 break; 6330 } 6331 } 6332 6333 if (is_found) 6334 return iter; 6335 6336 return NULL; 6337 } 6338 6339 int rvu_nix_mcast_get_mce_index(struct rvu *rvu, u16 pcifunc, u32 mcast_grp_idx) 6340 { 6341 struct nix_mcast_grp_elem *elem; 6342 struct nix_mcast_grp *mcast_grp; 6343 struct nix_hw *nix_hw; 6344 int blkaddr, ret; 6345 6346 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6347 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6348 if (!nix_hw) 6349 return NIX_AF_ERR_INVALID_NIXBLK; 6350 6351 mcast_grp = &nix_hw->mcast_grp; 6352 mutex_lock(&mcast_grp->mcast_grp_lock); 6353 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); 6354 if (!elem) 6355 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6356 else 6357 ret = elem->mce_start_index; 6358 6359 mutex_unlock(&mcast_grp->mcast_grp_lock); 6360 return ret; 6361 } 6362 6363 void rvu_nix_mcast_flr_free_entries(struct rvu *rvu, u16 pcifunc) 6364 { 6365 struct nix_mcast_grp_destroy_req dreq = { 0 }; 6366 struct nix_mcast_grp_update_req ureq = { 0 }; 6367 struct nix_mcast_grp_update_rsp ursp = { 0 }; 6368 struct nix_mcast_grp_elem *elem, *tmp; 6369 struct nix_mcast_grp *mcast_grp; 6370 struct nix_hw *nix_hw; 6371 int blkaddr; 6372 6373 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6374 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6375 if (!nix_hw) 6376 return; 6377 6378 mcast_grp = &nix_hw->mcast_grp; 6379 6380 mutex_lock(&mcast_grp->mcast_grp_lock); 6381 list_for_each_entry_safe(elem, tmp, &mcast_grp->mcast_grp_head, list) { 6382 struct nix_mce_list *mce_list; 6383 struct hlist_node *tmp; 6384 struct mce *mce; 6385 6386 /* If the pcifunc which created the multicast/mirror 6387 * group received an FLR, then delete the entire group. 6388 */ 6389 if (elem->pcifunc == pcifunc) { 6390 /* Delete group */ 6391 dreq.hdr.pcifunc = elem->pcifunc; 6392 dreq.mcast_grp_idx = elem->mcast_grp_idx; 6393 dreq.is_af = 1; 6394 rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL); 6395 continue; 6396 } 6397 6398 /* Iterate the group elements and delete the element which 6399 * received the FLR. 6400 */ 6401 mce_list = &elem->mcast_mce_list; 6402 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) { 6403 if (mce->pcifunc == pcifunc) { 6404 ureq.hdr.pcifunc = pcifunc; 6405 ureq.num_mce_entry = 1; 6406 ureq.mcast_grp_idx = elem->mcast_grp_idx; 6407 ureq.op = NIX_MCAST_OP_DEL_ENTRY; 6408 ureq.pcifunc[0] = pcifunc; 6409 ureq.is_af = 1; 6410 rvu_mbox_handler_nix_mcast_grp_update(rvu, &ureq, &ursp); 6411 break; 6412 } 6413 } 6414 } 6415 mutex_unlock(&mcast_grp->mcast_grp_lock); 6416 } 6417 6418 int rvu_nix_mcast_update_mcam_entry(struct rvu *rvu, u16 pcifunc, 6419 u32 mcast_grp_idx, u16 mcam_index) 6420 { 6421 struct nix_mcast_grp_elem *elem; 6422 struct nix_mcast_grp *mcast_grp; 6423 struct nix_hw *nix_hw; 6424 int blkaddr, ret = 0; 6425 6426 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 6427 nix_hw = get_nix_hw(rvu->hw, blkaddr); 6428 if (!nix_hw) 6429 return NIX_AF_ERR_INVALID_NIXBLK; 6430 6431 mcast_grp = &nix_hw->mcast_grp; 6432 mutex_lock(&mcast_grp->mcast_grp_lock); 6433 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx); 6434 if (!elem) 6435 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6436 else 6437 elem->mcam_index = mcam_index; 6438 6439 mutex_unlock(&mcast_grp->mcast_grp_lock); 6440 return ret; 6441 } 6442 6443 int rvu_mbox_handler_nix_mcast_grp_create(struct rvu *rvu, 6444 struct nix_mcast_grp_create_req *req, 6445 struct nix_mcast_grp_create_rsp *rsp) 6446 { 6447 struct nix_mcast_grp_elem *elem; 6448 struct nix_mcast_grp *mcast_grp; 6449 struct nix_hw *nix_hw; 6450 int blkaddr, err; 6451 6452 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6453 if (err) 6454 return err; 6455 6456 mcast_grp = &nix_hw->mcast_grp; 6457 elem = kzalloc_obj(*elem); 6458 if (!elem) 6459 return -ENOMEM; 6460 6461 INIT_HLIST_HEAD(&elem->mcast_mce_list.head); 6462 elem->mcam_index = -1; 6463 elem->mce_start_index = -1; 6464 elem->pcifunc = req->hdr.pcifunc; 6465 elem->dir = req->dir; 6466 elem->mcast_grp_idx = mcast_grp->next_grp_index++; 6467 6468 mutex_lock(&mcast_grp->mcast_grp_lock); 6469 list_add_tail(&elem->list, &mcast_grp->mcast_grp_head); 6470 mcast_grp->count++; 6471 mutex_unlock(&mcast_grp->mcast_grp_lock); 6472 6473 rsp->mcast_grp_idx = elem->mcast_grp_idx; 6474 return 0; 6475 } 6476 6477 int rvu_mbox_handler_nix_mcast_grp_destroy(struct rvu *rvu, 6478 struct nix_mcast_grp_destroy_req *req, 6479 struct msg_rsp *rsp) 6480 { 6481 struct npc_delete_flow_req uninstall_req = { 0 }; 6482 struct npc_delete_flow_rsp uninstall_rsp = { 0 }; 6483 struct nix_mcast_grp_elem *elem; 6484 struct nix_mcast_grp *mcast_grp; 6485 int blkaddr, err, ret = 0; 6486 struct nix_mcast *mcast; 6487 struct nix_hw *nix_hw; 6488 6489 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6490 if (err) 6491 return err; 6492 6493 mcast_grp = &nix_hw->mcast_grp; 6494 6495 /* If AF is requesting for the deletion, 6496 * then AF is already taking the lock 6497 */ 6498 if (!req->is_af) 6499 mutex_lock(&mcast_grp->mcast_grp_lock); 6500 6501 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); 6502 if (!elem) { 6503 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6504 goto unlock_grp; 6505 } 6506 6507 /* If no mce entries are associated with the group 6508 * then just remove it from the global list. 6509 */ 6510 if (!elem->mcast_mce_list.count) 6511 goto delete_grp; 6512 6513 /* Delete the associated mcam entry and 6514 * remove all mce entries from the group 6515 */ 6516 mcast = &nix_hw->mcast; 6517 mutex_lock(&mcast->mce_lock); 6518 if (elem->mcam_index != -1) { 6519 uninstall_req.hdr.pcifunc = req->hdr.pcifunc; 6520 uninstall_req.entry = elem->mcam_index; 6521 rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &uninstall_rsp); 6522 } 6523 6524 nix_free_mce_list(mcast, elem->mcast_mce_list.count, 6525 elem->mce_start_index, elem->dir); 6526 nix_delete_mcast_mce_list(&elem->mcast_mce_list); 6527 mutex_unlock(&mcast->mce_lock); 6528 6529 delete_grp: 6530 list_del(&elem->list); 6531 kfree(elem); 6532 mcast_grp->count--; 6533 6534 unlock_grp: 6535 if (!req->is_af) 6536 mutex_unlock(&mcast_grp->mcast_grp_lock); 6537 6538 return ret; 6539 } 6540 6541 int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu, 6542 struct nix_mcast_grp_update_req *req, 6543 struct nix_mcast_grp_update_rsp *rsp) 6544 { 6545 struct nix_mcast_grp_destroy_req dreq = { 0 }; 6546 struct npc_mcam *mcam = &rvu->hw->mcam; 6547 struct nix_mcast_grp_elem *elem; 6548 struct nix_mcast_grp *mcast_grp; 6549 int blkaddr, err, npc_blkaddr; 6550 u16 prev_count, new_count; 6551 struct nix_mcast *mcast; 6552 struct nix_hw *nix_hw; 6553 int i, ret; 6554 6555 if (!req->num_mce_entry) 6556 return 0; 6557 6558 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 6559 if (err) 6560 return err; 6561 6562 mcast_grp = &nix_hw->mcast_grp; 6563 6564 /* If AF is requesting for the updation, 6565 * then AF is already taking the lock 6566 */ 6567 if (!req->is_af) 6568 mutex_lock(&mcast_grp->mcast_grp_lock); 6569 6570 elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx); 6571 if (!elem) { 6572 ret = NIX_AF_ERR_INVALID_MCAST_GRP; 6573 goto unlock_grp; 6574 } 6575 6576 /* If any pcifunc matches the group's pcifunc, then we can 6577 * delete the entire group. 6578 */ 6579 if (req->op == NIX_MCAST_OP_DEL_ENTRY) { 6580 for (i = 0; i < req->num_mce_entry; i++) { 6581 if (elem->pcifunc == req->pcifunc[i]) { 6582 /* Delete group */ 6583 dreq.hdr.pcifunc = elem->pcifunc; 6584 dreq.mcast_grp_idx = elem->mcast_grp_idx; 6585 dreq.is_af = 1; 6586 rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL); 6587 ret = 0; 6588 goto unlock_grp; 6589 } 6590 } 6591 } 6592 6593 mcast = &nix_hw->mcast; 6594 mutex_lock(&mcast->mce_lock); 6595 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 6596 if (elem->mcam_index != -1) 6597 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, false); 6598 6599 prev_count = elem->mcast_mce_list.count; 6600 if (req->op == NIX_MCAST_OP_ADD_ENTRY) { 6601 new_count = prev_count + req->num_mce_entry; 6602 if (prev_count) 6603 nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir); 6604 6605 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir); 6606 6607 /* It is possible not to get contiguous memory */ 6608 if (elem->mce_start_index < 0) { 6609 if (elem->mcam_index != -1) { 6610 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6611 elem->mcam_index, true); 6612 ret = NIX_AF_ERR_NON_CONTIG_MCE_LIST; 6613 goto unlock_mce; 6614 } 6615 } 6616 6617 ret = nix_add_mce_list_entry(rvu, nix_hw, elem, req); 6618 if (ret) { 6619 nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir); 6620 if (prev_count) 6621 elem->mce_start_index = nix_alloc_mce_list(mcast, 6622 prev_count, 6623 elem->dir); 6624 6625 if (elem->mcam_index != -1) 6626 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6627 elem->mcam_index, true); 6628 6629 goto unlock_mce; 6630 } 6631 } else { 6632 if (!prev_count || prev_count < req->num_mce_entry) { 6633 if (elem->mcam_index != -1) 6634 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, 6635 elem->mcam_index, true); 6636 ret = NIX_AF_ERR_INVALID_MCAST_DEL_REQ; 6637 goto unlock_mce; 6638 } 6639 6640 nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir); 6641 new_count = prev_count - req->num_mce_entry; 6642 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir); 6643 ret = nix_del_mce_list_entry(rvu, nix_hw, elem, req); 6644 if (ret) { 6645 nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir); 6646 elem->mce_start_index = nix_alloc_mce_list(mcast, prev_count, elem->dir); 6647 if (elem->mcam_index != -1) 6648 npc_enable_mcam_entry(rvu, mcam, 6649 npc_blkaddr, 6650 elem->mcam_index, 6651 true); 6652 6653 goto unlock_mce; 6654 } 6655 } 6656 6657 if (elem->mcam_index == -1) { 6658 rsp->mce_start_index = elem->mce_start_index; 6659 ret = 0; 6660 goto unlock_mce; 6661 } 6662 6663 nix_mcast_update_action(rvu, elem); 6664 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true); 6665 rsp->mce_start_index = elem->mce_start_index; 6666 ret = 0; 6667 6668 unlock_mce: 6669 mutex_unlock(&mcast->mce_lock); 6670 6671 unlock_grp: 6672 if (!req->is_af) 6673 mutex_unlock(&mcast_grp->mcast_grp_lock); 6674 6675 return ret; 6676 } 6677 6678 /* On CN10k and older series of silicons, hardware may incorrectly 6679 * assert XOFF on certain channels. Issue a write on NIX_AF_RX_CHANX_CFG 6680 * to broadcacst XON on the same. 6681 */ 6682 void rvu_block_bcast_xon(struct rvu *rvu, int blkaddr) 6683 { 6684 struct rvu_block *block = &rvu->hw->block[blkaddr]; 6685 u64 cfg; 6686 6687 if (!block->implemented || is_cn20k(rvu->pdev)) 6688 return; 6689 6690 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(0)); 6691 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(0), cfg); 6692 } 6693