1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/interrupt.h> 9 #include <linux/pci.h> 10 #include <net/page_pool/helpers.h> 11 #include <net/tso.h> 12 #include <linux/bitfield.h> 13 #include <linux/dcbnl.h> 14 #include <net/xfrm.h> 15 16 #include "otx2_reg.h" 17 #include "otx2_common.h" 18 #include "otx2_struct.h" 19 #include "cn10k.h" 20 #include "otx2_xsk.h" 21 22 static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf) 23 { 24 return IS_ENABLED(CONFIG_DCB) && !!pfvf->pfc_en; 25 } 26 27 static void otx2_nix_rq_op_stats(struct queue_stats *stats, 28 struct otx2_nic *pfvf, int qidx) 29 { 30 u64 incr = (u64)qidx << 32; 31 void __iomem *ptr; 32 33 ptr = otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_OCTS); 34 stats->bytes = otx2_atomic64_add(incr, ptr); 35 36 ptr = otx2_get_regaddr(pfvf, NIX_LF_RQ_OP_PKTS); 37 stats->pkts = otx2_atomic64_add(incr, ptr); 38 } 39 40 static void otx2_nix_sq_op_stats(struct queue_stats *stats, 41 struct otx2_nic *pfvf, int qidx) 42 { 43 u64 incr = (u64)qidx << 32; 44 void __iomem *ptr; 45 46 ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_OCTS); 47 stats->bytes = otx2_atomic64_add(incr, ptr); 48 49 ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_PKTS); 50 stats->pkts = otx2_atomic64_add(incr, ptr); 51 } 52 53 void otx2_update_lmac_stats(struct otx2_nic *pfvf) 54 { 55 struct msg_req *req; 56 57 if (!netif_running(pfvf->netdev)) 58 return; 59 60 mutex_lock(&pfvf->mbox.lock); 61 req = otx2_mbox_alloc_msg_cgx_stats(&pfvf->mbox); 62 if (!req) { 63 mutex_unlock(&pfvf->mbox.lock); 64 return; 65 } 66 67 otx2_sync_mbox_msg(&pfvf->mbox); 68 mutex_unlock(&pfvf->mbox.lock); 69 } 70 71 void otx2_update_lmac_fec_stats(struct otx2_nic *pfvf) 72 { 73 struct msg_req *req; 74 75 if (!netif_running(pfvf->netdev)) 76 return; 77 mutex_lock(&pfvf->mbox.lock); 78 req = otx2_mbox_alloc_msg_cgx_fec_stats(&pfvf->mbox); 79 if (req) 80 otx2_sync_mbox_msg(&pfvf->mbox); 81 mutex_unlock(&pfvf->mbox.lock); 82 } 83 84 int otx2_update_rq_stats(struct otx2_nic *pfvf, int qidx) 85 { 86 struct otx2_rcv_queue *rq = &pfvf->qset.rq[qidx]; 87 88 if (!pfvf->qset.rq) 89 return 0; 90 91 otx2_nix_rq_op_stats(&rq->stats, pfvf, qidx); 92 return 1; 93 } 94 EXPORT_SYMBOL(otx2_update_rq_stats); 95 96 int otx2_update_sq_stats(struct otx2_nic *pfvf, int qidx) 97 { 98 struct otx2_snd_queue *sq = &pfvf->qset.sq[qidx]; 99 100 if (!pfvf->qset.sq) 101 return 0; 102 103 if (qidx >= pfvf->hw.non_qos_queues) { 104 if (!test_bit(qidx - pfvf->hw.non_qos_queues, pfvf->qos.qos_sq_bmap)) 105 return 0; 106 } 107 108 otx2_nix_sq_op_stats(&sq->stats, pfvf, qidx); 109 return 1; 110 } 111 EXPORT_SYMBOL(otx2_update_sq_stats); 112 113 void otx2_get_dev_stats(struct otx2_nic *pfvf) 114 { 115 struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; 116 117 dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); 118 dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); 119 dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); 120 dev_stats->rx_mcast_frames = OTX2_GET_RX_STATS(RX_MCAST); 121 dev_stats->rx_ucast_frames = OTX2_GET_RX_STATS(RX_UCAST); 122 dev_stats->rx_frames = dev_stats->rx_bcast_frames + 123 dev_stats->rx_mcast_frames + 124 dev_stats->rx_ucast_frames; 125 126 dev_stats->tx_bytes = OTX2_GET_TX_STATS(TX_OCTS); 127 dev_stats->tx_drops = OTX2_GET_TX_STATS(TX_DROP) + 128 (unsigned long)atomic_long_read(&dev_stats->tx_discards); 129 130 dev_stats->tx_bcast_frames = OTX2_GET_TX_STATS(TX_BCAST); 131 dev_stats->tx_mcast_frames = OTX2_GET_TX_STATS(TX_MCAST); 132 dev_stats->tx_ucast_frames = OTX2_GET_TX_STATS(TX_UCAST); 133 dev_stats->tx_frames = dev_stats->tx_bcast_frames + 134 dev_stats->tx_mcast_frames + 135 dev_stats->tx_ucast_frames; 136 } 137 138 void otx2_get_stats64(struct net_device *netdev, 139 struct rtnl_link_stats64 *stats) 140 { 141 struct otx2_nic *pfvf = netdev_priv(netdev); 142 struct otx2_dev_stats *dev_stats; 143 144 otx2_get_dev_stats(pfvf); 145 146 dev_stats = &pfvf->hw.dev_stats; 147 stats->rx_bytes = dev_stats->rx_bytes; 148 stats->rx_packets = dev_stats->rx_frames; 149 stats->rx_dropped = dev_stats->rx_drops; 150 stats->multicast = dev_stats->rx_mcast_frames; 151 152 stats->tx_bytes = dev_stats->tx_bytes; 153 stats->tx_packets = dev_stats->tx_frames; 154 stats->tx_dropped = dev_stats->tx_drops; 155 } 156 EXPORT_SYMBOL(otx2_get_stats64); 157 158 /* Sync MAC address with RVU AF */ 159 static int otx2_hw_set_mac_addr(struct otx2_nic *pfvf, u8 *mac) 160 { 161 struct nix_set_mac_addr *req; 162 int err; 163 164 mutex_lock(&pfvf->mbox.lock); 165 req = otx2_mbox_alloc_msg_nix_set_mac_addr(&pfvf->mbox); 166 if (!req) { 167 mutex_unlock(&pfvf->mbox.lock); 168 return -ENOMEM; 169 } 170 171 ether_addr_copy(req->mac_addr, mac); 172 173 err = otx2_sync_mbox_msg(&pfvf->mbox); 174 mutex_unlock(&pfvf->mbox.lock); 175 return err; 176 } 177 178 static int otx2_hw_get_mac_addr(struct otx2_nic *pfvf, 179 struct net_device *netdev) 180 { 181 struct nix_get_mac_addr_rsp *rsp; 182 struct mbox_msghdr *msghdr; 183 struct msg_req *req; 184 int err; 185 186 mutex_lock(&pfvf->mbox.lock); 187 req = otx2_mbox_alloc_msg_nix_get_mac_addr(&pfvf->mbox); 188 if (!req) { 189 mutex_unlock(&pfvf->mbox.lock); 190 return -ENOMEM; 191 } 192 193 err = otx2_sync_mbox_msg(&pfvf->mbox); 194 if (err) { 195 mutex_unlock(&pfvf->mbox.lock); 196 return err; 197 } 198 199 msghdr = otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 200 if (IS_ERR(msghdr)) { 201 mutex_unlock(&pfvf->mbox.lock); 202 return PTR_ERR(msghdr); 203 } 204 rsp = (struct nix_get_mac_addr_rsp *)msghdr; 205 eth_hw_addr_set(netdev, rsp->mac_addr); 206 mutex_unlock(&pfvf->mbox.lock); 207 208 return 0; 209 } 210 211 int otx2_set_mac_address(struct net_device *netdev, void *p) 212 { 213 struct otx2_nic *pfvf = netdev_priv(netdev); 214 struct sockaddr *addr = p; 215 216 if (!is_valid_ether_addr(addr->sa_data)) 217 return -EADDRNOTAVAIL; 218 219 if (!otx2_hw_set_mac_addr(pfvf, addr->sa_data)) { 220 eth_hw_addr_set(netdev, addr->sa_data); 221 /* update dmac field in vlan offload rule */ 222 if (netif_running(netdev) && 223 pfvf->flags & OTX2_FLAG_RX_VLAN_SUPPORT) 224 otx2_install_rxvlan_offload_flow(pfvf); 225 /* update dmac address in ntuple and DMAC filter list */ 226 if (pfvf->flags & OTX2_FLAG_DMACFLTR_SUPPORT) 227 otx2_dmacflt_update_pfmac_flow(pfvf); 228 } else { 229 return -EPERM; 230 } 231 232 return 0; 233 } 234 EXPORT_SYMBOL(otx2_set_mac_address); 235 236 int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) 237 { 238 struct nix_frs_cfg *req; 239 u16 maxlen; 240 int err; 241 242 maxlen = pfvf->hw.max_mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; 243 244 mutex_lock(&pfvf->mbox.lock); 245 req = otx2_mbox_alloc_msg_nix_set_hw_frs(&pfvf->mbox); 246 if (!req) { 247 mutex_unlock(&pfvf->mbox.lock); 248 return -ENOMEM; 249 } 250 251 req->maxlen = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; 252 253 /* Use max receive length supported by hardware for loopback devices */ 254 if (is_otx2_lbkvf(pfvf->pdev)) 255 req->maxlen = maxlen; 256 257 err = otx2_sync_mbox_msg(&pfvf->mbox); 258 mutex_unlock(&pfvf->mbox.lock); 259 return err; 260 } 261 EXPORT_SYMBOL(otx2_hw_set_mtu); 262 263 int otx2_config_pause_frm(struct otx2_nic *pfvf) 264 { 265 struct cgx_pause_frm_cfg *req; 266 int err; 267 268 if (is_otx2_lbkvf(pfvf->pdev) || is_otx2_sdp_rep(pfvf->pdev)) 269 return 0; 270 271 mutex_lock(&pfvf->mbox.lock); 272 req = otx2_mbox_alloc_msg_cgx_cfg_pause_frm(&pfvf->mbox); 273 if (!req) { 274 err = -ENOMEM; 275 goto unlock; 276 } 277 278 req->rx_pause = !!(pfvf->flags & OTX2_FLAG_RX_PAUSE_ENABLED); 279 req->tx_pause = !!(pfvf->flags & OTX2_FLAG_TX_PAUSE_ENABLED); 280 req->set = 1; 281 282 err = otx2_sync_mbox_msg(&pfvf->mbox); 283 unlock: 284 mutex_unlock(&pfvf->mbox.lock); 285 return err; 286 } 287 EXPORT_SYMBOL(otx2_config_pause_frm); 288 289 int otx2_set_flowkey_cfg(struct otx2_nic *pfvf) 290 { 291 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 292 struct nix_rss_flowkey_cfg_rsp *rsp; 293 struct nix_rss_flowkey_cfg *req; 294 int err; 295 296 mutex_lock(&pfvf->mbox.lock); 297 req = otx2_mbox_alloc_msg_nix_rss_flowkey_cfg(&pfvf->mbox); 298 if (!req) { 299 mutex_unlock(&pfvf->mbox.lock); 300 return -ENOMEM; 301 } 302 req->mcam_index = -1; /* Default or reserved index */ 303 req->flowkey_cfg = rss->flowkey_cfg; 304 req->group = DEFAULT_RSS_CONTEXT_GROUP; 305 306 err = otx2_sync_mbox_msg(&pfvf->mbox); 307 if (err) 308 goto fail; 309 310 rsp = (struct nix_rss_flowkey_cfg_rsp *) 311 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 312 if (IS_ERR(rsp)) { 313 err = PTR_ERR(rsp); 314 goto fail; 315 } 316 317 pfvf->hw.flowkey_alg_idx = rsp->alg_idx; 318 fail: 319 mutex_unlock(&pfvf->mbox.lock); 320 return err; 321 } 322 323 int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id, const u32 *ind_tbl) 324 { 325 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 326 const int index = rss->rss_size * ctx_id; 327 struct mbox *mbox = &pfvf->mbox; 328 struct nix_aq_enq_req *aq; 329 int idx, err; 330 331 mutex_lock(&mbox->lock); 332 ind_tbl = ind_tbl ?: rss->ind_tbl; 333 /* Get memory to put this msg */ 334 for (idx = 0; idx < rss->rss_size; idx++) { 335 /* Ignore the queue if AF_XDP zero copy is enabled */ 336 if (test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) 337 continue; 338 339 aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); 340 if (!aq) { 341 /* The shared memory buffer can be full. 342 * Flush it and retry 343 */ 344 err = otx2_sync_mbox_msg(mbox); 345 if (err) { 346 mutex_unlock(&mbox->lock); 347 return err; 348 } 349 aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); 350 if (!aq) { 351 mutex_unlock(&mbox->lock); 352 return -ENOMEM; 353 } 354 } 355 356 aq->rss.rq = ind_tbl[idx]; 357 358 /* Fill AQ info */ 359 aq->qidx = index + idx; 360 aq->ctype = NIX_AQ_CTYPE_RSS; 361 aq->op = NIX_AQ_INSTOP_INIT; 362 } 363 err = otx2_sync_mbox_msg(mbox); 364 mutex_unlock(&mbox->lock); 365 return err; 366 } 367 368 void otx2_set_rss_key(struct otx2_nic *pfvf) 369 { 370 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 371 u64 *key = (u64 *)&rss->key[4]; 372 int idx; 373 374 /* 352bit or 44byte key needs to be configured as below 375 * NIX_LF_RX_SECRETX0 = key<351:288> 376 * NIX_LF_RX_SECRETX1 = key<287:224> 377 * NIX_LF_RX_SECRETX2 = key<223:160> 378 * NIX_LF_RX_SECRETX3 = key<159:96> 379 * NIX_LF_RX_SECRETX4 = key<95:32> 380 * NIX_LF_RX_SECRETX5<63:32> = key<31:0> 381 */ 382 otx2_write64(pfvf, NIX_LF_RX_SECRETX(5), 383 (u64)(*((u32 *)&rss->key)) << 32); 384 idx = sizeof(rss->key) / sizeof(u64); 385 while (idx > 0) { 386 idx--; 387 otx2_write64(pfvf, NIX_LF_RX_SECRETX(idx), *key++); 388 } 389 } 390 391 int otx2_rss_init(struct otx2_nic *pfvf) 392 { 393 struct otx2_rss_info *rss = &pfvf->hw.rss_info; 394 int idx, ret = 0; 395 396 rss->rss_size = sizeof(*rss->ind_tbl); 397 398 /* Init RSS key if it is not setup already */ 399 if (!rss->enable) 400 netdev_rss_key_fill(rss->key, sizeof(rss->key)); 401 otx2_set_rss_key(pfvf); 402 403 if (!netif_is_rxfh_configured(pfvf->netdev)) 404 for (idx = 0; idx < rss->rss_size; idx++) 405 rss->ind_tbl[idx] = 406 ethtool_rxfh_indir_default(idx, 407 pfvf->hw.rx_queues); 408 409 ret = otx2_set_rss_table(pfvf, DEFAULT_RSS_CONTEXT_GROUP, NULL); 410 if (ret) 411 return ret; 412 413 /* Flowkey or hash config to be used for generating flow tag */ 414 rss->flowkey_cfg = rss->enable ? rss->flowkey_cfg : 415 NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6 | 416 NIX_FLOW_KEY_TYPE_TCP | NIX_FLOW_KEY_TYPE_UDP | 417 NIX_FLOW_KEY_TYPE_SCTP | NIX_FLOW_KEY_TYPE_VLAN | 418 NIX_FLOW_KEY_TYPE_IPV4_PROTO; 419 420 ret = otx2_set_flowkey_cfg(pfvf); 421 if (ret) 422 return ret; 423 424 rss->enable = true; 425 return 0; 426 } 427 428 /* Setup UDP segmentation algorithm in HW */ 429 static void otx2_setup_udp_segmentation(struct nix_lso_format_cfg *lso, bool v4) 430 { 431 struct nix_lso_format *field; 432 433 field = (struct nix_lso_format *)&lso->fields[0]; 434 lso->field_mask = GENMASK(18, 0); 435 436 /* IP's Length field */ 437 field->layer = NIX_TXLAYER_OL3; 438 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ 439 field->offset = v4 ? 2 : 4; 440 field->sizem1 = 1; /* i.e 2 bytes */ 441 field->alg = NIX_LSOALG_ADD_PAYLEN; 442 field++; 443 444 /* No ID field in IPv6 header */ 445 if (v4) { 446 /* Increment IPID */ 447 field->layer = NIX_TXLAYER_OL3; 448 field->offset = 4; 449 field->sizem1 = 1; /* i.e 2 bytes */ 450 field->alg = NIX_LSOALG_ADD_SEGNUM; 451 field++; 452 } 453 454 /* Update length in UDP header */ 455 field->layer = NIX_TXLAYER_OL4; 456 field->offset = 4; 457 field->sizem1 = 1; 458 field->alg = NIX_LSOALG_ADD_PAYLEN; 459 } 460 461 /* Setup segmentation algorithms in HW and retrieve algorithm index */ 462 void otx2_setup_segmentation(struct otx2_nic *pfvf) 463 { 464 struct nix_lso_format_cfg_rsp *rsp; 465 struct nix_lso_format_cfg *lso; 466 struct otx2_hw *hw = &pfvf->hw; 467 int err; 468 469 mutex_lock(&pfvf->mbox.lock); 470 471 /* UDPv4 segmentation */ 472 lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); 473 if (!lso) 474 goto fail; 475 476 /* Setup UDP/IP header fields that HW should update per segment */ 477 otx2_setup_udp_segmentation(lso, true); 478 479 err = otx2_sync_mbox_msg(&pfvf->mbox); 480 if (err) 481 goto fail; 482 483 rsp = (struct nix_lso_format_cfg_rsp *) 484 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); 485 if (IS_ERR(rsp)) 486 goto fail; 487 488 hw->lso_udpv4_idx = rsp->lso_format_idx; 489 490 /* UDPv6 segmentation */ 491 lso = otx2_mbox_alloc_msg_nix_lso_format_cfg(&pfvf->mbox); 492 if (!lso) 493 goto fail; 494 495 /* Setup UDP/IP header fields that HW should update per segment */ 496 otx2_setup_udp_segmentation(lso, false); 497 498 err = otx2_sync_mbox_msg(&pfvf->mbox); 499 if (err) 500 goto fail; 501 502 rsp = (struct nix_lso_format_cfg_rsp *) 503 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &lso->hdr); 504 if (IS_ERR(rsp)) 505 goto fail; 506 507 hw->lso_udpv6_idx = rsp->lso_format_idx; 508 mutex_unlock(&pfvf->mbox.lock); 509 return; 510 fail: 511 mutex_unlock(&pfvf->mbox.lock); 512 netdev_info(pfvf->netdev, 513 "Failed to get LSO index for UDP GSO offload, disabling\n"); 514 pfvf->netdev->hw_features &= ~NETIF_F_GSO_UDP_L4; 515 } 516 517 void otx2_config_irq_coalescing(struct otx2_nic *pfvf, int qidx) 518 { 519 /* Configure CQE interrupt coalescing parameters 520 * 521 * HW triggers an irq when ECOUNT > cq_ecount_wait, hence 522 * set 1 less than cq_ecount_wait. And cq_time_wait is in 523 * usecs, convert that to 100ns count. 524 */ 525 otx2_write64(pfvf, NIX_LF_CINTX_WAIT(qidx), 526 ((u64)(pfvf->hw.cq_time_wait * 10) << 48) | 527 ((u64)pfvf->hw.cq_qcount_wait << 32) | 528 (pfvf->hw.cq_ecount_wait - 1)); 529 } 530 531 static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, 532 dma_addr_t *dma) 533 { 534 unsigned int offset = 0; 535 struct page *page; 536 size_t sz; 537 538 sz = SKB_DATA_ALIGN(pool->rbsize); 539 sz = ALIGN(sz, OTX2_ALIGN); 540 541 page = page_pool_alloc_frag(pool->page_pool, &offset, sz, GFP_ATOMIC); 542 if (unlikely(!page)) 543 return -ENOMEM; 544 545 *dma = page_pool_get_dma_addr(page) + offset; 546 return 0; 547 } 548 549 static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 550 dma_addr_t *dma, int qidx, int idx) 551 { 552 u8 *buf; 553 554 if (pool->xsk_pool) 555 return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx); 556 557 if (pool->page_pool) 558 return otx2_alloc_pool_buf(pfvf, pool, dma); 559 560 buf = napi_alloc_frag_align(pool->rbsize, OTX2_ALIGN); 561 if (unlikely(!buf)) 562 return -ENOMEM; 563 564 *dma = dma_map_single_attrs(pfvf->dev, buf, pool->rbsize, 565 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 566 if (unlikely(dma_mapping_error(pfvf->dev, *dma))) { 567 page_frag_free(buf); 568 return -ENOMEM; 569 } 570 571 return 0; 572 } 573 574 int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, 575 dma_addr_t *dma, int qidx, int idx) 576 { 577 int ret; 578 579 local_bh_disable(); 580 ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx); 581 local_bh_enable(); 582 return ret; 583 } 584 585 int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, 586 dma_addr_t *dma) 587 { 588 if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma, 589 cq->cq_idx, cq->pool_ptrs - 1))) 590 return -ENOMEM; 591 return 0; 592 } 593 594 void otx2_tx_timeout(struct net_device *netdev, unsigned int txq) 595 { 596 struct otx2_nic *pfvf = netdev_priv(netdev); 597 598 schedule_work(&pfvf->reset_task); 599 } 600 EXPORT_SYMBOL(otx2_tx_timeout); 601 602 void otx2_get_mac_from_af(struct net_device *netdev) 603 { 604 struct otx2_nic *pfvf = netdev_priv(netdev); 605 int err; 606 607 err = otx2_hw_get_mac_addr(pfvf, netdev); 608 if (err) 609 dev_warn(pfvf->dev, "Failed to read mac from hardware\n"); 610 611 /* If AF doesn't provide a valid MAC, generate a random one */ 612 if (!is_valid_ether_addr(netdev->dev_addr)) 613 eth_hw_addr_random(netdev); 614 } 615 EXPORT_SYMBOL(otx2_get_mac_from_af); 616 617 int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) 618 { 619 u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; 620 struct otx2_hw *hw = &pfvf->hw; 621 struct nix_txschq_config *req; 622 u64 schq, parent; 623 u64 dwrr_val; 624 625 dwrr_val = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 626 627 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); 628 if (!req) 629 return -ENOMEM; 630 631 req->lvl = lvl; 632 req->num_regs = 1; 633 634 schq_list = hw->txschq_list; 635 #ifdef CONFIG_DCB 636 if (txschq_for_pfc) 637 schq_list = pfvf->pfc_schq_list; 638 #endif 639 640 schq = schq_list[lvl][prio]; 641 /* Set topology e.t.c configuration */ 642 if (lvl == NIX_TXSCH_LVL_SMQ) { 643 req->reg[0] = NIX_AF_SMQX_CFG(schq); 644 req->regval[0] = ((u64)pfvf->tx_max_pktlen << 8) | OTX2_MIN_MTU; 645 req->regval[0] |= (0x20ULL << 51) | (0x80ULL << 39) | 646 (0x2ULL << 36); 647 /* Set link type for DWRR MTU selection on CN10K silicons */ 648 if (!is_dev_otx2(pfvf->pdev)) 649 req->regval[0] |= FIELD_PREP(GENMASK_ULL(58, 57), 650 (u64)hw->smq_link_type); 651 req->num_regs++; 652 /* MDQ config */ 653 parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; 654 req->reg[1] = NIX_AF_MDQX_PARENT(schq); 655 req->regval[1] = parent << 16; 656 req->num_regs++; 657 /* Set DWRR quantum */ 658 req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); 659 req->regval[2] = dwrr_val; 660 } else if (lvl == NIX_TXSCH_LVL_TL4) { 661 int sdp_chan = hw->tx_chan_base + prio; 662 663 if (is_otx2_sdp_rep(pfvf->pdev)) 664 prio = 0; 665 parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; 666 req->reg[0] = NIX_AF_TL4X_PARENT(schq); 667 req->regval[0] = (u64)parent << 16; 668 req->num_regs++; 669 req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); 670 req->regval[1] = dwrr_val; 671 if (is_otx2_sdp_rep(pfvf->pdev)) { 672 req->num_regs++; 673 req->reg[2] = NIX_AF_TL4X_SDP_LINK_CFG(schq); 674 req->regval[2] = BIT_ULL(12) | BIT_ULL(13) | 675 (sdp_chan & 0xff); 676 } 677 } else if (lvl == NIX_TXSCH_LVL_TL3) { 678 parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; 679 req->reg[0] = NIX_AF_TL3X_PARENT(schq); 680 req->regval[0] = (u64)parent << 16; 681 req->num_regs++; 682 req->reg[1] = NIX_AF_TL3X_SCHEDULE(schq); 683 req->regval[1] = dwrr_val; 684 if (lvl == hw->txschq_link_cfg_lvl && 685 !is_otx2_sdp_rep(pfvf->pdev)) { 686 req->num_regs++; 687 req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); 688 /* Enable this queue and backpressure 689 * and set relative channel 690 */ 691 req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; 692 } 693 } else if (lvl == NIX_TXSCH_LVL_TL2) { 694 parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; 695 req->reg[0] = NIX_AF_TL2X_PARENT(schq); 696 req->regval[0] = (u64)parent << 16; 697 698 req->num_regs++; 699 req->reg[1] = NIX_AF_TL2X_SCHEDULE(schq); 700 req->regval[1] = (u64)hw->txschq_aggr_lvl_rr_prio << 24 | dwrr_val; 701 702 if (lvl == hw->txschq_link_cfg_lvl && 703 !is_otx2_sdp_rep(pfvf->pdev)) { 704 req->num_regs++; 705 req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); 706 /* Enable this queue and backpressure 707 * and set relative channel 708 */ 709 req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; 710 } 711 } else if (lvl == NIX_TXSCH_LVL_TL1) { 712 /* Default config for TL1. 713 * For VF this is always ignored. 714 */ 715 716 /* On CN10K, if RR_WEIGHT is greater than 16384, HW will 717 * clip it to 16384, so configuring a 24bit max value 718 * will work on both OTx2 and CN10K. 719 */ 720 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq); 721 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM; 722 723 req->num_regs++; 724 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq); 725 req->regval[1] = hw->txschq_aggr_lvl_rr_prio << 1; 726 727 req->num_regs++; 728 req->reg[2] = NIX_AF_TL1X_CIR(schq); 729 req->regval[2] = 0; 730 } 731 732 return otx2_sync_mbox_msg(&pfvf->mbox); 733 } 734 EXPORT_SYMBOL(otx2_txschq_config); 735 736 int otx2_smq_flush(struct otx2_nic *pfvf, int smq) 737 { 738 struct nix_txschq_config *req; 739 int rc; 740 741 mutex_lock(&pfvf->mbox.lock); 742 743 req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); 744 if (!req) { 745 mutex_unlock(&pfvf->mbox.lock); 746 return -ENOMEM; 747 } 748 749 req->lvl = NIX_TXSCH_LVL_SMQ; 750 req->reg[0] = NIX_AF_SMQX_CFG(smq); 751 req->regval[0] |= BIT_ULL(49); 752 req->num_regs++; 753 754 rc = otx2_sync_mbox_msg(&pfvf->mbox); 755 mutex_unlock(&pfvf->mbox.lock); 756 return rc; 757 } 758 EXPORT_SYMBOL(otx2_smq_flush); 759 760 int otx2_txsch_alloc(struct otx2_nic *pfvf) 761 { 762 int chan_cnt = pfvf->hw.tx_chan_cnt; 763 struct nix_txsch_alloc_req *req; 764 struct nix_txsch_alloc_rsp *rsp; 765 int lvl, schq, rc; 766 767 /* Get memory to put this msg */ 768 req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); 769 if (!req) 770 return -ENOMEM; 771 772 /* Request one schq per level */ 773 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) 774 req->schq[lvl] = 1; 775 776 if (is_otx2_sdp_rep(pfvf->pdev) && chan_cnt > 1) { 777 req->schq[NIX_TXSCH_LVL_SMQ] = chan_cnt; 778 req->schq[NIX_TXSCH_LVL_TL4] = chan_cnt; 779 } 780 781 rc = otx2_sync_mbox_msg(&pfvf->mbox); 782 if (rc) 783 return rc; 784 785 rsp = (struct nix_txsch_alloc_rsp *) 786 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 787 if (IS_ERR(rsp)) 788 return PTR_ERR(rsp); 789 790 /* Setup transmit scheduler list */ 791 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 792 pfvf->hw.txschq_cnt[lvl] = rsp->schq[lvl]; 793 for (schq = 0; schq < rsp->schq[lvl]; schq++) 794 pfvf->hw.txschq_list[lvl][schq] = 795 rsp->schq_list[lvl][schq]; 796 } 797 798 pfvf->hw.txschq_link_cfg_lvl = rsp->link_cfg_lvl; 799 pfvf->hw.txschq_aggr_lvl_rr_prio = rsp->aggr_lvl_rr_prio; 800 801 return 0; 802 } 803 804 void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq) 805 { 806 struct nix_txsch_free_req *free_req; 807 int err; 808 809 mutex_lock(&pfvf->mbox.lock); 810 811 free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); 812 if (!free_req) { 813 mutex_unlock(&pfvf->mbox.lock); 814 netdev_err(pfvf->netdev, 815 "Failed alloc txschq free req\n"); 816 return; 817 } 818 819 free_req->schq_lvl = lvl; 820 free_req->schq = schq; 821 822 err = otx2_sync_mbox_msg(&pfvf->mbox); 823 if (err) { 824 netdev_err(pfvf->netdev, 825 "Failed stop txschq %d at level %d\n", schq, lvl); 826 } 827 828 mutex_unlock(&pfvf->mbox.lock); 829 } 830 EXPORT_SYMBOL(otx2_txschq_free_one); 831 832 void otx2_txschq_stop(struct otx2_nic *pfvf) 833 { 834 int lvl, schq, idx; 835 836 /* free non QOS TLx nodes */ 837 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 838 for (idx = 0; idx < pfvf->hw.txschq_cnt[lvl]; idx++) { 839 otx2_txschq_free_one(pfvf, lvl, 840 pfvf->hw.txschq_list[lvl][idx]); 841 } 842 } 843 844 /* Clear the txschq list */ 845 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 846 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) 847 pfvf->hw.txschq_list[lvl][schq] = 0; 848 } 849 850 } 851 852 void otx2_sqb_flush(struct otx2_nic *pfvf) 853 { 854 int qidx, sqe_tail, sqe_head; 855 struct otx2_snd_queue *sq; 856 void __iomem *ptr; 857 u64 incr, val; 858 859 ptr = otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_STATUS); 860 for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { 861 sq = &pfvf->qset.sq[qidx]; 862 if (!sq->sqb_ptrs) 863 continue; 864 865 incr = (u64)qidx << 32; 866 val = otx2_atomic64_add(incr, ptr); 867 sqe_head = (val >> 20) & 0x3F; 868 sqe_tail = (val >> 28) & 0x3F; 869 if (sqe_head != sqe_tail) 870 usleep_range(50, 60); 871 } 872 } 873 874 /* RED and drop levels of CQ on packet reception. 875 * For CQ level is measure of emptiness ( 0x0 = full, 255 = empty). 876 */ 877 #define RQ_PASS_LVL_CQ(skid, qsize) ((((skid) + 16) * 256) / (qsize)) 878 #define RQ_DROP_LVL_CQ(skid, qsize) (((skid) * 256) / (qsize)) 879 880 /* RED and drop levels of AURA for packet reception. 881 * For AURA level is measure of fullness (0x0 = empty, 255 = full). 882 * Eg: For RQ length 1K, for pass/drop level 204/230. 883 * RED accepts pkts if free pointers > 102 & <= 205. 884 * Drops pkts if free pointers < 102. 885 */ 886 #define RQ_BP_LVL_AURA (255 - ((85 * 256) / 100)) /* BP when 85% is full */ 887 #define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ 888 #define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ 889 890 int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) 891 { 892 struct otx2_qset *qset = &pfvf->qset; 893 struct nix_aq_enq_req *aq; 894 895 /* Get memory to put this msg */ 896 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 897 if (!aq) 898 return -ENOMEM; 899 900 aq->rq.cq = qidx; 901 aq->rq.ena = 1; 902 aq->rq.pb_caching = 1; 903 aq->rq.lpb_aura = lpb_aura; /* Use large packet buffer aura */ 904 aq->rq.lpb_sizem1 = (DMA_BUFFER_LEN(pfvf->rbsize) / 8) - 1; 905 aq->rq.xqe_imm_size = 0; /* Copying of packet to CQE not needed */ 906 aq->rq.flow_tagw = 32; /* Copy full 32bit flow_tag to CQE header */ 907 aq->rq.qint_idx = 0; 908 aq->rq.lpb_drop_ena = 1; /* Enable RED dropping for AURA */ 909 aq->rq.xqe_drop_ena = 1; /* Enable RED dropping for CQ/SSO */ 910 aq->rq.xqe_pass = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 911 aq->rq.xqe_drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 912 aq->rq.lpb_aura_pass = RQ_PASS_LVL_AURA; 913 aq->rq.lpb_aura_drop = RQ_DROP_LVL_AURA; 914 915 /* Fill AQ info */ 916 aq->qidx = qidx; 917 aq->ctype = NIX_AQ_CTYPE_RQ; 918 aq->op = NIX_AQ_INSTOP_INIT; 919 920 return otx2_sync_mbox_msg(&pfvf->mbox); 921 } 922 923 int otx2_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura) 924 { 925 struct otx2_nic *pfvf = dev; 926 struct otx2_snd_queue *sq; 927 struct nix_aq_enq_req *aq; 928 929 sq = &pfvf->qset.sq[qidx]; 930 sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); 931 /* Get memory to put this msg */ 932 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 933 if (!aq) 934 return -ENOMEM; 935 936 aq->sq.cq = pfvf->hw.rx_queues + qidx; 937 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 938 aq->sq.cq_ena = 1; 939 aq->sq.ena = 1; 940 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); 941 aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 942 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset; 943 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 944 aq->sq.sqb_aura = sqb_aura; 945 aq->sq.sq_int_ena = NIX_SQINT_BITS; 946 aq->sq.qint_idx = 0; 947 /* Due pipelining impact minimum 2000 unused SQ CQE's 948 * need to maintain to avoid CQ overflow. 949 */ 950 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 951 952 /* Fill AQ info */ 953 aq->qidx = qidx; 954 aq->ctype = NIX_AQ_CTYPE_SQ; 955 aq->op = NIX_AQ_INSTOP_INIT; 956 957 return otx2_sync_mbox_msg(&pfvf->mbox); 958 } 959 960 int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) 961 { 962 struct otx2_qset *qset = &pfvf->qset; 963 struct otx2_snd_queue *sq; 964 struct otx2_pool *pool; 965 u8 chan_offset; 966 int err; 967 968 pool = &pfvf->qset.pool[sqb_aura]; 969 sq = &qset->sq[qidx]; 970 sq->sqe_size = NIX_SQESZ_W16 ? 64 : 128; 971 sq->sqe_cnt = qset->sqe_cnt; 972 973 err = qmem_alloc(pfvf->dev, &sq->sqe, 1, sq->sqe_size); 974 if (err) 975 return err; 976 977 /* Allocate memory for NIX SQE (which includes NIX SG) and CPT SG. 978 * SG of NIX and CPT are same in size. Allocate memory for CPT SG 979 * same as NIX SQE for base address alignment. 980 * Layout of a NIX SQE and CPT SG entry: 981 * ----------------------------- 982 * | CPT Scatter Gather | 983 * | (SQE SIZE) | 984 * | | 985 * ----------------------------- 986 * | NIX SQE | 987 * | (SQE SIZE) | 988 * | | 989 * ----------------------------- 990 */ 991 err = qmem_alloc(pfvf->dev, &sq->sqe_ring, qset->sqe_cnt, 992 sq->sqe_size * 2); 993 if (err) 994 return err; 995 996 err = qmem_alloc(pfvf->dev, &sq->cpt_resp, qset->sqe_cnt, 64); 997 if (err) 998 return err; 999 1000 if (qidx < pfvf->hw.tx_queues) { 1001 err = qmem_alloc(pfvf->dev, &sq->tso_hdrs, qset->sqe_cnt, 1002 TSO_HEADER_SIZE); 1003 if (err) 1004 return err; 1005 } 1006 1007 sq->sqe_base = sq->sqe->base; 1008 sq->sg = kcalloc(qset->sqe_cnt, sizeof(struct sg_list), GFP_KERNEL); 1009 if (!sq->sg) 1010 return -ENOMEM; 1011 1012 if (pfvf->ptp && qidx < pfvf->hw.tx_queues) { 1013 err = qmem_alloc(pfvf->dev, &sq->timestamps, qset->sqe_cnt, 1014 sizeof(*sq->timestamps)); 1015 if (err) { 1016 kfree(sq->sg); 1017 sq->sg = NULL; 1018 return err; 1019 } 1020 } 1021 1022 sq->head = 0; 1023 sq->cons_head = 0; 1024 sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; 1025 sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; 1026 /* Set SQE threshold to 10% of total SQEs */ 1027 sq->sqe_thresh = ((sq->num_sqbs * sq->sqe_per_sqb) * 10) / 100; 1028 sq->aura_id = sqb_aura; 1029 sq->aura_fc_addr = pool->fc_addr->base; 1030 sq->io_addr = (__force u64)otx2_get_regaddr(pfvf, NIX_LF_OP_SENDX(0)); 1031 1032 sq->stats.bytes = 0; 1033 sq->stats.pkts = 0; 1034 /* Attach XSK_BUFF_POOL to XDP queue */ 1035 if (qidx > pfvf->hw.xdp_queues) 1036 otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues)); 1037 1038 1039 chan_offset = qidx % pfvf->hw.tx_chan_cnt; 1040 err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura); 1041 if (err) { 1042 kfree(sq->sg); 1043 sq->sg = NULL; 1044 return err; 1045 } 1046 1047 return 0; 1048 1049 } 1050 1051 int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) 1052 { 1053 struct otx2_qset *qset = &pfvf->qset; 1054 int err, pool_id, non_xdp_queues; 1055 struct nix_aq_enq_req *aq; 1056 struct otx2_cq_queue *cq; 1057 struct otx2_pool *pool; 1058 1059 cq = &qset->cq[qidx]; 1060 cq->cq_idx = qidx; 1061 non_xdp_queues = pfvf->hw.rx_queues + pfvf->hw.tx_queues; 1062 if (qidx < pfvf->hw.rx_queues) { 1063 cq->cq_type = CQ_RX; 1064 cq->cint_idx = qidx; 1065 cq->cqe_cnt = qset->rqe_cnt; 1066 if (pfvf->xdp_prog) { 1067 xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0); 1068 pool = &qset->pool[qidx]; 1069 if (pool->xsk_pool) { 1070 xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, 1071 MEM_TYPE_XSK_BUFF_POOL, 1072 NULL); 1073 xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq); 1074 } else if (pool->page_pool) { 1075 xdp_rxq_info_reg_mem_model(&cq->xdp_rxq, 1076 MEM_TYPE_PAGE_POOL, 1077 pool->page_pool); 1078 } 1079 } 1080 } else if (qidx < non_xdp_queues) { 1081 cq->cq_type = CQ_TX; 1082 cq->cint_idx = qidx - pfvf->hw.rx_queues; 1083 cq->cqe_cnt = qset->sqe_cnt; 1084 } else { 1085 if (pfvf->hw.xdp_queues && 1086 qidx < non_xdp_queues + pfvf->hw.xdp_queues) { 1087 cq->cq_type = CQ_XDP; 1088 cq->cint_idx = qidx - non_xdp_queues; 1089 cq->cqe_cnt = qset->sqe_cnt; 1090 } else { 1091 cq->cq_type = CQ_QOS; 1092 cq->cint_idx = qidx - non_xdp_queues - 1093 pfvf->hw.xdp_queues; 1094 cq->cqe_cnt = qset->sqe_cnt; 1095 } 1096 } 1097 cq->cqe_size = pfvf->qset.xqe_size; 1098 1099 /* Allocate memory for CQEs */ 1100 err = qmem_alloc(pfvf->dev, &cq->cqe, cq->cqe_cnt, cq->cqe_size); 1101 if (err) 1102 return err; 1103 1104 /* Save CQE CPU base for faster reference */ 1105 cq->cqe_base = cq->cqe->base; 1106 /* In case where all RQs auras point to single pool, 1107 * all CQs receive buffer pool also point to same pool. 1108 */ 1109 pool_id = ((cq->cq_type == CQ_RX) && 1110 (pfvf->hw.rqpool_cnt != pfvf->hw.rx_queues)) ? 0 : qidx; 1111 cq->rbpool = &qset->pool[pool_id]; 1112 cq->refill_task_sched = false; 1113 1114 /* Get memory to put this msg */ 1115 aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 1116 if (!aq) 1117 return -ENOMEM; 1118 1119 aq->cq.ena = 1; 1120 aq->cq.qsize = Q_SIZE(cq->cqe_cnt, 4); 1121 aq->cq.caching = 1; 1122 aq->cq.base = cq->cqe->iova; 1123 aq->cq.cint_idx = cq->cint_idx; 1124 aq->cq.cq_err_int_ena = NIX_CQERRINT_BITS; 1125 aq->cq.qint_idx = 0; 1126 aq->cq.avg_level = 255; 1127 1128 if (qidx < pfvf->hw.rx_queues) { 1129 aq->cq.drop = RQ_DROP_LVL_CQ(pfvf->hw.rq_skid, cq->cqe_cnt); 1130 aq->cq.drop_ena = 1; 1131 1132 if (!is_otx2_lbkvf(pfvf->pdev)) { 1133 /* Enable receive CQ backpressure */ 1134 aq->cq.bp_ena = 1; 1135 #ifdef CONFIG_DCB 1136 aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]]; 1137 #else 1138 aq->cq.bpid = pfvf->bpid[0]; 1139 #endif 1140 1141 /* Set backpressure level is same as cq pass level */ 1142 aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); 1143 } 1144 } 1145 1146 /* Fill AQ info */ 1147 aq->qidx = qidx; 1148 aq->ctype = NIX_AQ_CTYPE_CQ; 1149 aq->op = NIX_AQ_INSTOP_INIT; 1150 1151 return otx2_sync_mbox_msg(&pfvf->mbox); 1152 } 1153 1154 static void otx2_pool_refill_task(struct work_struct *work) 1155 { 1156 struct otx2_cq_queue *cq; 1157 struct refill_work *wrk; 1158 struct otx2_nic *pfvf; 1159 int qidx; 1160 1161 wrk = container_of(work, struct refill_work, pool_refill_work.work); 1162 pfvf = wrk->pf; 1163 qidx = wrk - pfvf->refill_wrk; 1164 cq = &pfvf->qset.cq[qidx]; 1165 1166 cq->refill_task_sched = false; 1167 1168 local_bh_disable(); 1169 napi_schedule(wrk->napi); 1170 local_bh_enable(); 1171 } 1172 1173 int otx2_config_nix_queues(struct otx2_nic *pfvf) 1174 { 1175 int qidx, err; 1176 1177 /* Initialize RX queues */ 1178 for (qidx = 0; qidx < pfvf->hw.rx_queues; qidx++) { 1179 u16 lpb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx); 1180 1181 err = otx2_rq_init(pfvf, qidx, lpb_aura); 1182 if (err) 1183 return err; 1184 } 1185 1186 /* Initialize TX queues */ 1187 for (qidx = 0; qidx < pfvf->hw.non_qos_queues; qidx++) { 1188 u16 sqb_aura = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 1189 1190 err = otx2_sq_init(pfvf, qidx, sqb_aura); 1191 if (err) 1192 return err; 1193 } 1194 1195 /* Initialize completion queues */ 1196 for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { 1197 err = otx2_cq_init(pfvf, qidx); 1198 if (err) 1199 return err; 1200 } 1201 1202 pfvf->cq_op_addr = (__force u64 *)otx2_get_regaddr(pfvf, 1203 NIX_LF_CQ_OP_STATUS); 1204 1205 /* Initialize work queue for receive buffer refill */ 1206 pfvf->refill_wrk = devm_kcalloc(pfvf->dev, pfvf->qset.cq_cnt, 1207 sizeof(struct refill_work), GFP_KERNEL); 1208 if (!pfvf->refill_wrk) 1209 return -ENOMEM; 1210 1211 for (qidx = 0; qidx < pfvf->qset.cq_cnt; qidx++) { 1212 pfvf->refill_wrk[qidx].pf = pfvf; 1213 INIT_DELAYED_WORK(&pfvf->refill_wrk[qidx].pool_refill_work, 1214 otx2_pool_refill_task); 1215 } 1216 return 0; 1217 } 1218 1219 int otx2_config_nix(struct otx2_nic *pfvf) 1220 { 1221 struct nix_lf_alloc_req *nixlf; 1222 struct nix_lf_alloc_rsp *rsp; 1223 int err; 1224 1225 pfvf->qset.xqe_size = pfvf->hw.xqe_size; 1226 1227 /* Get memory to put this msg */ 1228 nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(&pfvf->mbox); 1229 if (!nixlf) 1230 return -ENOMEM; 1231 1232 /* Set RQ/SQ/CQ counts */ 1233 nixlf->rq_cnt = pfvf->hw.rx_queues; 1234 nixlf->sq_cnt = otx2_get_total_tx_queues(pfvf); 1235 nixlf->cq_cnt = pfvf->qset.cq_cnt; 1236 nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE; 1237 nixlf->rss_grps = MAX_RSS_GROUPS; 1238 nixlf->xqe_sz = pfvf->hw.xqe_size == 128 ? NIX_XQESZ_W16 : NIX_XQESZ_W64; 1239 /* We don't know absolute NPA LF idx attached. 1240 * AF will replace 'RVU_DEFAULT_PF_FUNC' with 1241 * NPA LF attached to this RVU PF/VF. 1242 */ 1243 nixlf->npa_func = RVU_DEFAULT_PF_FUNC; 1244 /* Disable alignment pad, enable L2 length check, 1245 * enable L4 TCP/UDP checksum verification. 1246 */ 1247 nixlf->rx_cfg = BIT_ULL(33) | BIT_ULL(35) | BIT_ULL(37); 1248 1249 err = otx2_sync_mbox_msg(&pfvf->mbox); 1250 if (err) 1251 return err; 1252 1253 rsp = (struct nix_lf_alloc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, 1254 &nixlf->hdr); 1255 if (IS_ERR(rsp)) 1256 return PTR_ERR(rsp); 1257 1258 if (rsp->qints < 1) 1259 return -ENXIO; 1260 1261 return rsp->hdr.rc; 1262 } 1263 1264 void otx2_sq_free_sqbs(struct otx2_nic *pfvf) 1265 { 1266 struct otx2_qset *qset = &pfvf->qset; 1267 struct otx2_hw *hw = &pfvf->hw; 1268 struct otx2_snd_queue *sq; 1269 int sqb, qidx; 1270 u64 iova, pa; 1271 1272 for (qidx = 0; qidx < otx2_get_total_tx_queues(pfvf); qidx++) { 1273 sq = &qset->sq[qidx]; 1274 if (!sq->sqb_ptrs) 1275 continue; 1276 for (sqb = 0; sqb < sq->sqb_count; sqb++) { 1277 if (!sq->sqb_ptrs[sqb]) 1278 continue; 1279 iova = sq->sqb_ptrs[sqb]; 1280 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 1281 dma_unmap_page_attrs(pfvf->dev, iova, hw->sqb_size, 1282 DMA_FROM_DEVICE, 1283 DMA_ATTR_SKIP_CPU_SYNC); 1284 put_page(virt_to_page(phys_to_virt(pa))); 1285 } 1286 sq->sqb_count = 0; 1287 } 1288 } 1289 1290 void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool, 1291 u64 iova, int size) 1292 { 1293 struct page *page; 1294 u64 pa; 1295 1296 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 1297 page = virt_to_head_page(phys_to_virt(pa)); 1298 if (pool->page_pool) { 1299 page_pool_put_full_page(pool->page_pool, page, true); 1300 } else if (pool->xsk_pool) { 1301 /* Note: No way of identifying xdp_buff */ 1302 } else { 1303 dma_unmap_page_attrs(pfvf->dev, iova, size, 1304 DMA_FROM_DEVICE, 1305 DMA_ATTR_SKIP_CPU_SYNC); 1306 1307 put_page(page); 1308 } 1309 } 1310 1311 void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type) 1312 { 1313 int pool_id, pool_start = 0, pool_end = 0, size = 0; 1314 struct otx2_pool *pool; 1315 u64 iova; 1316 int idx; 1317 1318 if (type == AURA_NIX_SQ) { 1319 pool_start = otx2_get_pool_idx(pfvf, type, 0); 1320 pool_end = pool_start + pfvf->hw.sqpool_cnt; 1321 size = pfvf->hw.sqb_size; 1322 } 1323 if (type == AURA_NIX_RQ) { 1324 pool_start = otx2_get_pool_idx(pfvf, type, 0); 1325 pool_end = pfvf->hw.rqpool_cnt; 1326 size = pfvf->rbsize; 1327 } 1328 1329 /* Free SQB and RQB pointers from the aura pool */ 1330 for (pool_id = pool_start; pool_id < pool_end; pool_id++) { 1331 pool = &pfvf->qset.pool[pool_id]; 1332 iova = otx2_aura_allocptr(pfvf, pool_id); 1333 while (iova) { 1334 if (type == AURA_NIX_RQ) 1335 iova -= OTX2_HEAD_ROOM; 1336 otx2_free_bufs(pfvf, pool, iova, size); 1337 iova = otx2_aura_allocptr(pfvf, pool_id); 1338 } 1339 1340 for (idx = 0 ; idx < pool->xdp_cnt; idx++) { 1341 if (!pool->xdp[idx]) 1342 continue; 1343 1344 xsk_buff_free(pool->xdp[idx]); 1345 } 1346 } 1347 } 1348 1349 void otx2_aura_pool_free(struct otx2_nic *pfvf) 1350 { 1351 struct otx2_pool *pool; 1352 int pool_id; 1353 1354 if (!pfvf->qset.pool) 1355 return; 1356 1357 for (pool_id = 0; pool_id < pfvf->hw.pool_cnt; pool_id++) { 1358 pool = &pfvf->qset.pool[pool_id]; 1359 qmem_free(pfvf->dev, pool->stack); 1360 qmem_free(pfvf->dev, pool->fc_addr); 1361 page_pool_destroy(pool->page_pool); 1362 devm_kfree(pfvf->dev, pool->xdp); 1363 pool->xsk_pool = NULL; 1364 } 1365 devm_kfree(pfvf->dev, pfvf->qset.pool); 1366 pfvf->qset.pool = NULL; 1367 } 1368 1369 int otx2_aura_init(struct otx2_nic *pfvf, int aura_id, 1370 int pool_id, int numptrs) 1371 { 1372 struct npa_aq_enq_req *aq; 1373 struct otx2_pool *pool; 1374 int err; 1375 1376 pool = &pfvf->qset.pool[pool_id]; 1377 1378 /* Allocate memory for HW to update Aura count. 1379 * Alloc one cache line, so that it fits all FC_STYPE modes. 1380 */ 1381 if (!pool->fc_addr) { 1382 err = qmem_alloc(pfvf->dev, &pool->fc_addr, 1, OTX2_ALIGN); 1383 if (err) 1384 return err; 1385 } 1386 1387 /* Initialize this aura's context via AF */ 1388 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1389 if (!aq) { 1390 /* Shared mbox memory buffer is full, flush it and retry */ 1391 err = otx2_sync_mbox_msg(&pfvf->mbox); 1392 if (err) 1393 return err; 1394 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1395 if (!aq) 1396 return -ENOMEM; 1397 } 1398 1399 aq->aura_id = aura_id; 1400 /* Will be filled by AF with correct pool context address */ 1401 aq->aura.pool_addr = pool_id; 1402 aq->aura.pool_caching = 1; 1403 aq->aura.shift = ilog2(numptrs) - 8; 1404 aq->aura.count = numptrs; 1405 aq->aura.limit = numptrs; 1406 aq->aura.avg_level = 255; 1407 aq->aura.ena = 1; 1408 aq->aura.fc_ena = 1; 1409 aq->aura.fc_addr = pool->fc_addr->iova; 1410 aq->aura.fc_hyst_bits = 0; /* Store count on all updates */ 1411 1412 /* Enable backpressure for RQ aura */ 1413 if (aura_id < pfvf->hw.rqpool_cnt && !is_otx2_lbkvf(pfvf->pdev)) { 1414 aq->aura.bp_ena = 0; 1415 /* If NIX1 LF is attached then specify NIX1_RX. 1416 * 1417 * Below NPA_AURA_S[BP_ENA] is set according to the 1418 * NPA_BPINTF_E enumeration given as: 1419 * 0x0 + a*0x1 where 'a' is 0 for NIX0_RX and 1 for NIX1_RX so 1420 * NIX0_RX is 0x0 + 0*0x1 = 0 1421 * NIX1_RX is 0x0 + 1*0x1 = 1 1422 * But in HRM it is given that 1423 * "NPA_AURA_S[BP_ENA](w1[33:32]) - Enable aura backpressure to 1424 * NIX-RX based on [BP] level. One bit per NIX-RX; index 1425 * enumerated by NPA_BPINTF_E." 1426 */ 1427 if (pfvf->nix_blkaddr == BLKADDR_NIX1) 1428 aq->aura.bp_ena = 1; 1429 #ifdef CONFIG_DCB 1430 aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]]; 1431 #else 1432 aq->aura.nix0_bpid = pfvf->bpid[0]; 1433 #endif 1434 1435 /* Set backpressure level for RQ's Aura */ 1436 aq->aura.bp = RQ_BP_LVL_AURA; 1437 } 1438 1439 /* Fill AQ info */ 1440 aq->ctype = NPA_AQ_CTYPE_AURA; 1441 aq->op = NPA_AQ_INSTOP_INIT; 1442 1443 return 0; 1444 } 1445 1446 int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, 1447 int stack_pages, int numptrs, int buf_size, int type) 1448 { 1449 struct page_pool_params pp_params = { 0 }; 1450 struct xsk_buff_pool *xsk_pool; 1451 struct npa_aq_enq_req *aq; 1452 struct otx2_pool *pool; 1453 int err; 1454 1455 pool = &pfvf->qset.pool[pool_id]; 1456 /* Alloc memory for stack which is used to store buffer pointers */ 1457 err = qmem_alloc(pfvf->dev, &pool->stack, 1458 stack_pages, pfvf->hw.stack_pg_bytes); 1459 if (err) 1460 return err; 1461 1462 pool->rbsize = buf_size; 1463 1464 /* Initialize this pool's context via AF */ 1465 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1466 if (!aq) { 1467 /* Shared mbox memory buffer is full, flush it and retry */ 1468 err = otx2_sync_mbox_msg(&pfvf->mbox); 1469 if (err) { 1470 qmem_free(pfvf->dev, pool->stack); 1471 return err; 1472 } 1473 aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 1474 if (!aq) { 1475 qmem_free(pfvf->dev, pool->stack); 1476 return -ENOMEM; 1477 } 1478 } 1479 1480 aq->aura_id = pool_id; 1481 aq->pool.stack_base = pool->stack->iova; 1482 aq->pool.stack_caching = 1; 1483 aq->pool.ena = 1; 1484 aq->pool.buf_size = buf_size / 128; 1485 aq->pool.stack_max_pages = stack_pages; 1486 aq->pool.shift = ilog2(numptrs) - 8; 1487 aq->pool.ptr_start = 0; 1488 aq->pool.ptr_end = ~0ULL; 1489 1490 /* Fill AQ info */ 1491 aq->ctype = NPA_AQ_CTYPE_POOL; 1492 aq->op = NPA_AQ_INSTOP_INIT; 1493 1494 if (type != AURA_NIX_RQ) 1495 return 0; 1496 1497 if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { 1498 pp_params.order = get_order(buf_size); 1499 pp_params.flags = PP_FLAG_DMA_MAP; 1500 pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); 1501 pp_params.nid = NUMA_NO_NODE; 1502 pp_params.dev = pfvf->dev; 1503 pp_params.dma_dir = DMA_FROM_DEVICE; 1504 pool->page_pool = page_pool_create(&pp_params); 1505 if (IS_ERR(pool->page_pool)) { 1506 netdev_err(pfvf->netdev, "Creation of page pool failed\n"); 1507 return PTR_ERR(pool->page_pool); 1508 } 1509 return 0; 1510 } 1511 1512 /* Set XSK pool to support AF_XDP zero-copy */ 1513 xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id); 1514 if (xsk_pool) { 1515 pool->xsk_pool = xsk_pool; 1516 pool->xdp_cnt = numptrs; 1517 pool->xdp = devm_kcalloc(pfvf->dev, 1518 numptrs, sizeof(struct xdp_buff *), GFP_KERNEL); 1519 if (IS_ERR(pool->xdp)) { 1520 netdev_err(pfvf->netdev, "Creation of xsk pool failed\n"); 1521 return PTR_ERR(pool->xdp); 1522 } 1523 } 1524 1525 return 0; 1526 } 1527 1528 int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) 1529 { 1530 int qidx, pool_id, stack_pages, num_sqbs; 1531 struct otx2_qset *qset = &pfvf->qset; 1532 struct otx2_hw *hw = &pfvf->hw; 1533 struct otx2_snd_queue *sq; 1534 struct otx2_pool *pool; 1535 dma_addr_t bufptr; 1536 int err, ptr; 1537 1538 /* Calculate number of SQBs needed. 1539 * 1540 * For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. 1541 * Last SQE is used for pointing to next SQB. 1542 */ 1543 num_sqbs = (hw->sqb_size / 128) - 1; 1544 num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; 1545 1546 /* Get no of stack pages needed */ 1547 stack_pages = 1548 (num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; 1549 1550 for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { 1551 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 1552 /* Initialize aura context */ 1553 err = otx2_aura_init(pfvf, pool_id, pool_id, num_sqbs); 1554 if (err) 1555 goto fail; 1556 1557 /* Initialize pool context */ 1558 err = otx2_pool_init(pfvf, pool_id, stack_pages, 1559 num_sqbs, hw->sqb_size, AURA_NIX_SQ); 1560 if (err) 1561 goto fail; 1562 } 1563 1564 /* Flush accumulated messages */ 1565 err = otx2_sync_mbox_msg(&pfvf->mbox); 1566 if (err) 1567 goto fail; 1568 1569 /* Allocate pointers and free them to aura/pool */ 1570 for (qidx = 0; qidx < hw->non_qos_queues; qidx++) { 1571 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 1572 pool = &pfvf->qset.pool[pool_id]; 1573 1574 sq = &qset->sq[qidx]; 1575 sq->sqb_count = 0; 1576 sq->sqb_ptrs = kcalloc(num_sqbs, sizeof(*sq->sqb_ptrs), GFP_KERNEL); 1577 if (!sq->sqb_ptrs) { 1578 err = -ENOMEM; 1579 goto err_mem; 1580 } 1581 1582 for (ptr = 0; ptr < num_sqbs; ptr++) { 1583 err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); 1584 if (err) { 1585 if (pool->xsk_pool) { 1586 ptr--; 1587 while (ptr >= 0) { 1588 xsk_buff_free(pool->xdp[ptr]); 1589 ptr--; 1590 } 1591 } 1592 goto err_mem; 1593 } 1594 1595 pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); 1596 sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; 1597 } 1598 } 1599 1600 err_mem: 1601 return err ? -ENOMEM : 0; 1602 1603 fail: 1604 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 1605 otx2_aura_pool_free(pfvf); 1606 return err; 1607 } 1608 1609 int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) 1610 { 1611 struct otx2_hw *hw = &pfvf->hw; 1612 int stack_pages, pool_id, rq; 1613 struct otx2_pool *pool; 1614 int err, ptr, num_ptrs; 1615 dma_addr_t bufptr; 1616 1617 num_ptrs = pfvf->qset.rqe_cnt; 1618 1619 stack_pages = 1620 (num_ptrs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; 1621 1622 for (rq = 0; rq < hw->rx_queues; rq++) { 1623 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, rq); 1624 /* Initialize aura context */ 1625 err = otx2_aura_init(pfvf, pool_id, pool_id, num_ptrs); 1626 if (err) 1627 goto fail; 1628 } 1629 for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { 1630 err = otx2_pool_init(pfvf, pool_id, stack_pages, 1631 num_ptrs, pfvf->rbsize, AURA_NIX_RQ); 1632 if (err) 1633 goto fail; 1634 } 1635 1636 /* Flush accumulated messages */ 1637 err = otx2_sync_mbox_msg(&pfvf->mbox); 1638 if (err) 1639 goto fail; 1640 1641 /* Allocate pointers and free them to aura/pool */ 1642 for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) { 1643 pool = &pfvf->qset.pool[pool_id]; 1644 1645 for (ptr = 0; ptr < num_ptrs; ptr++) { 1646 err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr); 1647 if (err) { 1648 if (pool->xsk_pool) { 1649 while (ptr) 1650 xsk_buff_free(pool->xdp[--ptr]); 1651 } 1652 return -ENOMEM; 1653 } 1654 1655 pfvf->hw_ops->aura_freeptr(pfvf, pool_id, 1656 pool->xsk_pool ? bufptr : 1657 bufptr + OTX2_HEAD_ROOM); 1658 } 1659 } 1660 return 0; 1661 fail: 1662 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 1663 otx2_aura_pool_free(pfvf); 1664 return err; 1665 } 1666 1667 int otx2_config_npa(struct otx2_nic *pfvf) 1668 { 1669 struct otx2_qset *qset = &pfvf->qset; 1670 struct npa_lf_alloc_req *npalf; 1671 struct otx2_hw *hw = &pfvf->hw; 1672 int aura_cnt; 1673 1674 /* Pool - Stack of free buffer pointers 1675 * Aura - Alloc/frees pointers from/to pool for NIX DMA. 1676 */ 1677 1678 if (!hw->pool_cnt) 1679 return -EINVAL; 1680 1681 qset->pool = devm_kcalloc(pfvf->dev, hw->pool_cnt, 1682 sizeof(struct otx2_pool), GFP_KERNEL); 1683 if (!qset->pool) 1684 return -ENOMEM; 1685 1686 /* Get memory to put this msg */ 1687 npalf = otx2_mbox_alloc_msg_npa_lf_alloc(&pfvf->mbox); 1688 if (!npalf) 1689 return -ENOMEM; 1690 1691 /* Set aura and pool counts */ 1692 npalf->nr_pools = hw->pool_cnt; 1693 aura_cnt = ilog2(roundup_pow_of_two(hw->pool_cnt)); 1694 npalf->aura_sz = (aura_cnt >= ilog2(128)) ? (aura_cnt - 6) : 1; 1695 1696 return otx2_sync_mbox_msg(&pfvf->mbox); 1697 } 1698 1699 int otx2_detach_resources(struct mbox *mbox) 1700 { 1701 struct rsrc_detach *detach; 1702 1703 mutex_lock(&mbox->lock); 1704 detach = otx2_mbox_alloc_msg_detach_resources(mbox); 1705 if (!detach) { 1706 mutex_unlock(&mbox->lock); 1707 return -ENOMEM; 1708 } 1709 1710 /* detach all */ 1711 detach->partial = false; 1712 1713 /* Send detach request to AF */ 1714 otx2_sync_mbox_msg(mbox); 1715 mutex_unlock(&mbox->lock); 1716 return 0; 1717 } 1718 EXPORT_SYMBOL(otx2_detach_resources); 1719 1720 int otx2_attach_npa_nix(struct otx2_nic *pfvf) 1721 { 1722 struct rsrc_attach *attach; 1723 struct msg_req *msix; 1724 int err; 1725 1726 mutex_lock(&pfvf->mbox.lock); 1727 /* Get memory to put this msg */ 1728 attach = otx2_mbox_alloc_msg_attach_resources(&pfvf->mbox); 1729 if (!attach) { 1730 mutex_unlock(&pfvf->mbox.lock); 1731 return -ENOMEM; 1732 } 1733 1734 attach->npalf = true; 1735 attach->nixlf = true; 1736 1737 /* Send attach request to AF */ 1738 err = otx2_sync_mbox_msg(&pfvf->mbox); 1739 if (err) { 1740 mutex_unlock(&pfvf->mbox.lock); 1741 return err; 1742 } 1743 1744 pfvf->nix_blkaddr = BLKADDR_NIX0; 1745 1746 /* If the platform has two NIX blocks then LF may be 1747 * allocated from NIX1. 1748 */ 1749 if (otx2_read64(pfvf, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_NIX1)) & 0x1FFULL) 1750 pfvf->nix_blkaddr = BLKADDR_NIX1; 1751 1752 /* Get NPA and NIX MSIX vector offsets */ 1753 msix = otx2_mbox_alloc_msg_msix_offset(&pfvf->mbox); 1754 if (!msix) { 1755 mutex_unlock(&pfvf->mbox.lock); 1756 return -ENOMEM; 1757 } 1758 1759 err = otx2_sync_mbox_msg(&pfvf->mbox); 1760 if (err) { 1761 mutex_unlock(&pfvf->mbox.lock); 1762 return err; 1763 } 1764 mutex_unlock(&pfvf->mbox.lock); 1765 1766 if (pfvf->hw.npa_msixoff == MSIX_VECTOR_INVALID || 1767 pfvf->hw.nix_msixoff == MSIX_VECTOR_INVALID) { 1768 dev_err(pfvf->dev, 1769 "RVUPF: Invalid MSIX vector offset for NPA/NIX\n"); 1770 return -EINVAL; 1771 } 1772 1773 return 0; 1774 } 1775 EXPORT_SYMBOL(otx2_attach_npa_nix); 1776 1777 void otx2_ctx_disable(struct mbox *mbox, int type, bool npa) 1778 { 1779 struct hwctx_disable_req *req; 1780 1781 mutex_lock(&mbox->lock); 1782 /* Request AQ to disable this context */ 1783 if (npa) 1784 req = otx2_mbox_alloc_msg_npa_hwctx_disable(mbox); 1785 else 1786 req = otx2_mbox_alloc_msg_nix_hwctx_disable(mbox); 1787 1788 if (!req) { 1789 mutex_unlock(&mbox->lock); 1790 return; 1791 } 1792 1793 req->ctype = type; 1794 1795 if (otx2_sync_mbox_msg(mbox)) 1796 dev_err(mbox->pfvf->dev, "%s failed to disable context\n", 1797 __func__); 1798 1799 mutex_unlock(&mbox->lock); 1800 } 1801 1802 int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable) 1803 { 1804 struct nix_bp_cfg_req *req; 1805 1806 if (enable) 1807 req = otx2_mbox_alloc_msg_nix_bp_enable(&pfvf->mbox); 1808 else 1809 req = otx2_mbox_alloc_msg_nix_bp_disable(&pfvf->mbox); 1810 1811 if (!req) 1812 return -ENOMEM; 1813 1814 req->chan_base = 0; 1815 if (otx2_is_pfc_enabled(pfvf)) { 1816 req->chan_cnt = IEEE_8021QAZ_MAX_TCS; 1817 req->bpid_per_chan = 1; 1818 } else { 1819 req->chan_cnt = pfvf->hw.rx_chan_cnt; 1820 req->bpid_per_chan = 0; 1821 } 1822 1823 return otx2_sync_mbox_msg(&pfvf->mbox); 1824 } 1825 EXPORT_SYMBOL(otx2_nix_config_bp); 1826 1827 int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable) 1828 { 1829 struct nix_bp_cfg_req *req; 1830 1831 if (enable) 1832 req = otx2_mbox_alloc_msg_nix_cpt_bp_enable(&pfvf->mbox); 1833 else 1834 req = otx2_mbox_alloc_msg_nix_cpt_bp_disable(&pfvf->mbox); 1835 1836 if (!req) 1837 return -ENOMEM; 1838 1839 req->chan_base = 0; 1840 if (otx2_is_pfc_enabled(pfvf)) { 1841 req->chan_cnt = IEEE_8021QAZ_MAX_TCS; 1842 req->bpid_per_chan = 1; 1843 } else { 1844 req->chan_cnt = pfvf->hw.rx_chan_cnt; 1845 req->bpid_per_chan = 0; 1846 } 1847 1848 return otx2_sync_mbox_msg(&pfvf->mbox); 1849 } 1850 EXPORT_SYMBOL(otx2_nix_cpt_config_bp); 1851 1852 /* Mbox message handlers */ 1853 void mbox_handler_cgx_stats(struct otx2_nic *pfvf, 1854 struct cgx_stats_rsp *rsp) 1855 { 1856 int id; 1857 1858 for (id = 0; id < CGX_RX_STATS_COUNT; id++) 1859 pfvf->hw.cgx_rx_stats[id] = rsp->rx_stats[id]; 1860 for (id = 0; id < CGX_TX_STATS_COUNT; id++) 1861 pfvf->hw.cgx_tx_stats[id] = rsp->tx_stats[id]; 1862 } 1863 1864 void mbox_handler_cgx_fec_stats(struct otx2_nic *pfvf, 1865 struct cgx_fec_stats_rsp *rsp) 1866 { 1867 pfvf->hw.cgx_fec_corr_blks += rsp->fec_corr_blks; 1868 pfvf->hw.cgx_fec_uncorr_blks += rsp->fec_uncorr_blks; 1869 } 1870 1871 void mbox_handler_npa_lf_alloc(struct otx2_nic *pfvf, 1872 struct npa_lf_alloc_rsp *rsp) 1873 { 1874 pfvf->hw.stack_pg_ptrs = rsp->stack_pg_ptrs; 1875 pfvf->hw.stack_pg_bytes = rsp->stack_pg_bytes; 1876 } 1877 EXPORT_SYMBOL(mbox_handler_npa_lf_alloc); 1878 1879 void mbox_handler_nix_lf_alloc(struct otx2_nic *pfvf, 1880 struct nix_lf_alloc_rsp *rsp) 1881 { 1882 pfvf->hw.sqb_size = rsp->sqb_size; 1883 pfvf->hw.rx_chan_base = rsp->rx_chan_base; 1884 pfvf->hw.tx_chan_base = rsp->tx_chan_base; 1885 pfvf->hw.rx_chan_cnt = rsp->rx_chan_cnt; 1886 pfvf->hw.tx_chan_cnt = rsp->tx_chan_cnt; 1887 pfvf->hw.lso_tsov4_idx = rsp->lso_tsov4_idx; 1888 pfvf->hw.lso_tsov6_idx = rsp->lso_tsov6_idx; 1889 pfvf->hw.cgx_links = rsp->cgx_links; 1890 pfvf->hw.lbk_links = rsp->lbk_links; 1891 pfvf->hw.tx_link = rsp->tx_link; 1892 } 1893 EXPORT_SYMBOL(mbox_handler_nix_lf_alloc); 1894 1895 void mbox_handler_msix_offset(struct otx2_nic *pfvf, 1896 struct msix_offset_rsp *rsp) 1897 { 1898 pfvf->hw.npa_msixoff = rsp->npa_msixoff; 1899 pfvf->hw.nix_msixoff = rsp->nix_msixoff; 1900 } 1901 EXPORT_SYMBOL(mbox_handler_msix_offset); 1902 1903 void mbox_handler_nix_bp_enable(struct otx2_nic *pfvf, 1904 struct nix_bp_cfg_rsp *rsp) 1905 { 1906 int chan, chan_id; 1907 1908 for (chan = 0; chan < rsp->chan_cnt; chan++) { 1909 chan_id = ((rsp->chan_bpid[chan] >> 10) & 0x7F); 1910 pfvf->bpid[chan_id] = rsp->chan_bpid[chan] & 0x3FF; 1911 } 1912 } 1913 EXPORT_SYMBOL(mbox_handler_nix_bp_enable); 1914 1915 void otx2_free_cints(struct otx2_nic *pfvf, int n) 1916 { 1917 struct otx2_qset *qset = &pfvf->qset; 1918 struct otx2_hw *hw = &pfvf->hw; 1919 int irq, qidx; 1920 1921 for (qidx = 0, irq = hw->nix_msixoff + NIX_LF_CINT_VEC_START; 1922 qidx < n; 1923 qidx++, irq++) { 1924 int vector = pci_irq_vector(pfvf->pdev, irq); 1925 1926 irq_set_affinity_hint(vector, NULL); 1927 free_cpumask_var(hw->affinity_mask[irq]); 1928 free_irq(vector, &qset->napi[qidx]); 1929 } 1930 } 1931 EXPORT_SYMBOL(otx2_free_cints); 1932 1933 void otx2_set_cints_affinity(struct otx2_nic *pfvf) 1934 { 1935 struct otx2_hw *hw = &pfvf->hw; 1936 int vec, cpu, irq, cint; 1937 1938 vec = hw->nix_msixoff + NIX_LF_CINT_VEC_START; 1939 cpu = cpumask_first(cpu_online_mask); 1940 1941 /* CQ interrupts */ 1942 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++, vec++) { 1943 if (!alloc_cpumask_var(&hw->affinity_mask[vec], GFP_KERNEL)) 1944 return; 1945 1946 cpumask_set_cpu(cpu, hw->affinity_mask[vec]); 1947 1948 irq = pci_irq_vector(pfvf->pdev, vec); 1949 irq_set_affinity_hint(irq, hw->affinity_mask[vec]); 1950 1951 cpu = cpumask_next(cpu, cpu_online_mask); 1952 if (unlikely(cpu >= nr_cpu_ids)) 1953 cpu = 0; 1954 } 1955 } 1956 1957 static u32 get_dwrr_mtu(struct otx2_nic *pfvf, struct nix_hw_info *hw) 1958 { 1959 if (is_otx2_lbkvf(pfvf->pdev)) { 1960 pfvf->hw.smq_link_type = SMQ_LINK_TYPE_LBK; 1961 return hw->lbk_dwrr_mtu; 1962 } 1963 1964 pfvf->hw.smq_link_type = SMQ_LINK_TYPE_RPM; 1965 return hw->rpm_dwrr_mtu; 1966 } 1967 1968 u16 otx2_get_max_mtu(struct otx2_nic *pfvf) 1969 { 1970 struct nix_hw_info *rsp; 1971 struct msg_req *req; 1972 u16 max_mtu; 1973 int rc; 1974 1975 mutex_lock(&pfvf->mbox.lock); 1976 1977 req = otx2_mbox_alloc_msg_nix_get_hw_info(&pfvf->mbox); 1978 if (!req) { 1979 rc = -ENOMEM; 1980 goto out; 1981 } 1982 1983 rc = otx2_sync_mbox_msg(&pfvf->mbox); 1984 if (!rc) { 1985 rsp = (struct nix_hw_info *) 1986 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 1987 if (IS_ERR(rsp)) { 1988 rc = PTR_ERR(rsp); 1989 goto out; 1990 } 1991 1992 /* HW counts VLAN insertion bytes (8 for double tag) 1993 * irrespective of whether SQE is requesting to insert VLAN 1994 * in the packet or not. Hence these 8 bytes have to be 1995 * discounted from max packet size otherwise HW will throw 1996 * SMQ errors 1997 */ 1998 max_mtu = rsp->max_mtu - 8 - OTX2_ETH_HLEN; 1999 2000 /* Also save DWRR MTU, needed for DWRR weight calculation */ 2001 pfvf->hw.dwrr_mtu = get_dwrr_mtu(pfvf, rsp); 2002 if (!pfvf->hw.dwrr_mtu) 2003 pfvf->hw.dwrr_mtu = 1; 2004 } 2005 2006 out: 2007 mutex_unlock(&pfvf->mbox.lock); 2008 if (rc) { 2009 dev_warn(pfvf->dev, 2010 "Failed to get MTU from hardware setting default value(1500)\n"); 2011 max_mtu = 1500; 2012 } 2013 return max_mtu; 2014 } 2015 EXPORT_SYMBOL(otx2_get_max_mtu); 2016 2017 int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features) 2018 { 2019 netdev_features_t changed = features ^ netdev->features; 2020 struct otx2_nic *pfvf = netdev_priv(netdev); 2021 bool ntuple = !!(features & NETIF_F_NTUPLE); 2022 bool tc = !!(features & NETIF_F_HW_TC); 2023 2024 if ((changed & NETIF_F_NTUPLE) && !ntuple) 2025 otx2_destroy_ntuple_flows(pfvf); 2026 2027 if ((changed & NETIF_F_NTUPLE) && ntuple) { 2028 if (!pfvf->flow_cfg->max_flows) { 2029 netdev_err(netdev, 2030 "Can't enable NTUPLE, MCAM entries not allocated\n"); 2031 return -EINVAL; 2032 } 2033 } 2034 2035 if ((changed & NETIF_F_HW_TC) && !tc && 2036 otx2_tc_flower_rule_cnt(pfvf)) { 2037 netdev_err(netdev, "Can't disable TC hardware offload while flows are active\n"); 2038 return -EBUSY; 2039 } 2040 2041 if ((changed & NETIF_F_NTUPLE) && ntuple && 2042 otx2_tc_flower_rule_cnt(pfvf) && !(changed & NETIF_F_HW_TC)) { 2043 netdev_err(netdev, 2044 "Can't enable NTUPLE when TC flower offload is active, disable TC rules and retry\n"); 2045 return -EINVAL; 2046 } 2047 2048 return 0; 2049 } 2050 EXPORT_SYMBOL(otx2_handle_ntuple_tc_features); 2051 2052 int otx2_set_hw_capabilities(struct otx2_nic *pfvf) 2053 { 2054 struct mbox *mbox = &pfvf->mbox; 2055 struct otx2_hw *hw = &pfvf->hw; 2056 struct get_hw_cap_rsp *rsp; 2057 struct msg_req *req; 2058 int ret = -ENOMEM; 2059 2060 mutex_lock(&mbox->lock); 2061 2062 req = otx2_mbox_alloc_msg_get_hw_cap(mbox); 2063 if (!req) 2064 goto fail; 2065 2066 ret = otx2_sync_mbox_msg(mbox); 2067 if (ret) 2068 goto fail; 2069 2070 rsp = (struct get_hw_cap_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, 2071 0, &req->hdr); 2072 if (IS_ERR(rsp)) { 2073 ret = -EINVAL; 2074 goto fail; 2075 } 2076 2077 if (rsp->hw_caps & HW_CAP_MACSEC) 2078 __set_bit(CN10K_HW_MACSEC, &hw->cap_flag); 2079 2080 mutex_unlock(&mbox->lock); 2081 2082 return 0; 2083 fail: 2084 dev_err(pfvf->dev, "Cannot get MACSEC capability from AF\n"); 2085 mutex_unlock(&mbox->lock); 2086 return ret; 2087 } 2088 2089 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ 2090 int __weak \ 2091 otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ 2092 struct _req_type *req, \ 2093 struct _rsp_type *rsp) \ 2094 { \ 2095 /* Nothing to do here */ \ 2096 return 0; \ 2097 } \ 2098 EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); 2099 MBOX_UP_CGX_MESSAGES 2100 MBOX_UP_MCS_MESSAGES 2101 #undef M 2102 2103 dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf, 2104 struct sk_buff *skb, int seg, int *len) 2105 { 2106 enum dma_data_direction dir = DMA_TO_DEVICE; 2107 const skb_frag_t *frag; 2108 struct page *page; 2109 int offset; 2110 2111 /* Crypto hardware need write permission for ipsec crypto offload */ 2112 if (unlikely(xfrm_offload(skb))) { 2113 dir = DMA_BIDIRECTIONAL; 2114 skb = skb_unshare(skb, GFP_ATOMIC); 2115 } 2116 2117 /* First segment is always skb->data */ 2118 if (!seg) { 2119 page = virt_to_page(skb->data); 2120 offset = offset_in_page(skb->data); 2121 *len = skb_headlen(skb); 2122 } else { 2123 frag = &skb_shinfo(skb)->frags[seg - 1]; 2124 page = skb_frag_page(frag); 2125 offset = skb_frag_off(frag); 2126 *len = skb_frag_size(frag); 2127 } 2128 return otx2_dma_map_page(pfvf, page, offset, *len, dir); 2129 } 2130 2131 void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg) 2132 { 2133 enum dma_data_direction dir = DMA_TO_DEVICE; 2134 struct sk_buff *skb = NULL; 2135 int seg; 2136 2137 skb = (struct sk_buff *)sg->skb; 2138 if (unlikely(xfrm_offload(skb))) 2139 dir = DMA_BIDIRECTIONAL; 2140 2141 for (seg = 0; seg < sg->num_segs; seg++) { 2142 otx2_dma_unmap_page(pfvf, sg->dma_addr[seg], 2143 sg->size[seg], dir); 2144 } 2145 sg->num_segs = 0; 2146 } 2147