1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019, Intel Corporation. */ 3 4 #include <net/xdp_sock_drv.h> 5 #include <linux/net/intel/libie/rx.h> 6 #include "ice_base.h" 7 #include "ice_lib.h" 8 #include "ice_dcb_lib.h" 9 #include "ice_sriov.h" 10 11 /** 12 * __ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI 13 * @qs_cfg: gathered variables needed for PF->VSI queues assignment 14 * 15 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 16 */ 17 static int __ice_vsi_get_qs_contig(struct ice_qs_cfg *qs_cfg) 18 { 19 unsigned int offset, i; 20 21 mutex_lock(qs_cfg->qs_mutex); 22 offset = bitmap_find_next_zero_area(qs_cfg->pf_map, qs_cfg->pf_map_size, 23 0, qs_cfg->q_count, 0); 24 if (offset >= qs_cfg->pf_map_size) { 25 mutex_unlock(qs_cfg->qs_mutex); 26 return -ENOMEM; 27 } 28 29 bitmap_set(qs_cfg->pf_map, offset, qs_cfg->q_count); 30 for (i = 0; i < qs_cfg->q_count; i++) 31 qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)(i + offset); 32 mutex_unlock(qs_cfg->qs_mutex); 33 34 return 0; 35 } 36 37 /** 38 * __ice_vsi_get_qs_sc - Assign a scattered queues from PF to VSI 39 * @qs_cfg: gathered variables needed for pf->vsi queues assignment 40 * 41 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 42 */ 43 static int __ice_vsi_get_qs_sc(struct ice_qs_cfg *qs_cfg) 44 { 45 unsigned int i, index = 0; 46 47 mutex_lock(qs_cfg->qs_mutex); 48 for (i = 0; i < qs_cfg->q_count; i++) { 49 index = find_next_zero_bit(qs_cfg->pf_map, 50 qs_cfg->pf_map_size, index); 51 if (index >= qs_cfg->pf_map_size) 52 goto err_scatter; 53 set_bit(index, qs_cfg->pf_map); 54 qs_cfg->vsi_map[i + qs_cfg->vsi_map_offset] = (u16)index; 55 } 56 mutex_unlock(qs_cfg->qs_mutex); 57 58 return 0; 59 err_scatter: 60 for (index = 0; index < i; index++) { 61 clear_bit(qs_cfg->vsi_map[index], qs_cfg->pf_map); 62 qs_cfg->vsi_map[index + qs_cfg->vsi_map_offset] = 0; 63 } 64 mutex_unlock(qs_cfg->qs_mutex); 65 66 return -ENOMEM; 67 } 68 69 /** 70 * ice_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 71 * @pf: the PF being configured 72 * @pf_q: the PF queue 73 * @ena: enable or disable state of the queue 74 * 75 * This routine will wait for the given Rx queue of the PF to reach the 76 * enabled or disabled state. 77 * Returns -ETIMEDOUT in case of failing to reach the requested state after 78 * multiple retries; else will return 0 in case of success. 79 */ 80 static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena) 81 { 82 int i; 83 84 for (i = 0; i < ICE_Q_WAIT_MAX_RETRY; i++) { 85 if (ena == !!(rd32(&pf->hw, QRX_CTRL(pf_q)) & 86 QRX_CTRL_QENA_STAT_M)) 87 return 0; 88 89 usleep_range(20, 40); 90 } 91 92 return -ETIMEDOUT; 93 } 94 95 /** 96 * ice_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 97 * @vsi: the VSI being configured 98 * @v_idx: index of the vector in the VSI struct 99 * 100 * We allocate one q_vector and set default value for ITR setting associated 101 * with this q_vector. If allocation fails we return -ENOMEM. 102 */ 103 static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) 104 { 105 struct ice_pf *pf = vsi->back; 106 struct ice_q_vector *q_vector; 107 int err; 108 109 /* allocate q_vector */ 110 q_vector = kzalloc(sizeof(*q_vector), GFP_KERNEL); 111 if (!q_vector) 112 return -ENOMEM; 113 114 q_vector->vsi = vsi; 115 q_vector->v_idx = v_idx; 116 q_vector->tx.itr_setting = ICE_DFLT_TX_ITR; 117 q_vector->rx.itr_setting = ICE_DFLT_RX_ITR; 118 q_vector->tx.itr_mode = ITR_DYNAMIC; 119 q_vector->rx.itr_mode = ITR_DYNAMIC; 120 q_vector->tx.type = ICE_TX_CONTAINER; 121 q_vector->rx.type = ICE_RX_CONTAINER; 122 q_vector->irq.index = -ENOENT; 123 124 if (vsi->type == ICE_VSI_VF) { 125 ice_calc_vf_reg_idx(vsi->vf, q_vector); 126 goto out; 127 } else if (vsi->type == ICE_VSI_CTRL && vsi->vf) { 128 struct ice_vsi *ctrl_vsi = ice_get_vf_ctrl_vsi(pf, vsi); 129 130 if (ctrl_vsi) { 131 if (unlikely(!ctrl_vsi->q_vectors)) { 132 err = -ENOENT; 133 goto err_free_q_vector; 134 } 135 136 q_vector->irq = ctrl_vsi->q_vectors[0]->irq; 137 goto skip_alloc; 138 } 139 } 140 141 q_vector->irq = ice_alloc_irq(pf, vsi->irq_dyn_alloc); 142 if (q_vector->irq.index < 0) { 143 err = -ENOMEM; 144 goto err_free_q_vector; 145 } 146 147 skip_alloc: 148 q_vector->reg_idx = q_vector->irq.index; 149 q_vector->vf_reg_idx = q_vector->irq.index; 150 151 /* This will not be called in the driver load path because the netdev 152 * will not be created yet. All other cases with register the NAPI 153 * handler here (i.e. resume, reset/rebuild, etc.) 154 */ 155 if (vsi->netdev) 156 netif_napi_add_config(vsi->netdev, &q_vector->napi, 157 ice_napi_poll, v_idx); 158 159 out: 160 /* tie q_vector and VSI together */ 161 vsi->q_vectors[v_idx] = q_vector; 162 163 return 0; 164 165 err_free_q_vector: 166 kfree(q_vector); 167 168 return err; 169 } 170 171 /** 172 * ice_free_q_vector - Free memory allocated for a specific interrupt vector 173 * @vsi: VSI having the memory freed 174 * @v_idx: index of the vector to be freed 175 */ 176 static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx) 177 { 178 struct ice_q_vector *q_vector; 179 struct ice_pf *pf = vsi->back; 180 struct ice_tx_ring *tx_ring; 181 struct ice_rx_ring *rx_ring; 182 struct device *dev; 183 184 dev = ice_pf_to_dev(pf); 185 if (!vsi->q_vectors[v_idx]) { 186 dev_dbg(dev, "Queue vector at index %d not found\n", v_idx); 187 return; 188 } 189 q_vector = vsi->q_vectors[v_idx]; 190 191 ice_for_each_tx_ring(tx_ring, vsi->q_vectors[v_idx]->tx) 192 tx_ring->q_vector = NULL; 193 194 ice_for_each_rx_ring(rx_ring, vsi->q_vectors[v_idx]->rx) 195 rx_ring->q_vector = NULL; 196 197 /* only VSI with an associated netdev is set up with NAPI */ 198 if (vsi->netdev) 199 netif_napi_del(&q_vector->napi); 200 201 /* release MSIX interrupt if q_vector had interrupt allocated */ 202 if (q_vector->irq.index < 0) 203 goto free_q_vector; 204 205 /* only free last VF ctrl vsi interrupt */ 206 if (vsi->type == ICE_VSI_CTRL && vsi->vf && 207 ice_get_vf_ctrl_vsi(pf, vsi)) 208 goto free_q_vector; 209 210 ice_free_irq(pf, q_vector->irq); 211 212 free_q_vector: 213 kfree(q_vector); 214 vsi->q_vectors[v_idx] = NULL; 215 } 216 217 /** 218 * ice_cfg_itr_gran - set the ITR granularity to 2 usecs if not already set 219 * @hw: board specific structure 220 */ 221 static void ice_cfg_itr_gran(struct ice_hw *hw) 222 { 223 u32 regval = rd32(hw, GLINT_CTL); 224 225 /* no need to update global register if ITR gran is already set */ 226 if (!(regval & GLINT_CTL_DIS_AUTOMASK_M) && 227 (FIELD_GET(GLINT_CTL_ITR_GRAN_200_M, regval) == ICE_ITR_GRAN_US) && 228 (FIELD_GET(GLINT_CTL_ITR_GRAN_100_M, regval) == ICE_ITR_GRAN_US) && 229 (FIELD_GET(GLINT_CTL_ITR_GRAN_50_M, regval) == ICE_ITR_GRAN_US) && 230 (FIELD_GET(GLINT_CTL_ITR_GRAN_25_M, regval) == ICE_ITR_GRAN_US)) 231 return; 232 233 regval = FIELD_PREP(GLINT_CTL_ITR_GRAN_200_M, ICE_ITR_GRAN_US) | 234 FIELD_PREP(GLINT_CTL_ITR_GRAN_100_M, ICE_ITR_GRAN_US) | 235 FIELD_PREP(GLINT_CTL_ITR_GRAN_50_M, ICE_ITR_GRAN_US) | 236 FIELD_PREP(GLINT_CTL_ITR_GRAN_25_M, ICE_ITR_GRAN_US); 237 wr32(hw, GLINT_CTL, regval); 238 } 239 240 /** 241 * ice_calc_txq_handle - calculate the queue handle 242 * @vsi: VSI that ring belongs to 243 * @ring: ring to get the absolute queue index 244 * @tc: traffic class number 245 */ 246 static u16 247 ice_calc_txq_handle(const struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) 248 { 249 WARN_ONCE(ice_ring_is_xdp(ring) && tc, "XDP ring can't belong to TC other than 0\n"); 250 251 if (ring->ch) 252 return ring->q_index - ring->ch->base_q; 253 254 /* Idea here for calculation is that we subtract the number of queue 255 * count from TC that ring belongs to from its absolute queue index 256 * and as a result we get the queue's index within TC. 257 */ 258 return ring->q_index - vsi->tc_cfg.tc_info[tc].qoffset; 259 } 260 261 /** 262 * ice_cfg_xps_tx_ring - Configure XPS for a Tx ring 263 * @ring: The Tx ring to configure 264 * 265 * This enables/disables XPS for a given Tx descriptor ring 266 * based on the TCs enabled for the VSI that ring belongs to. 267 */ 268 static void ice_cfg_xps_tx_ring(struct ice_tx_ring *ring) 269 { 270 if (!ring->q_vector || !ring->netdev) 271 return; 272 273 /* We only initialize XPS once, so as not to overwrite user settings */ 274 if (test_and_set_bit(ICE_TX_XPS_INIT_DONE, ring->xps_state)) 275 return; 276 277 netif_set_xps_queue(ring->netdev, 278 &ring->q_vector->napi.config->affinity_mask, 279 ring->q_index); 280 } 281 282 /** 283 * ice_set_txq_ctx_vmvf - set queue context VM/VF type and number by VSI type 284 * @ring: the Tx ring to configure 285 * @vmvf_type: VM/VF type 286 * @vmvf_num: VM/VF number 287 * 288 * Return: 0 on success and a negative value on error. 289 */ 290 static int 291 ice_set_txq_ctx_vmvf(struct ice_tx_ring *ring, u8 *vmvf_type, u16 *vmvf_num) 292 { 293 struct ice_vsi *vsi = ring->vsi; 294 struct ice_hw *hw; 295 296 hw = &vsi->back->hw; 297 298 /* queue belongs to a specific VSI type 299 * VF / VM index should be programmed per vmvf_type setting: 300 * for vmvf_type = VF, it is VF number between 0-256 301 * for vmvf_type = VM, it is VM number between 0-767 302 * for PF or EMP this field should be set to zero 303 */ 304 switch (vsi->type) { 305 case ICE_VSI_LB: 306 case ICE_VSI_CTRL: 307 case ICE_VSI_PF: 308 if (ring->ch) 309 *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; 310 else 311 *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; 312 break; 313 case ICE_VSI_VF: 314 /* Firmware expects vmvf_num to be absolute VF ID */ 315 *vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; 316 *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; 317 break; 318 case ICE_VSI_SF: 319 *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; 320 break; 321 default: 322 dev_info(ice_pf_to_dev(vsi->back), 323 "Unable to set VMVF type for VSI type %d\n", 324 vsi->type); 325 return -EINVAL; 326 } 327 return 0; 328 } 329 330 /** 331 * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance 332 * @ring: the Tx ring to configure 333 * @tlan_ctx: pointer to the Tx LAN queue context structure to be initialized 334 * @pf_q: queue index in the PF space 335 * 336 * Configure the Tx descriptor ring in TLAN context. 337 * 338 * Return: 0 on success and a negative value on error. 339 */ 340 static int 341 ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) 342 { 343 struct ice_vsi *vsi = ring->vsi; 344 struct ice_hw *hw; 345 int err; 346 347 hw = &vsi->back->hw; 348 tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; 349 tlan_ctx->port_num = vsi->port_info->lport; 350 351 /* Transmit Queue Length */ 352 tlan_ctx->qlen = ring->count; 353 354 ice_set_cgd_num(tlan_ctx, ring->dcb_tc); 355 356 /* PF number */ 357 tlan_ctx->pf_num = hw->pf_id; 358 359 err = ice_set_txq_ctx_vmvf(ring, &tlan_ctx->vmvf_type, 360 &tlan_ctx->vmvf_num); 361 if (err) 362 return err; 363 364 /* make sure the context is associated with the right VSI */ 365 if (ring->ch) 366 tlan_ctx->src_vsi = ring->ch->vsi_num; 367 else 368 tlan_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); 369 370 /* Restrict Tx timestamps to the PF VSI */ 371 switch (vsi->type) { 372 case ICE_VSI_PF: 373 tlan_ctx->tsyn_ena = 1; 374 break; 375 default: 376 break; 377 } 378 379 tlan_ctx->quanta_prof_idx = ring->quanta_prof_id; 380 381 tlan_ctx->tso_ena = ICE_TX_LEGACY; 382 tlan_ctx->tso_qnum = pf_q; 383 384 /* Legacy or Advanced Host Interface: 385 * 0: Advanced Host Interface 386 * 1: Legacy Host Interface 387 */ 388 tlan_ctx->legacy_int = ICE_TX_LEGACY; 389 390 return 0; 391 } 392 393 /** 394 * ice_setup_txtime_ctx - setup a struct ice_txtime_ctx instance 395 * @ring: the tstamp ring to configure 396 * @txtime_ctx: pointer to the Tx time queue context structure to be initialized 397 * 398 * Return: 0 on success and a negative value on error. 399 */ 400 static int 401 ice_setup_txtime_ctx(const struct ice_tstamp_ring *ring, 402 struct ice_txtime_ctx *txtime_ctx) 403 { 404 struct ice_tx_ring *tx_ring = ring->tx_ring; 405 struct ice_vsi *vsi = tx_ring->vsi; 406 struct ice_hw *hw = &vsi->back->hw; 407 int err; 408 409 txtime_ctx->base = ring->dma >> ICE_TXTIME_CTX_BASE_S; 410 411 /* Tx time Queue Length */ 412 txtime_ctx->qlen = ring->count; 413 txtime_ctx->txtime_ena_q = 1; 414 415 /* PF number */ 416 txtime_ctx->pf_num = hw->pf_id; 417 418 err = ice_set_txq_ctx_vmvf(tx_ring, &txtime_ctx->vmvf_type, 419 &txtime_ctx->vmvf_num); 420 if (err) 421 return err; 422 423 /* make sure the context is associated with the right VSI */ 424 if (tx_ring->ch) 425 txtime_ctx->src_vsi = tx_ring->ch->vsi_num; 426 else 427 txtime_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); 428 429 txtime_ctx->ts_res = ICE_TXTIME_CTX_RESOLUTION_128NS; 430 txtime_ctx->drbell_mode_32 = ICE_TXTIME_CTX_DRBELL_MODE_32; 431 txtime_ctx->ts_fetch_prof_id = ICE_TXTIME_CTX_FETCH_PROF_ID_0; 432 433 return 0; 434 } 435 436 /** 437 * ice_calc_ts_ring_count - calculate the number of Tx time stamp descriptors 438 * @tx_ring: Tx ring to calculate the count for 439 * 440 * Return: the number of Tx time stamp descriptors. 441 */ 442 u16 ice_calc_ts_ring_count(struct ice_tx_ring *tx_ring) 443 { 444 u16 prof = ICE_TXTIME_CTX_FETCH_PROF_ID_0; 445 struct ice_vsi *vsi = tx_ring->vsi; 446 struct ice_hw *hw = &vsi->back->hw; 447 u16 max_fetch_desc = 0, fetch, i; 448 u32 reg; 449 450 for (i = 0; i < ICE_TXTIME_FETCH_PROFILE_CNT; i++) { 451 reg = rd32(hw, E830_GLTXTIME_FETCH_PROFILE(prof, 0)); 452 fetch = FIELD_GET(E830_GLTXTIME_FETCH_PROFILE_FETCH_TS_DESC_M, 453 reg); 454 max_fetch_desc = max(fetch, max_fetch_desc); 455 } 456 457 if (!max_fetch_desc) 458 max_fetch_desc = ICE_TXTIME_FETCH_TS_DESC_DFLT; 459 460 max_fetch_desc = ALIGN(max_fetch_desc, ICE_REQ_DESC_MULTIPLE); 461 462 return tx_ring->count + max_fetch_desc; 463 } 464 465 /** 466 * ice_setup_rx_ctx - Configure a receive ring context 467 * @ring: The Rx ring to configure 468 * 469 * Configure the Rx descriptor ring in RLAN context. 470 */ 471 static int ice_setup_rx_ctx(struct ice_rx_ring *ring) 472 { 473 struct ice_vsi *vsi = ring->vsi; 474 u32 rxdid = ICE_RXDID_FLEX_NIC; 475 struct ice_rlan_ctx rlan_ctx; 476 struct ice_hw *hw; 477 u16 pf_q; 478 int err; 479 480 hw = &vsi->back->hw; 481 482 /* what is Rx queue number in global space of 2K Rx queues */ 483 pf_q = vsi->rxq_map[ring->q_index]; 484 485 /* clear the context structure first */ 486 memset(&rlan_ctx, 0, sizeof(rlan_ctx)); 487 488 /* Receive Queue Base Address. 489 * Indicates the starting address of the descriptor queue defined in 490 * 128 Byte units. 491 */ 492 rlan_ctx.base = ring->dma >> ICE_RLAN_BASE_S; 493 494 rlan_ctx.qlen = ring->count; 495 496 /* Receive Packet Data Buffer Size. 497 * The Packet Data Buffer Size is defined in 128 byte units. 498 */ 499 rlan_ctx.dbuf = DIV_ROUND_UP(ring->rx_buf_len, 500 BIT_ULL(ICE_RLAN_CTX_DBUF_S)); 501 502 /* use 32 byte descriptors */ 503 rlan_ctx.dsize = 1; 504 505 /* Strip the Ethernet CRC bytes before the packet is posted to host 506 * memory. 507 */ 508 rlan_ctx.crcstrip = !(ring->flags & ICE_RX_FLAGS_CRC_STRIP_DIS); 509 510 /* L2TSEL flag defines the reported L2 Tags in the receive descriptor 511 * and it needs to remain 1 for non-DVM capable configurations to not 512 * break backward compatibility for VF drivers. Setting this field to 0 513 * will cause the single/outer VLAN tag to be stripped to the L2TAG2_2ND 514 * field in the Rx descriptor. Setting it to 1 allows the VLAN tag to 515 * be stripped in L2TAG1 of the Rx descriptor, which is where VFs will 516 * check for the tag 517 */ 518 if (ice_is_dvm_ena(hw)) 519 if (vsi->type == ICE_VSI_VF && 520 ice_vf_is_port_vlan_ena(vsi->vf)) 521 rlan_ctx.l2tsel = 1; 522 else 523 rlan_ctx.l2tsel = 0; 524 else 525 rlan_ctx.l2tsel = 1; 526 527 if (ring->hdr_pp) { 528 rlan_ctx.hbuf = ring->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S; 529 rlan_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 530 531 /* 532 * If the frame is TCP/UDP/SCTP, it will be split by the 533 * payload. 534 * If not, but it's an IPv4/IPv6 frame, it will be split by 535 * the IP header. 536 * If not IP, it will be split by the Ethernet header. 537 * 538 * In any case, the header buffer will never be left empty. 539 */ 540 rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2 | 541 ICE_RLAN_RX_HSPLIT_0_SPLIT_IP | 542 ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP | 543 ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP; 544 } else { 545 rlan_ctx.hbuf = 0; 546 rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT; 547 rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT; 548 } 549 550 rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT; 551 552 /* This controls whether VLAN is stripped from inner headers 553 * The VLAN in the inner L2 header is stripped to the receive 554 * descriptor if enabled by this flag. 555 */ 556 rlan_ctx.showiv = 0; 557 558 /* Max packet size for this queue - must not be set to a larger value 559 * than 5 x DBUF 560 */ 561 rlan_ctx.rxmax = min_t(u32, vsi->max_frame, 562 ICE_MAX_CHAINED_RX_BUFS * ring->rx_buf_len); 563 564 /* Rx queue threshold in units of 64 */ 565 rlan_ctx.lrxqthresh = 1; 566 567 /* Enable descriptor prefetch */ 568 rlan_ctx.prefena = 1; 569 570 /* PF acts as uplink for switchdev; set flex descriptor with src_vsi 571 * metadata and flags to allow redirecting to PR netdev 572 */ 573 if (ice_is_eswitch_mode_switchdev(vsi->back)) { 574 ring->flags |= ICE_RX_FLAGS_MULTIDEV; 575 rxdid = ICE_RXDID_FLEX_NIC_2; 576 } 577 578 /* Enable Flexible Descriptors in the queue context which 579 * allows this driver to select a specific receive descriptor format 580 * increasing context priority to pick up profile ID; default is 0x01; 581 * setting to 0x03 to ensure profile is programming if prev context is 582 * of same priority 583 */ 584 if (vsi->type != ICE_VSI_VF) 585 ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3, true); 586 587 /* Absolute queue number out of 2K needs to be passed */ 588 err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q); 589 if (err) { 590 dev_err(ice_pf_to_dev(vsi->back), "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", 591 pf_q, err); 592 return -EIO; 593 } 594 595 if (vsi->type == ICE_VSI_VF) 596 return 0; 597 598 /* init queue specific tail register */ 599 ring->tail = hw->hw_addr + QRX_TAIL(pf_q); 600 writel(0, ring->tail); 601 602 return 0; 603 } 604 605 static int ice_rxq_pp_create(struct ice_rx_ring *rq) 606 { 607 struct libeth_fq fq = { 608 .count = rq->count, 609 .nid = NUMA_NO_NODE, 610 .hsplit = rq->vsi->hsplit, 611 .xdp = ice_is_xdp_ena_vsi(rq->vsi), 612 .buf_len = LIBIE_MAX_RX_BUF_LEN, 613 }; 614 int err; 615 616 err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); 617 if (err) 618 return err; 619 620 rq->pp = fq.pp; 621 rq->rx_fqes = fq.fqes; 622 rq->truesize = fq.truesize; 623 rq->rx_buf_len = fq.buf_len; 624 625 if (!fq.hsplit) 626 return 0; 627 628 fq = (struct libeth_fq){ 629 .count = rq->count, 630 .type = LIBETH_FQE_HDR, 631 .nid = NUMA_NO_NODE, 632 .xdp = ice_is_xdp_ena_vsi(rq->vsi), 633 }; 634 635 err = libeth_rx_fq_create(&fq, &rq->q_vector->napi); 636 if (err) 637 goto destroy; 638 639 rq->hdr_pp = fq.pp; 640 rq->hdr_fqes = fq.fqes; 641 rq->hdr_truesize = fq.truesize; 642 rq->rx_hdr_len = fq.buf_len; 643 644 return 0; 645 646 destroy: 647 ice_rxq_pp_destroy(rq); 648 649 return err; 650 } 651 652 /** 653 * ice_vsi_cfg_rxq - Configure an Rx queue 654 * @ring: the ring being configured 655 * 656 * Return 0 on success and a negative value on error. 657 */ 658 static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) 659 { 660 struct device *dev = ice_pf_to_dev(ring->vsi->back); 661 u32 num_bufs = ICE_DESC_UNUSED(ring); 662 u32 rx_buf_len; 663 int err; 664 665 if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF) { 666 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 667 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 668 ring->q_index, 669 ring->q_vector->napi.napi_id, 670 ring->rx_buf_len); 671 if (err) 672 return err; 673 } 674 675 ice_rx_xsk_pool(ring); 676 err = ice_realloc_rx_xdp_bufs(ring, ring->xsk_pool); 677 if (err) 678 return err; 679 680 if (ring->xsk_pool) { 681 xdp_rxq_info_unreg(&ring->xdp_rxq); 682 683 rx_buf_len = 684 xsk_pool_get_rx_frame_size(ring->xsk_pool); 685 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 686 ring->q_index, 687 ring->q_vector->napi.napi_id, 688 rx_buf_len); 689 if (err) 690 return err; 691 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 692 MEM_TYPE_XSK_BUFF_POOL, 693 NULL); 694 if (err) 695 return err; 696 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 697 698 dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 699 ring->q_index); 700 } else { 701 err = ice_rxq_pp_create(ring); 702 if (err) 703 return err; 704 705 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 706 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 707 ring->q_index, 708 ring->q_vector->napi.napi_id, 709 ring->rx_buf_len); 710 if (err) 711 goto err_destroy_fq; 712 } 713 xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, 714 ring->pp); 715 } 716 } 717 718 ring->xdp.data = NULL; 719 err = ice_setup_rx_ctx(ring); 720 if (err) { 721 dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", 722 ring->q_index, err); 723 goto err_destroy_fq; 724 } 725 726 if (ring->xsk_pool) { 727 bool ok; 728 729 if (!xsk_buff_can_alloc(ring->xsk_pool, num_bufs)) { 730 dev_warn(dev, "XSK buffer pool does not provide enough addresses to fill %d buffers on Rx ring %d\n", 731 num_bufs, ring->q_index); 732 dev_warn(dev, "Change Rx ring/fill queue size to avoid performance issues\n"); 733 734 return 0; 735 } 736 737 ok = ice_alloc_rx_bufs_zc(ring, ring->xsk_pool, num_bufs); 738 if (!ok) { 739 u16 pf_q = ring->vsi->rxq_map[ring->q_index]; 740 741 dev_info(dev, "Failed to allocate some buffers on XSK buffer pool enabled Rx ring %d (pf_q %d)\n", 742 ring->q_index, pf_q); 743 } 744 745 return 0; 746 } 747 748 if (ring->vsi->type == ICE_VSI_CTRL) 749 ice_init_ctrl_rx_descs(ring, num_bufs); 750 else 751 err = ice_alloc_rx_bufs(ring, num_bufs); 752 753 if (err) 754 goto err_destroy_fq; 755 756 return 0; 757 758 err_destroy_fq: 759 ice_rxq_pp_destroy(ring); 760 761 return err; 762 } 763 764 int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx) 765 { 766 if (q_idx >= vsi->num_rxq) 767 return -EINVAL; 768 769 return ice_vsi_cfg_rxq(vsi->rx_rings[q_idx]); 770 } 771 772 /** 773 * ice_vsi_cfg_frame_size - setup max frame size and Rx buffer length 774 * @vsi: VSI 775 * @ring: Rx ring to configure 776 * 777 * Determine the maximum frame size and Rx buffer length to use for a PF VSI. 778 * Set these in the associated Rx ring structure. 779 */ 780 static void ice_vsi_cfg_frame_size(struct ice_vsi *vsi, struct ice_rx_ring *ring) 781 { 782 if (!vsi->netdev) { 783 vsi->max_frame = ICE_MAX_FRAME_LEGACY_RX; 784 } else { 785 vsi->max_frame = ICE_AQ_SET_MAC_FRAME_SIZE_MAX; 786 } 787 } 788 789 /** 790 * ice_vsi_cfg_rxqs - Configure the VSI for Rx 791 * @vsi: the VSI being configured 792 * 793 * Return 0 on success and a negative value on error 794 * Configure the Rx VSI for operation. 795 */ 796 int ice_vsi_cfg_rxqs(struct ice_vsi *vsi) 797 { 798 u16 i; 799 800 /* set up individual rings */ 801 ice_for_each_rxq(vsi, i) { 802 struct ice_rx_ring *ring = vsi->rx_rings[i]; 803 int err; 804 805 if (vsi->type != ICE_VSI_VF) 806 ice_vsi_cfg_frame_size(vsi, ring); 807 808 err = ice_vsi_cfg_rxq(ring); 809 if (err) 810 return err; 811 } 812 813 return 0; 814 } 815 816 /** 817 * __ice_vsi_get_qs - helper function for assigning queues from PF to VSI 818 * @qs_cfg: gathered variables needed for pf->vsi queues assignment 819 * 820 * This function first tries to find contiguous space. If it is not successful, 821 * it tries with the scatter approach. 822 * 823 * Return 0 on success and -ENOMEM in case of no left space in PF queue bitmap 824 */ 825 int __ice_vsi_get_qs(struct ice_qs_cfg *qs_cfg) 826 { 827 int ret = 0; 828 829 ret = __ice_vsi_get_qs_contig(qs_cfg); 830 if (ret) { 831 /* contig failed, so try with scatter approach */ 832 qs_cfg->mapping_mode = ICE_VSI_MAP_SCATTER; 833 qs_cfg->q_count = min_t(unsigned int, qs_cfg->q_count, 834 qs_cfg->scatter_count); 835 ret = __ice_vsi_get_qs_sc(qs_cfg); 836 } 837 return ret; 838 } 839 840 /** 841 * ice_vsi_ctrl_one_rx_ring - start/stop VSI's Rx ring with no busy wait 842 * @vsi: the VSI being configured 843 * @ena: start or stop the Rx ring 844 * @rxq_idx: 0-based Rx queue index for the VSI passed in 845 * @wait: wait or don't wait for configuration to finish in hardware 846 * 847 * Return 0 on success and negative on error. 848 */ 849 int 850 ice_vsi_ctrl_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx, bool wait) 851 { 852 int pf_q = vsi->rxq_map[rxq_idx]; 853 struct ice_pf *pf = vsi->back; 854 struct ice_hw *hw = &pf->hw; 855 u32 rx_reg; 856 857 rx_reg = rd32(hw, QRX_CTRL(pf_q)); 858 859 /* Skip if the queue is already in the requested state */ 860 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M)) 861 return 0; 862 863 /* turn on/off the queue */ 864 if (ena) 865 rx_reg |= QRX_CTRL_QENA_REQ_M; 866 else 867 rx_reg &= ~QRX_CTRL_QENA_REQ_M; 868 wr32(hw, QRX_CTRL(pf_q), rx_reg); 869 870 if (!wait) 871 return 0; 872 873 ice_flush(hw); 874 return ice_pf_rxq_wait(pf, pf_q, ena); 875 } 876 877 /** 878 * ice_vsi_wait_one_rx_ring - wait for a VSI's Rx ring to be stopped/started 879 * @vsi: the VSI being configured 880 * @ena: true/false to verify Rx ring has been enabled/disabled respectively 881 * @rxq_idx: 0-based Rx queue index for the VSI passed in 882 * 883 * This routine will wait for the given Rx queue of the VSI to reach the 884 * enabled or disabled state. Returns -ETIMEDOUT in case of failing to reach 885 * the requested state after multiple retries; else will return 0 in case of 886 * success. 887 */ 888 int ice_vsi_wait_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx) 889 { 890 int pf_q = vsi->rxq_map[rxq_idx]; 891 struct ice_pf *pf = vsi->back; 892 893 return ice_pf_rxq_wait(pf, pf_q, ena); 894 } 895 896 /** 897 * ice_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 898 * @vsi: the VSI being configured 899 * 900 * We allocate one q_vector per queue interrupt. If allocation fails we 901 * return -ENOMEM. 902 */ 903 int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi) 904 { 905 struct device *dev = ice_pf_to_dev(vsi->back); 906 u16 v_idx; 907 int err; 908 909 if (vsi->q_vectors[0]) { 910 dev_dbg(dev, "VSI %d has existing q_vectors\n", vsi->vsi_num); 911 return -EEXIST; 912 } 913 914 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) { 915 err = ice_vsi_alloc_q_vector(vsi, v_idx); 916 if (err) 917 goto err_out; 918 } 919 920 return 0; 921 922 err_out: 923 924 dev_info(dev, "Failed to allocate %d q_vectors for VSI %d, new value %d", 925 vsi->num_q_vectors, vsi->vsi_num, v_idx); 926 vsi->num_q_vectors = v_idx; 927 return v_idx ? 0 : err; 928 } 929 930 /** 931 * ice_vsi_map_rings_to_vectors - Map VSI rings to interrupt vectors 932 * @vsi: the VSI being configured 933 * 934 * This function maps descriptor rings to the queue-specific vectors allotted 935 * through the MSI-X enabling code. On a constrained vector budget, we map Tx 936 * and Rx rings to the vector as "efficiently" as possible. 937 */ 938 void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi) 939 { 940 int q_vectors = vsi->num_q_vectors; 941 u16 tx_rings_rem, rx_rings_rem; 942 int v_id; 943 944 /* initially assigning remaining rings count to VSIs num queue value */ 945 tx_rings_rem = vsi->num_txq; 946 rx_rings_rem = vsi->num_rxq; 947 948 for (v_id = 0; v_id < q_vectors; v_id++) { 949 struct ice_q_vector *q_vector = vsi->q_vectors[v_id]; 950 u8 tx_rings_per_v, rx_rings_per_v; 951 u16 q_id, q_base; 952 953 /* Tx rings mapping to vector */ 954 tx_rings_per_v = (u8)DIV_ROUND_UP(tx_rings_rem, 955 q_vectors - v_id); 956 q_vector->num_ring_tx = tx_rings_per_v; 957 q_vector->tx.tx_ring = NULL; 958 q_vector->tx.itr_idx = ICE_TX_ITR; 959 q_base = vsi->num_txq - tx_rings_rem; 960 961 for (q_id = q_base; q_id < (q_base + tx_rings_per_v); q_id++) { 962 struct ice_tx_ring *tx_ring = vsi->tx_rings[q_id]; 963 964 tx_ring->q_vector = q_vector; 965 tx_ring->next = q_vector->tx.tx_ring; 966 q_vector->tx.tx_ring = tx_ring; 967 } 968 tx_rings_rem -= tx_rings_per_v; 969 970 /* Rx rings mapping to vector */ 971 rx_rings_per_v = (u8)DIV_ROUND_UP(rx_rings_rem, 972 q_vectors - v_id); 973 q_vector->num_ring_rx = rx_rings_per_v; 974 q_vector->rx.rx_ring = NULL; 975 q_vector->rx.itr_idx = ICE_RX_ITR; 976 q_base = vsi->num_rxq - rx_rings_rem; 977 978 for (q_id = q_base; q_id < (q_base + rx_rings_per_v); q_id++) { 979 struct ice_rx_ring *rx_ring = vsi->rx_rings[q_id]; 980 981 rx_ring->q_vector = q_vector; 982 rx_ring->next = q_vector->rx.rx_ring; 983 q_vector->rx.rx_ring = rx_ring; 984 } 985 rx_rings_rem -= rx_rings_per_v; 986 } 987 988 if (ice_is_xdp_ena_vsi(vsi)) 989 ice_map_xdp_rings(vsi); 990 } 991 992 /** 993 * ice_vsi_free_q_vectors - Free memory allocated for interrupt vectors 994 * @vsi: the VSI having memory freed 995 */ 996 void ice_vsi_free_q_vectors(struct ice_vsi *vsi) 997 { 998 int v_idx; 999 1000 ice_for_each_q_vector(vsi, v_idx) 1001 ice_free_q_vector(vsi, v_idx); 1002 1003 vsi->num_q_vectors = 0; 1004 } 1005 1006 /** 1007 * ice_cfg_tstamp - Configure Tx time stamp queue 1008 * @tx_ring: Tx ring to be configured with timestamping 1009 * 1010 * Return: 0 on success and a negative value on error. 1011 */ 1012 static int 1013 ice_cfg_tstamp(struct ice_tx_ring *tx_ring) 1014 { 1015 DEFINE_RAW_FLEX(struct ice_aqc_set_txtime_qgrp, txtime_qg_buf, 1016 txtimeqs, 1); 1017 u8 txtime_buf_len = struct_size(txtime_qg_buf, txtimeqs, 1); 1018 struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; 1019 struct ice_txtime_ctx txtime_ctx = {}; 1020 struct ice_vsi *vsi = tx_ring->vsi; 1021 struct ice_pf *pf = vsi->back; 1022 struct ice_hw *hw = &pf->hw; 1023 u16 pf_q = tx_ring->reg_idx; 1024 int err; 1025 1026 err = ice_setup_txtime_ctx(tstamp_ring, &txtime_ctx); 1027 if (err) { 1028 dev_err(ice_pf_to_dev(pf), "Failed to setup Tx time queue context for queue %d, error: %d\n", 1029 pf_q, err); 1030 return err; 1031 } 1032 ice_pack_txtime_ctx(&txtime_ctx, 1033 &txtime_qg_buf->txtimeqs[0].txtime_ctx); 1034 1035 tstamp_ring->tail = hw->hw_addr + E830_GLQTX_TXTIME_DBELL_LSB(pf_q); 1036 return ice_aq_set_txtimeq(hw, pf_q, 1, txtime_qg_buf, 1037 txtime_buf_len, NULL); 1038 } 1039 1040 /** 1041 * ice_vsi_cfg_txq - Configure single Tx queue 1042 * @vsi: the VSI that queue belongs to 1043 * @ring: Tx ring to be configured 1044 * @qg_buf: queue group buffer 1045 * 1046 * Return: 0 on success and a negative value on error. 1047 */ 1048 static int 1049 ice_vsi_cfg_txq(const struct ice_vsi *vsi, struct ice_tx_ring *ring, 1050 struct ice_aqc_add_tx_qgrp *qg_buf) 1051 { 1052 u8 buf_len = struct_size(qg_buf, txqs, 1); 1053 struct ice_tlan_ctx tlan_ctx = { 0 }; 1054 struct ice_aqc_add_txqs_perq *txq; 1055 struct ice_channel *ch = ring->ch; 1056 struct ice_pf *pf = vsi->back; 1057 struct ice_hw *hw = &pf->hw; 1058 u32 pf_q, vsi_idx; 1059 int status; 1060 u8 tc; 1061 1062 /* Configure XPS */ 1063 ice_cfg_xps_tx_ring(ring); 1064 1065 pf_q = ring->reg_idx; 1066 status = ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); 1067 if (status) { 1068 dev_err(ice_pf_to_dev(pf), "Failed to setup Tx context for queue %d, error: %d\n", 1069 pf_q, status); 1070 return status; 1071 } 1072 /* copy context contents into the qg_buf */ 1073 qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q); 1074 ice_pack_txq_ctx(&tlan_ctx, &qg_buf->txqs[0].txq_ctx); 1075 1076 /* init queue specific tail reg. It is referred as 1077 * transmit comm scheduler queue doorbell. 1078 */ 1079 ring->tail = hw->hw_addr + QTX_COMM_DBELL(pf_q); 1080 1081 if (IS_ENABLED(CONFIG_DCB)) 1082 tc = ring->dcb_tc; 1083 else 1084 tc = 0; 1085 1086 /* Add unique software queue handle of the Tx queue per 1087 * TC into the VSI Tx ring 1088 */ 1089 ring->q_handle = ice_calc_txq_handle(vsi, ring, tc); 1090 1091 if (ch) { 1092 tc = 0; 1093 vsi_idx = ch->ch_vsi->idx; 1094 } else { 1095 vsi_idx = vsi->idx; 1096 } 1097 1098 status = ice_ena_vsi_txq(vsi->port_info, vsi_idx, tc, ring->q_handle, 1099 1, qg_buf, buf_len, NULL); 1100 if (status) { 1101 dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %d\n", 1102 status); 1103 return status; 1104 } 1105 1106 /* Add Tx Queue TEID into the VSI Tx ring from the 1107 * response. This will complete configuring and 1108 * enabling the queue. 1109 */ 1110 txq = &qg_buf->txqs[0]; 1111 if (pf_q == le16_to_cpu(txq->txq_id)) 1112 ring->txq_teid = le32_to_cpu(txq->q_teid); 1113 1114 if (ice_is_txtime_ena(ring)) { 1115 status = ice_alloc_setup_tstamp_ring(ring); 1116 if (status) { 1117 dev_err(ice_pf_to_dev(pf), 1118 "Failed to allocate Tx timestamp ring, error: %d\n", 1119 status); 1120 goto err_setup_tstamp; 1121 } 1122 1123 status = ice_cfg_tstamp(ring); 1124 if (status) { 1125 dev_err(ice_pf_to_dev(pf), "Failed to set Tx Time queue context, error: %d\n", 1126 status); 1127 goto err_cfg_tstamp; 1128 } 1129 } 1130 return 0; 1131 1132 err_cfg_tstamp: 1133 ice_free_tx_tstamp_ring(ring); 1134 err_setup_tstamp: 1135 ice_dis_vsi_txq(vsi->port_info, vsi_idx, tc, 1, &ring->q_handle, 1136 &ring->reg_idx, &ring->txq_teid, ICE_NO_RESET, 1137 tlan_ctx.vmvf_num, NULL); 1138 1139 return status; 1140 } 1141 1142 int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_tx_ring **tx_rings, 1143 u16 q_idx) 1144 { 1145 DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); 1146 1147 if (q_idx >= vsi->alloc_txq || !tx_rings || !tx_rings[q_idx]) 1148 return -EINVAL; 1149 1150 qg_buf->num_txqs = 1; 1151 1152 return ice_vsi_cfg_txq(vsi, tx_rings[q_idx], qg_buf); 1153 } 1154 1155 /** 1156 * ice_vsi_cfg_txqs - Configure the VSI for Tx 1157 * @vsi: the VSI being configured 1158 * @rings: Tx ring array to be configured 1159 * @count: number of Tx ring array elements 1160 * 1161 * Return 0 on success and a negative value on error 1162 * Configure the Tx VSI for operation. 1163 */ 1164 static int 1165 ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_tx_ring **rings, u16 count) 1166 { 1167 DEFINE_RAW_FLEX(struct ice_aqc_add_tx_qgrp, qg_buf, txqs, 1); 1168 int err = 0; 1169 u16 q_idx; 1170 1171 qg_buf->num_txqs = 1; 1172 1173 for (q_idx = 0; q_idx < count; q_idx++) { 1174 err = ice_vsi_cfg_txq(vsi, rings[q_idx], qg_buf); 1175 if (err) 1176 break; 1177 } 1178 1179 return err; 1180 } 1181 1182 /** 1183 * ice_vsi_cfg_lan_txqs - Configure the VSI for Tx 1184 * @vsi: the VSI being configured 1185 * 1186 * Return 0 on success and a negative value on error 1187 * Configure the Tx VSI for operation. 1188 */ 1189 int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi) 1190 { 1191 return ice_vsi_cfg_txqs(vsi, vsi->tx_rings, vsi->num_txq); 1192 } 1193 1194 /** 1195 * ice_vsi_cfg_xdp_txqs - Configure Tx queues dedicated for XDP in given VSI 1196 * @vsi: the VSI being configured 1197 * 1198 * Return 0 on success and a negative value on error 1199 * Configure the Tx queues dedicated for XDP in given VSI for operation. 1200 */ 1201 int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi) 1202 { 1203 int ret; 1204 int i; 1205 1206 ret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings, vsi->num_xdp_txq); 1207 if (ret) 1208 return ret; 1209 1210 ice_for_each_rxq(vsi, i) 1211 ice_tx_xsk_pool(vsi, i); 1212 1213 return 0; 1214 } 1215 1216 /** 1217 * ice_cfg_itr - configure the initial interrupt throttle values 1218 * @hw: pointer to the HW structure 1219 * @q_vector: interrupt vector that's being configured 1220 * 1221 * Configure interrupt throttling values for the ring containers that are 1222 * associated with the interrupt vector passed in. 1223 */ 1224 void ice_cfg_itr(struct ice_hw *hw, struct ice_q_vector *q_vector) 1225 { 1226 ice_cfg_itr_gran(hw); 1227 1228 if (q_vector->num_ring_rx) 1229 ice_write_itr(&q_vector->rx, q_vector->rx.itr_setting); 1230 1231 if (q_vector->num_ring_tx) 1232 ice_write_itr(&q_vector->tx, q_vector->tx.itr_setting); 1233 1234 ice_write_intrl(q_vector, q_vector->intrl); 1235 } 1236 1237 /** 1238 * ice_cfg_txq_interrupt - configure interrupt on Tx queue 1239 * @vsi: the VSI being configured 1240 * @txq: Tx queue being mapped to MSI-X vector 1241 * @msix_idx: MSI-X vector index within the function 1242 * @itr_idx: ITR index of the interrupt cause 1243 * 1244 * Configure interrupt on Tx queue by associating Tx queue to MSI-X vector 1245 * within the function space. 1246 */ 1247 void 1248 ice_cfg_txq_interrupt(struct ice_vsi *vsi, u16 txq, u16 msix_idx, u16 itr_idx) 1249 { 1250 struct ice_pf *pf = vsi->back; 1251 struct ice_hw *hw = &pf->hw; 1252 u32 val; 1253 1254 itr_idx = FIELD_PREP(QINT_TQCTL_ITR_INDX_M, itr_idx); 1255 1256 val = QINT_TQCTL_CAUSE_ENA_M | itr_idx | 1257 FIELD_PREP(QINT_TQCTL_MSIX_INDX_M, msix_idx); 1258 1259 wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val); 1260 if (ice_is_xdp_ena_vsi(vsi)) { 1261 u32 xdp_txq = txq + vsi->num_xdp_txq; 1262 1263 wr32(hw, QINT_TQCTL(vsi->txq_map[xdp_txq]), 1264 val); 1265 } 1266 ice_flush(hw); 1267 } 1268 1269 /** 1270 * ice_cfg_rxq_interrupt - configure interrupt on Rx queue 1271 * @vsi: the VSI being configured 1272 * @rxq: Rx queue being mapped to MSI-X vector 1273 * @msix_idx: MSI-X vector index within the function 1274 * @itr_idx: ITR index of the interrupt cause 1275 * 1276 * Configure interrupt on Rx queue by associating Rx queue to MSI-X vector 1277 * within the function space. 1278 */ 1279 void 1280 ice_cfg_rxq_interrupt(struct ice_vsi *vsi, u16 rxq, u16 msix_idx, u16 itr_idx) 1281 { 1282 struct ice_pf *pf = vsi->back; 1283 struct ice_hw *hw = &pf->hw; 1284 u32 val; 1285 1286 itr_idx = FIELD_PREP(QINT_RQCTL_ITR_INDX_M, itr_idx); 1287 1288 val = QINT_RQCTL_CAUSE_ENA_M | itr_idx | 1289 FIELD_PREP(QINT_RQCTL_MSIX_INDX_M, msix_idx); 1290 1291 wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val); 1292 1293 ice_flush(hw); 1294 } 1295 1296 /** 1297 * ice_trigger_sw_intr - trigger a software interrupt 1298 * @hw: pointer to the HW structure 1299 * @q_vector: interrupt vector to trigger the software interrupt for 1300 */ 1301 void ice_trigger_sw_intr(struct ice_hw *hw, const struct ice_q_vector *q_vector) 1302 { 1303 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 1304 (ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S) | 1305 GLINT_DYN_CTL_SWINT_TRIG_M | 1306 GLINT_DYN_CTL_INTENA_M); 1307 } 1308 1309 /** 1310 * ice_vsi_stop_tx_ring - Disable single Tx ring 1311 * @vsi: the VSI being configured 1312 * @rst_src: reset source 1313 * @rel_vmvf_num: Relative ID of VF/VM 1314 * @ring: Tx ring to be stopped 1315 * @txq_meta: Meta data of Tx ring to be stopped 1316 */ 1317 int 1318 ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, 1319 u16 rel_vmvf_num, struct ice_tx_ring *ring, 1320 struct ice_txq_meta *txq_meta) 1321 { 1322 struct ice_pf *pf = vsi->back; 1323 struct ice_q_vector *q_vector; 1324 struct ice_hw *hw = &pf->hw; 1325 int status; 1326 u32 val; 1327 1328 /* clear cause_ena bit for disabled queues */ 1329 val = rd32(hw, QINT_TQCTL(ring->reg_idx)); 1330 val &= ~QINT_TQCTL_CAUSE_ENA_M; 1331 wr32(hw, QINT_TQCTL(ring->reg_idx), val); 1332 1333 /* software is expected to wait for 100 ns */ 1334 ndelay(100); 1335 1336 /* trigger a software interrupt for the vector 1337 * associated to the queue to schedule NAPI handler 1338 */ 1339 q_vector = ring->q_vector; 1340 if (q_vector && !(vsi->vf && ice_is_vf_disabled(vsi->vf))) 1341 ice_trigger_sw_intr(hw, q_vector); 1342 1343 status = ice_dis_vsi_txq(vsi->port_info, txq_meta->vsi_idx, 1344 txq_meta->tc, 1, &txq_meta->q_handle, 1345 &txq_meta->q_id, &txq_meta->q_teid, rst_src, 1346 rel_vmvf_num, NULL); 1347 1348 /* if the disable queue command was exercised during an 1349 * active reset flow, -EBUSY is returned. 1350 * This is not an error as the reset operation disables 1351 * queues at the hardware level anyway. 1352 */ 1353 if (status == -EBUSY) { 1354 dev_dbg(ice_pf_to_dev(vsi->back), "Reset in progress. LAN Tx queues already disabled\n"); 1355 } else if (status == -ENOENT) { 1356 dev_dbg(ice_pf_to_dev(vsi->back), "LAN Tx queues do not exist, nothing to disable\n"); 1357 } else if (status) { 1358 dev_dbg(ice_pf_to_dev(vsi->back), "Failed to disable LAN Tx queues, error: %d\n", 1359 status); 1360 return status; 1361 } 1362 1363 return 0; 1364 } 1365 1366 /** 1367 * ice_fill_txq_meta - Prepare the Tx queue's meta data 1368 * @vsi: VSI that ring belongs to 1369 * @ring: ring that txq_meta will be based on 1370 * @txq_meta: a helper struct that wraps Tx queue's information 1371 * 1372 * Set up a helper struct that will contain all the necessary fields that 1373 * are needed for stopping Tx queue 1374 */ 1375 void 1376 ice_fill_txq_meta(const struct ice_vsi *vsi, struct ice_tx_ring *ring, 1377 struct ice_txq_meta *txq_meta) 1378 { 1379 struct ice_channel *ch = ring->ch; 1380 u8 tc; 1381 1382 if (IS_ENABLED(CONFIG_DCB)) 1383 tc = ring->dcb_tc; 1384 else 1385 tc = 0; 1386 1387 txq_meta->q_id = ring->reg_idx; 1388 txq_meta->q_teid = ring->txq_teid; 1389 txq_meta->q_handle = ring->q_handle; 1390 if (ch) { 1391 txq_meta->vsi_idx = ch->ch_vsi->idx; 1392 txq_meta->tc = 0; 1393 } else { 1394 txq_meta->vsi_idx = vsi->idx; 1395 txq_meta->tc = tc; 1396 } 1397 } 1398 1399 /** 1400 * ice_qp_reset_stats - Resets all stats for rings of given index 1401 * @vsi: VSI that contains rings of interest 1402 * @q_idx: ring index in array 1403 */ 1404 static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) 1405 { 1406 struct ice_vsi_stats *vsi_stat; 1407 struct ice_pf *pf; 1408 1409 pf = vsi->back; 1410 if (!pf->vsi_stats) 1411 return; 1412 1413 vsi_stat = pf->vsi_stats[vsi->idx]; 1414 if (!vsi_stat) 1415 return; 1416 1417 memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, 1418 sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); 1419 memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, 1420 sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); 1421 if (vsi->xdp_rings) 1422 memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, 1423 sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); 1424 } 1425 1426 /** 1427 * ice_qp_clean_rings - Cleans all the rings of a given index 1428 * @vsi: VSI that contains rings of interest 1429 * @q_idx: ring index in array 1430 */ 1431 static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) 1432 { 1433 ice_clean_tx_ring(vsi->tx_rings[q_idx]); 1434 if (vsi->xdp_rings) 1435 ice_clean_tx_ring(vsi->xdp_rings[q_idx]); 1436 ice_clean_rx_ring(vsi->rx_rings[q_idx]); 1437 } 1438 1439 /** 1440 * ice_qp_dis - Disables a queue pair 1441 * @vsi: VSI of interest 1442 * @q_idx: ring index in array 1443 * 1444 * Returns 0 on success, negative on failure. 1445 */ 1446 int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) 1447 { 1448 struct ice_txq_meta txq_meta = { }; 1449 struct ice_q_vector *q_vector; 1450 struct ice_tx_ring *tx_ring; 1451 struct ice_rx_ring *rx_ring; 1452 int fail = 0; 1453 int err; 1454 1455 if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) 1456 return -EINVAL; 1457 1458 tx_ring = vsi->tx_rings[q_idx]; 1459 rx_ring = vsi->rx_rings[q_idx]; 1460 q_vector = rx_ring->q_vector; 1461 1462 synchronize_net(); 1463 netif_carrier_off(vsi->netdev); 1464 netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 1465 1466 ice_qvec_dis_irq(vsi, rx_ring, q_vector); 1467 ice_qvec_toggle_napi(vsi, q_vector, false); 1468 1469 ice_fill_txq_meta(vsi, tx_ring, &txq_meta); 1470 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); 1471 if (!fail) 1472 fail = err; 1473 if (vsi->xdp_rings) { 1474 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 1475 1476 memset(&txq_meta, 0, sizeof(txq_meta)); 1477 ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); 1478 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, 1479 &txq_meta); 1480 if (!fail) 1481 fail = err; 1482 } 1483 1484 ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false); 1485 ice_qp_clean_rings(vsi, q_idx); 1486 ice_qp_reset_stats(vsi, q_idx); 1487 1488 return fail; 1489 } 1490 1491 /** 1492 * ice_qp_ena - Enables a queue pair 1493 * @vsi: VSI of interest 1494 * @q_idx: ring index in array 1495 * 1496 * Returns 0 on success, negative on failure. 1497 */ 1498 int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) 1499 { 1500 struct ice_q_vector *q_vector; 1501 int fail = 0; 1502 bool link_up; 1503 int err; 1504 1505 err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx); 1506 if (!fail) 1507 fail = err; 1508 1509 if (ice_is_xdp_ena_vsi(vsi)) { 1510 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; 1511 1512 err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx); 1513 if (!fail) 1514 fail = err; 1515 ice_set_ring_xdp(xdp_ring); 1516 ice_tx_xsk_pool(vsi, q_idx); 1517 } 1518 1519 err = ice_vsi_cfg_single_rxq(vsi, q_idx); 1520 if (!fail) 1521 fail = err; 1522 1523 q_vector = vsi->rx_rings[q_idx]->q_vector; 1524 ice_qvec_cfg_msix(vsi, q_vector, q_idx); 1525 1526 err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); 1527 if (!fail) 1528 fail = err; 1529 1530 ice_qvec_toggle_napi(vsi, q_vector, true); 1531 ice_qvec_ena_irq(vsi, q_vector); 1532 1533 /* make sure NAPI sees updated ice_{t,x}_ring::xsk_pool */ 1534 synchronize_net(); 1535 ice_get_link_status(vsi->port_info, &link_up); 1536 if (link_up) { 1537 netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); 1538 netif_carrier_on(vsi->netdev); 1539 } 1540 1541 return fail; 1542 } 1543