1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2023 Intel Corporation */ 3 4 #include "idpf.h" 5 #include "idpf_ptp.h" 6 #include "idpf_virtchnl.h" 7 #include "xdp.h" 8 #include "xsk.h" 9 10 #define idpf_tx_buf_next(buf) (*(u32 *)&(buf)->priv) 11 LIBETH_SQE_CHECK_PRIV(u32); 12 13 /** 14 * idpf_chk_linearize - Check if skb exceeds max descriptors per packet 15 * @skb: send buffer 16 * @max_bufs: maximum scatter gather buffers for single packet 17 * @count: number of buffers this packet needs 18 * 19 * Make sure we don't exceed maximum scatter gather buffers for a single 20 * packet. 21 * TSO case has been handled earlier from idpf_features_check(). 22 */ 23 static bool idpf_chk_linearize(const struct sk_buff *skb, 24 unsigned int max_bufs, 25 unsigned int count) 26 { 27 if (likely(count <= max_bufs)) 28 return false; 29 30 if (skb_is_gso(skb)) 31 return false; 32 33 return true; 34 } 35 36 /** 37 * idpf_tx_timeout - Respond to a Tx Hang 38 * @netdev: network interface device structure 39 * @txqueue: TX queue 40 */ 41 void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 42 { 43 struct idpf_adapter *adapter = idpf_netdev_to_adapter(netdev); 44 45 adapter->tx_timeout_count++; 46 47 netdev_err(netdev, "Detected Tx timeout: Count %d, Queue %d\n", 48 adapter->tx_timeout_count, txqueue); 49 if (!idpf_is_reset_in_prog(adapter)) { 50 set_bit(IDPF_HR_FUNC_RESET, adapter->flags); 51 queue_delayed_work(adapter->vc_event_wq, 52 &adapter->vc_event_task, 53 msecs_to_jiffies(10)); 54 } 55 } 56 57 static void idpf_tx_buf_clean(struct idpf_tx_queue *txq) 58 { 59 struct libeth_sq_napi_stats ss = { }; 60 struct xdp_frame_bulk bq; 61 struct libeth_cq_pp cp = { 62 .dev = txq->dev, 63 .bq = &bq, 64 .ss = &ss, 65 }; 66 67 xdp_frame_bulk_init(&bq); 68 69 /* Free all the Tx buffer sk_buffs */ 70 for (u32 i = 0; i < txq->buf_pool_size; i++) 71 libeth_tx_complete_any(&txq->tx_buf[i], &cp); 72 73 xdp_flush_frame_bulk(&bq); 74 } 75 76 /** 77 * idpf_tx_buf_rel_all - Free any empty Tx buffers 78 * @txq: queue to be cleaned 79 */ 80 static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 81 { 82 /* Buffers already cleared, nothing to do */ 83 if (!txq->tx_buf) 84 return; 85 86 if (idpf_queue_has(XSK, txq)) 87 idpf_xsksq_clean(txq); 88 else 89 idpf_tx_buf_clean(txq); 90 91 kfree(txq->tx_buf); 92 txq->tx_buf = NULL; 93 } 94 95 /** 96 * idpf_tx_desc_rel - Free Tx resources per queue 97 * @txq: Tx descriptor ring for a specific queue 98 * 99 * Free all transmit software resources 100 */ 101 static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) 102 { 103 bool xdp = idpf_queue_has(XDP, txq); 104 105 if (xdp) 106 libeth_xdpsq_deinit_timer(txq->timer); 107 108 idpf_tx_buf_rel_all(txq); 109 110 if (!xdp) 111 netdev_tx_reset_subqueue(txq->netdev, txq->idx); 112 113 idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX); 114 115 if (!txq->desc_ring) 116 return; 117 118 if (!xdp && txq->refillq) 119 kfree(txq->refillq->ring); 120 121 dmam_free_coherent(txq->dev, txq->size, txq->desc_ring, txq->dma); 122 txq->desc_ring = NULL; 123 txq->next_to_use = 0; 124 txq->next_to_clean = 0; 125 } 126 127 /** 128 * idpf_compl_desc_rel - Free completion resources per queue 129 * @complq: completion queue 130 * 131 * Free all completion software resources. 132 */ 133 static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) 134 { 135 idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 136 137 if (!complq->desc_ring) 138 return; 139 140 dma_free_coherent(complq->netdev->dev.parent, complq->size, 141 complq->desc_ring, complq->dma); 142 complq->desc_ring = NULL; 143 complq->next_to_use = 0; 144 complq->next_to_clean = 0; 145 } 146 147 /** 148 * idpf_tx_desc_rel_all - Free Tx Resources for All Queues 149 * @vport: virtual port structure 150 * 151 * Free all transmit software resources 152 */ 153 static void idpf_tx_desc_rel_all(struct idpf_vport *vport) 154 { 155 int i, j; 156 157 if (!vport->txq_grps) 158 return; 159 160 for (i = 0; i < vport->num_txq_grp; i++) { 161 struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; 162 163 for (j = 0; j < txq_grp->num_txq; j++) 164 idpf_tx_desc_rel(txq_grp->txqs[j]); 165 166 if (idpf_is_queue_model_split(vport->txq_model)) 167 idpf_compl_desc_rel(txq_grp->complq); 168 } 169 } 170 171 /** 172 * idpf_tx_buf_alloc_all - Allocate memory for all buffer resources 173 * @tx_q: queue for which the buffers are allocated 174 * 175 * Returns 0 on success, negative on failure 176 */ 177 static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) 178 { 179 /* Allocate book keeping buffers only. Buffers to be supplied to HW 180 * are allocated by kernel network stack and received as part of skb 181 */ 182 if (idpf_queue_has(FLOW_SCH_EN, tx_q)) 183 tx_q->buf_pool_size = U16_MAX; 184 else 185 tx_q->buf_pool_size = tx_q->desc_count; 186 tx_q->tx_buf = kcalloc(tx_q->buf_pool_size, sizeof(*tx_q->tx_buf), 187 GFP_KERNEL); 188 if (!tx_q->tx_buf) 189 return -ENOMEM; 190 191 return 0; 192 } 193 194 /** 195 * idpf_tx_desc_alloc - Allocate the Tx descriptors 196 * @vport: vport to allocate resources for 197 * @tx_q: the tx ring to set up 198 * 199 * Returns 0 on success, negative on failure 200 */ 201 static int idpf_tx_desc_alloc(const struct idpf_vport *vport, 202 struct idpf_tx_queue *tx_q) 203 { 204 struct device *dev = tx_q->dev; 205 struct idpf_sw_queue *refillq; 206 int err; 207 208 err = idpf_tx_buf_alloc_all(tx_q); 209 if (err) 210 goto err_alloc; 211 212 tx_q->size = tx_q->desc_count * sizeof(*tx_q->base_tx); 213 214 /* Allocate descriptors also round up to nearest 4K */ 215 tx_q->size = ALIGN(tx_q->size, 4096); 216 tx_q->desc_ring = dmam_alloc_coherent(dev, tx_q->size, &tx_q->dma, 217 GFP_KERNEL); 218 if (!tx_q->desc_ring) { 219 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 220 tx_q->size); 221 err = -ENOMEM; 222 goto err_alloc; 223 } 224 225 tx_q->next_to_use = 0; 226 tx_q->next_to_clean = 0; 227 idpf_queue_set(GEN_CHK, tx_q); 228 229 idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX); 230 231 if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) 232 return 0; 233 234 refillq = tx_q->refillq; 235 refillq->desc_count = tx_q->buf_pool_size; 236 refillq->ring = kcalloc(refillq->desc_count, sizeof(u32), 237 GFP_KERNEL); 238 if (!refillq->ring) { 239 err = -ENOMEM; 240 goto err_alloc; 241 } 242 243 for (unsigned int i = 0; i < refillq->desc_count; i++) 244 refillq->ring[i] = 245 FIELD_PREP(IDPF_RFL_BI_BUFID_M, i) | 246 FIELD_PREP(IDPF_RFL_BI_GEN_M, 247 idpf_queue_has(GEN_CHK, refillq)); 248 249 /* Go ahead and flip the GEN bit since this counts as filling 250 * up the ring, i.e. we already ring wrapped. 251 */ 252 idpf_queue_change(GEN_CHK, refillq); 253 254 tx_q->last_re = tx_q->desc_count - IDPF_TX_SPLITQ_RE_MIN_GAP; 255 256 return 0; 257 258 err_alloc: 259 idpf_tx_desc_rel(tx_q); 260 261 return err; 262 } 263 264 /** 265 * idpf_compl_desc_alloc - allocate completion descriptors 266 * @vport: vport to allocate resources for 267 * @complq: completion queue to set up 268 * 269 * Return: 0 on success, -errno on failure. 270 */ 271 static int idpf_compl_desc_alloc(const struct idpf_vport *vport, 272 struct idpf_compl_queue *complq) 273 { 274 u32 desc_size; 275 276 desc_size = idpf_queue_has(FLOW_SCH_EN, complq) ? 277 sizeof(*complq->comp) : sizeof(*complq->comp_4b); 278 complq->size = array_size(complq->desc_count, desc_size); 279 280 complq->desc_ring = dma_alloc_coherent(complq->netdev->dev.parent, 281 complq->size, &complq->dma, 282 GFP_KERNEL); 283 if (!complq->desc_ring) 284 return -ENOMEM; 285 286 complq->next_to_use = 0; 287 complq->next_to_clean = 0; 288 idpf_queue_set(GEN_CHK, complq); 289 290 idpf_xsk_setup_queue(vport, complq, 291 VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 292 293 return 0; 294 } 295 296 /** 297 * idpf_tx_desc_alloc_all - allocate all queues Tx resources 298 * @vport: virtual port private structure 299 * 300 * Returns 0 on success, negative on failure 301 */ 302 static int idpf_tx_desc_alloc_all(struct idpf_vport *vport) 303 { 304 int err = 0; 305 int i, j; 306 307 /* Setup buffer queues. In single queue model buffer queues and 308 * completion queues will be same 309 */ 310 for (i = 0; i < vport->num_txq_grp; i++) { 311 for (j = 0; j < vport->txq_grps[i].num_txq; j++) { 312 struct idpf_tx_queue *txq = vport->txq_grps[i].txqs[j]; 313 314 err = idpf_tx_desc_alloc(vport, txq); 315 if (err) { 316 pci_err(vport->adapter->pdev, 317 "Allocation for Tx Queue %u failed\n", 318 i); 319 goto err_out; 320 } 321 } 322 323 if (!idpf_is_queue_model_split(vport->txq_model)) 324 continue; 325 326 /* Setup completion queues */ 327 err = idpf_compl_desc_alloc(vport, vport->txq_grps[i].complq); 328 if (err) { 329 pci_err(vport->adapter->pdev, 330 "Allocation for Tx Completion Queue %u failed\n", 331 i); 332 goto err_out; 333 } 334 } 335 336 err_out: 337 if (err) 338 idpf_tx_desc_rel_all(vport); 339 340 return err; 341 } 342 343 /** 344 * idpf_rx_page_rel - Release an rx buffer page 345 * @rx_buf: the buffer to free 346 */ 347 static void idpf_rx_page_rel(struct libeth_fqe *rx_buf) 348 { 349 if (unlikely(!rx_buf->netmem)) 350 return; 351 352 libeth_rx_recycle_slow(rx_buf->netmem); 353 354 rx_buf->netmem = 0; 355 rx_buf->offset = 0; 356 } 357 358 /** 359 * idpf_rx_hdr_buf_rel_all - Release header buffer memory 360 * @bufq: queue to use 361 */ 362 static void idpf_rx_hdr_buf_rel_all(struct idpf_buf_queue *bufq) 363 { 364 struct libeth_fq fq = { 365 .fqes = bufq->hdr_buf, 366 .pp = bufq->hdr_pp, 367 }; 368 369 for (u32 i = 0; i < bufq->desc_count; i++) 370 idpf_rx_page_rel(&bufq->hdr_buf[i]); 371 372 libeth_rx_fq_destroy(&fq); 373 bufq->hdr_buf = NULL; 374 bufq->hdr_pp = NULL; 375 } 376 377 /** 378 * idpf_rx_buf_rel_bufq - Free all Rx buffer resources for a buffer queue 379 * @bufq: queue to be cleaned 380 */ 381 static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq) 382 { 383 struct libeth_fq fq = { 384 .fqes = bufq->buf, 385 .pp = bufq->pp, 386 }; 387 388 /* queue already cleared, nothing to do */ 389 if (!bufq->buf) 390 return; 391 392 if (idpf_queue_has(XSK, bufq)) { 393 idpf_xskfq_rel(bufq); 394 return; 395 } 396 397 /* Free all the bufs allocated and given to hw on Rx queue */ 398 for (u32 i = 0; i < bufq->desc_count; i++) 399 idpf_rx_page_rel(&bufq->buf[i]); 400 401 if (idpf_queue_has(HSPLIT_EN, bufq)) 402 idpf_rx_hdr_buf_rel_all(bufq); 403 404 libeth_rx_fq_destroy(&fq); 405 bufq->buf = NULL; 406 bufq->pp = NULL; 407 } 408 409 /** 410 * idpf_rx_buf_rel_all - Free all Rx buffer resources for a receive queue 411 * @rxq: queue to be cleaned 412 */ 413 static void idpf_rx_buf_rel_all(struct idpf_rx_queue *rxq) 414 { 415 struct libeth_fq fq = { 416 .fqes = rxq->rx_buf, 417 .pp = rxq->pp, 418 }; 419 420 if (!rxq->rx_buf) 421 return; 422 423 for (u32 i = 0; i < rxq->desc_count; i++) 424 idpf_rx_page_rel(&rxq->rx_buf[i]); 425 426 libeth_rx_fq_destroy(&fq); 427 rxq->rx_buf = NULL; 428 rxq->pp = NULL; 429 } 430 431 /** 432 * idpf_rx_desc_rel - Free a specific Rx q resources 433 * @rxq: queue to clean the resources from 434 * @dev: device to free DMA memory 435 * @model: single or split queue model 436 * 437 * Free a specific rx queue resources 438 */ 439 static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, 440 u32 model) 441 { 442 if (!rxq) 443 return; 444 445 if (!idpf_queue_has(XSK, rxq)) 446 libeth_xdp_return_stash(&rxq->xdp); 447 448 if (!idpf_is_queue_model_split(model)) 449 idpf_rx_buf_rel_all(rxq); 450 451 idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX); 452 453 rxq->next_to_alloc = 0; 454 rxq->next_to_clean = 0; 455 rxq->next_to_use = 0; 456 if (!rxq->desc_ring) 457 return; 458 459 dmam_free_coherent(dev, rxq->size, rxq->desc_ring, rxq->dma); 460 rxq->desc_ring = NULL; 461 } 462 463 /** 464 * idpf_rx_desc_rel_bufq - free buffer queue resources 465 * @bufq: buffer queue to clean the resources from 466 * @dev: device to free DMA memory 467 */ 468 static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, 469 struct device *dev) 470 { 471 if (!bufq) 472 return; 473 474 idpf_rx_buf_rel_bufq(bufq); 475 idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 476 477 bufq->next_to_alloc = 0; 478 bufq->next_to_clean = 0; 479 bufq->next_to_use = 0; 480 481 if (!bufq->split_buf) 482 return; 483 484 dma_free_coherent(dev, bufq->size, bufq->split_buf, bufq->dma); 485 bufq->split_buf = NULL; 486 } 487 488 /** 489 * idpf_rx_desc_rel_all - Free Rx Resources for All Queues 490 * @vport: virtual port structure 491 * 492 * Free all rx queues resources 493 */ 494 static void idpf_rx_desc_rel_all(struct idpf_vport *vport) 495 { 496 struct device *dev = &vport->adapter->pdev->dev; 497 struct idpf_rxq_group *rx_qgrp; 498 u16 num_rxq; 499 int i, j; 500 501 if (!vport->rxq_grps) 502 return; 503 504 for (i = 0; i < vport->num_rxq_grp; i++) { 505 rx_qgrp = &vport->rxq_grps[i]; 506 507 if (!idpf_is_queue_model_split(vport->rxq_model)) { 508 for (j = 0; j < rx_qgrp->singleq.num_rxq; j++) 509 idpf_rx_desc_rel(rx_qgrp->singleq.rxqs[j], dev, 510 VIRTCHNL2_QUEUE_MODEL_SINGLE); 511 continue; 512 } 513 514 num_rxq = rx_qgrp->splitq.num_rxq_sets; 515 for (j = 0; j < num_rxq; j++) 516 idpf_rx_desc_rel(&rx_qgrp->splitq.rxq_sets[j]->rxq, 517 dev, VIRTCHNL2_QUEUE_MODEL_SPLIT); 518 519 if (!rx_qgrp->splitq.bufq_sets) 520 continue; 521 522 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) { 523 struct idpf_bufq_set *bufq_set = 524 &rx_qgrp->splitq.bufq_sets[j]; 525 526 idpf_rx_desc_rel_bufq(&bufq_set->bufq, dev); 527 } 528 } 529 } 530 531 /** 532 * idpf_rx_buf_hw_update - Store the new tail and head values 533 * @bufq: queue to bump 534 * @val: new head index 535 */ 536 static void idpf_rx_buf_hw_update(struct idpf_buf_queue *bufq, u32 val) 537 { 538 bufq->next_to_use = val; 539 540 if (unlikely(!bufq->tail)) 541 return; 542 543 /* writel has an implicit memory barrier */ 544 writel(val, bufq->tail); 545 } 546 547 /** 548 * idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers 549 * @bufq: ring to use 550 * 551 * Returns 0 on success, negative on failure. 552 */ 553 static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq) 554 { 555 struct libeth_fq fq = { 556 .count = bufq->desc_count, 557 .type = LIBETH_FQE_HDR, 558 .xdp = idpf_xdp_enabled(bufq->q_vector->vport), 559 .nid = idpf_q_vector_to_mem(bufq->q_vector), 560 }; 561 int ret; 562 563 ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi); 564 if (ret) 565 return ret; 566 567 bufq->hdr_pp = fq.pp; 568 bufq->hdr_buf = fq.fqes; 569 bufq->hdr_truesize = fq.truesize; 570 bufq->rx_hbuf_size = fq.buf_len; 571 572 return 0; 573 } 574 575 /** 576 * idpf_post_buf_refill - Post buffer id to refill queue 577 * @refillq: refill queue to post to 578 * @buf_id: buffer id to post 579 */ 580 static void idpf_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id) 581 { 582 u32 nta = refillq->next_to_use; 583 584 /* store the buffer ID and the SW maintained GEN bit to the refillq */ 585 refillq->ring[nta] = 586 FIELD_PREP(IDPF_RFL_BI_BUFID_M, buf_id) | 587 FIELD_PREP(IDPF_RFL_BI_GEN_M, 588 idpf_queue_has(GEN_CHK, refillq)); 589 590 if (unlikely(++nta == refillq->desc_count)) { 591 nta = 0; 592 idpf_queue_change(GEN_CHK, refillq); 593 } 594 595 refillq->next_to_use = nta; 596 } 597 598 /** 599 * idpf_rx_post_buf_desc - Post buffer to bufq descriptor ring 600 * @bufq: buffer queue to post to 601 * @buf_id: buffer id to post 602 * 603 * Returns false if buffer could not be allocated, true otherwise. 604 */ 605 static bool idpf_rx_post_buf_desc(struct idpf_buf_queue *bufq, u16 buf_id) 606 { 607 struct virtchnl2_splitq_rx_buf_desc *splitq_rx_desc = NULL; 608 struct libeth_fq_fp fq = { 609 .count = bufq->desc_count, 610 }; 611 u16 nta = bufq->next_to_alloc; 612 dma_addr_t addr; 613 614 splitq_rx_desc = &bufq->split_buf[nta]; 615 616 if (idpf_queue_has(HSPLIT_EN, bufq)) { 617 fq.pp = bufq->hdr_pp; 618 fq.fqes = bufq->hdr_buf; 619 fq.truesize = bufq->hdr_truesize; 620 621 addr = libeth_rx_alloc(&fq, buf_id); 622 if (addr == DMA_MAPPING_ERROR) 623 return false; 624 625 splitq_rx_desc->hdr_addr = cpu_to_le64(addr); 626 } 627 628 fq.pp = bufq->pp; 629 fq.fqes = bufq->buf; 630 fq.truesize = bufq->truesize; 631 632 addr = libeth_rx_alloc(&fq, buf_id); 633 if (addr == DMA_MAPPING_ERROR) 634 return false; 635 636 splitq_rx_desc->pkt_addr = cpu_to_le64(addr); 637 splitq_rx_desc->qword0.buf_id = cpu_to_le16(buf_id); 638 639 nta++; 640 if (unlikely(nta == bufq->desc_count)) 641 nta = 0; 642 bufq->next_to_alloc = nta; 643 644 return true; 645 } 646 647 /** 648 * idpf_rx_post_init_bufs - Post initial buffers to bufq 649 * @bufq: buffer queue to post working set to 650 * @working_set: number of buffers to put in working set 651 * 652 * Returns true if @working_set bufs were posted successfully, false otherwise. 653 */ 654 static bool idpf_rx_post_init_bufs(struct idpf_buf_queue *bufq, 655 u16 working_set) 656 { 657 int i; 658 659 for (i = 0; i < working_set; i++) { 660 if (!idpf_rx_post_buf_desc(bufq, i)) 661 return false; 662 } 663 664 idpf_rx_buf_hw_update(bufq, ALIGN_DOWN(bufq->next_to_alloc, 665 IDPF_RX_BUF_STRIDE)); 666 667 return true; 668 } 669 670 /** 671 * idpf_rx_buf_alloc_singleq - Allocate memory for all buffer resources 672 * @rxq: queue for which the buffers are allocated 673 * 674 * Return: 0 on success, -ENOMEM on failure. 675 */ 676 static int idpf_rx_buf_alloc_singleq(struct idpf_rx_queue *rxq) 677 { 678 if (idpf_rx_singleq_buf_hw_alloc_all(rxq, rxq->desc_count - 1)) 679 goto err; 680 681 return 0; 682 683 err: 684 idpf_rx_buf_rel_all(rxq); 685 686 return -ENOMEM; 687 } 688 689 /** 690 * idpf_rx_bufs_init_singleq - Initialize page pool and allocate Rx bufs 691 * @rxq: buffer queue to create page pool for 692 * 693 * Return: 0 on success, -errno on failure. 694 */ 695 static int idpf_rx_bufs_init_singleq(struct idpf_rx_queue *rxq) 696 { 697 struct libeth_fq fq = { 698 .count = rxq->desc_count, 699 .type = LIBETH_FQE_MTU, 700 .buf_len = IDPF_RX_MAX_BUF_SZ, 701 .nid = idpf_q_vector_to_mem(rxq->q_vector), 702 }; 703 int ret; 704 705 ret = libeth_rx_fq_create(&fq, &rxq->q_vector->napi); 706 if (ret) 707 return ret; 708 709 rxq->pp = fq.pp; 710 rxq->rx_buf = fq.fqes; 711 rxq->truesize = fq.truesize; 712 rxq->rx_buf_size = fq.buf_len; 713 714 return idpf_rx_buf_alloc_singleq(rxq); 715 } 716 717 /** 718 * idpf_rx_buf_alloc_all - Allocate memory for all buffer resources 719 * @rxbufq: queue for which the buffers are allocated 720 * 721 * Returns 0 on success, negative on failure 722 */ 723 static int idpf_rx_buf_alloc_all(struct idpf_buf_queue *rxbufq) 724 { 725 int err = 0; 726 727 if (idpf_queue_has(HSPLIT_EN, rxbufq)) { 728 err = idpf_rx_hdr_buf_alloc_all(rxbufq); 729 if (err) 730 goto rx_buf_alloc_all_out; 731 } 732 733 /* Allocate buffers to be given to HW. */ 734 if (!idpf_rx_post_init_bufs(rxbufq, IDPF_RX_BUFQ_WORKING_SET(rxbufq))) 735 err = -ENOMEM; 736 737 rx_buf_alloc_all_out: 738 if (err) 739 idpf_rx_buf_rel_bufq(rxbufq); 740 741 return err; 742 } 743 744 /** 745 * idpf_rx_bufs_init - Initialize page pool, allocate rx bufs, and post to HW 746 * @bufq: buffer queue to create page pool for 747 * @type: type of Rx buffers to allocate 748 * 749 * Returns 0 on success, negative on failure 750 */ 751 static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, 752 enum libeth_fqe_type type) 753 { 754 struct libeth_fq fq = { 755 .truesize = bufq->truesize, 756 .count = bufq->desc_count, 757 .type = type, 758 .buf_len = IDPF_RX_MAX_BUF_SZ, 759 .hsplit = idpf_queue_has(HSPLIT_EN, bufq), 760 .xdp = idpf_xdp_enabled(bufq->q_vector->vport), 761 .nid = idpf_q_vector_to_mem(bufq->q_vector), 762 }; 763 int ret; 764 765 if (idpf_queue_has(XSK, bufq)) 766 return idpf_xskfq_init(bufq); 767 768 ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi); 769 if (ret) 770 return ret; 771 772 bufq->pp = fq.pp; 773 bufq->buf = fq.fqes; 774 bufq->truesize = fq.truesize; 775 bufq->rx_buf_size = fq.buf_len; 776 777 return idpf_rx_buf_alloc_all(bufq); 778 } 779 780 /** 781 * idpf_rx_bufs_init_all - Initialize all RX bufs 782 * @vport: virtual port struct 783 * 784 * Returns 0 on success, negative on failure 785 */ 786 int idpf_rx_bufs_init_all(struct idpf_vport *vport) 787 { 788 bool split = idpf_is_queue_model_split(vport->rxq_model); 789 int i, j, err; 790 791 idpf_xdp_copy_prog_to_rqs(vport, vport->xdp_prog); 792 793 for (i = 0; i < vport->num_rxq_grp; i++) { 794 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 795 u32 truesize = 0; 796 797 /* Allocate bufs for the rxq itself in singleq */ 798 if (!split) { 799 int num_rxq = rx_qgrp->singleq.num_rxq; 800 801 for (j = 0; j < num_rxq; j++) { 802 struct idpf_rx_queue *q; 803 804 q = rx_qgrp->singleq.rxqs[j]; 805 err = idpf_rx_bufs_init_singleq(q); 806 if (err) 807 return err; 808 } 809 810 continue; 811 } 812 813 /* Otherwise, allocate bufs for the buffer queues */ 814 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) { 815 enum libeth_fqe_type type; 816 struct idpf_buf_queue *q; 817 818 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 819 q->truesize = truesize; 820 821 type = truesize ? LIBETH_FQE_SHORT : LIBETH_FQE_MTU; 822 823 err = idpf_rx_bufs_init(q, type); 824 if (err) 825 return err; 826 827 truesize = q->truesize >> 1; 828 } 829 } 830 831 return 0; 832 } 833 834 /** 835 * idpf_rx_desc_alloc - Allocate queue Rx resources 836 * @vport: vport to allocate resources for 837 * @rxq: Rx queue for which the resources are setup 838 * 839 * Returns 0 on success, negative on failure 840 */ 841 static int idpf_rx_desc_alloc(const struct idpf_vport *vport, 842 struct idpf_rx_queue *rxq) 843 { 844 struct device *dev = &vport->adapter->pdev->dev; 845 846 rxq->size = rxq->desc_count * sizeof(union virtchnl2_rx_desc); 847 848 /* Allocate descriptors and also round up to nearest 4K */ 849 rxq->size = ALIGN(rxq->size, 4096); 850 rxq->desc_ring = dmam_alloc_coherent(dev, rxq->size, 851 &rxq->dma, GFP_KERNEL); 852 if (!rxq->desc_ring) { 853 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 854 rxq->size); 855 return -ENOMEM; 856 } 857 858 rxq->next_to_alloc = 0; 859 rxq->next_to_clean = 0; 860 rxq->next_to_use = 0; 861 idpf_queue_set(GEN_CHK, rxq); 862 863 idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX); 864 865 return 0; 866 } 867 868 /** 869 * idpf_bufq_desc_alloc - Allocate buffer queue descriptor ring 870 * @vport: vport to allocate resources for 871 * @bufq: buffer queue for which the resources are set up 872 * 873 * Return: 0 on success, -ENOMEM on failure. 874 */ 875 static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, 876 struct idpf_buf_queue *bufq) 877 { 878 struct device *dev = &vport->adapter->pdev->dev; 879 880 bufq->size = array_size(bufq->desc_count, sizeof(*bufq->split_buf)); 881 882 bufq->split_buf = dma_alloc_coherent(dev, bufq->size, &bufq->dma, 883 GFP_KERNEL); 884 if (!bufq->split_buf) 885 return -ENOMEM; 886 887 bufq->next_to_alloc = 0; 888 bufq->next_to_clean = 0; 889 bufq->next_to_use = 0; 890 idpf_queue_set(GEN_CHK, bufq); 891 892 idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 893 894 return 0; 895 } 896 897 /** 898 * idpf_rx_desc_alloc_all - allocate all RX queues resources 899 * @vport: virtual port structure 900 * 901 * Returns 0 on success, negative on failure 902 */ 903 static int idpf_rx_desc_alloc_all(struct idpf_vport *vport) 904 { 905 struct idpf_rxq_group *rx_qgrp; 906 int i, j, err; 907 u16 num_rxq; 908 909 for (i = 0; i < vport->num_rxq_grp; i++) { 910 rx_qgrp = &vport->rxq_grps[i]; 911 if (idpf_is_queue_model_split(vport->rxq_model)) 912 num_rxq = rx_qgrp->splitq.num_rxq_sets; 913 else 914 num_rxq = rx_qgrp->singleq.num_rxq; 915 916 for (j = 0; j < num_rxq; j++) { 917 struct idpf_rx_queue *q; 918 919 if (idpf_is_queue_model_split(vport->rxq_model)) 920 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 921 else 922 q = rx_qgrp->singleq.rxqs[j]; 923 924 err = idpf_rx_desc_alloc(vport, q); 925 if (err) { 926 pci_err(vport->adapter->pdev, 927 "Memory allocation for Rx queue %u from queue group %u failed\n", 928 j, i); 929 goto err_out; 930 } 931 } 932 933 if (!idpf_is_queue_model_split(vport->rxq_model)) 934 continue; 935 936 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) { 937 struct idpf_buf_queue *q; 938 939 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 940 941 err = idpf_bufq_desc_alloc(vport, q); 942 if (err) { 943 pci_err(vport->adapter->pdev, 944 "Memory allocation for Rx Buffer Queue %u from queue group %u failed\n", 945 j, i); 946 goto err_out; 947 } 948 } 949 } 950 951 return 0; 952 953 err_out: 954 idpf_rx_desc_rel_all(vport); 955 956 return err; 957 } 958 959 static int idpf_init_queue_set(const struct idpf_queue_set *qs) 960 { 961 const struct idpf_vport *vport = qs->vport; 962 bool splitq; 963 int err; 964 965 splitq = idpf_is_queue_model_split(vport->rxq_model); 966 967 for (u32 i = 0; i < qs->num; i++) { 968 const struct idpf_queue_ptr *q = &qs->qs[i]; 969 struct idpf_buf_queue *bufq; 970 971 switch (q->type) { 972 case VIRTCHNL2_QUEUE_TYPE_RX: 973 err = idpf_rx_desc_alloc(vport, q->rxq); 974 if (err) 975 break; 976 977 err = idpf_xdp_rxq_info_init(q->rxq); 978 if (err) 979 break; 980 981 if (!splitq) 982 err = idpf_rx_bufs_init_singleq(q->rxq); 983 984 break; 985 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 986 bufq = q->bufq; 987 988 err = idpf_bufq_desc_alloc(vport, bufq); 989 if (err) 990 break; 991 992 for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) { 993 struct idpf_buf_queue * const *bufqs; 994 enum libeth_fqe_type type; 995 u32 ts; 996 997 bufqs = bufq->q_vector->bufq; 998 if (bufqs[j] != bufq) 999 continue; 1000 1001 if (j) { 1002 type = LIBETH_FQE_SHORT; 1003 ts = bufqs[j - 1]->truesize >> 1; 1004 } else { 1005 type = LIBETH_FQE_MTU; 1006 ts = 0; 1007 } 1008 1009 bufq->truesize = ts; 1010 1011 err = idpf_rx_bufs_init(bufq, type); 1012 break; 1013 } 1014 1015 break; 1016 case VIRTCHNL2_QUEUE_TYPE_TX: 1017 err = idpf_tx_desc_alloc(vport, q->txq); 1018 break; 1019 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1020 err = idpf_compl_desc_alloc(vport, q->complq); 1021 break; 1022 default: 1023 continue; 1024 } 1025 1026 if (err) 1027 return err; 1028 } 1029 1030 return 0; 1031 } 1032 1033 static void idpf_clean_queue_set(const struct idpf_queue_set *qs) 1034 { 1035 const struct idpf_vport *vport = qs->vport; 1036 struct device *dev = vport->netdev->dev.parent; 1037 1038 for (u32 i = 0; i < qs->num; i++) { 1039 const struct idpf_queue_ptr *q = &qs->qs[i]; 1040 1041 switch (q->type) { 1042 case VIRTCHNL2_QUEUE_TYPE_RX: 1043 idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model); 1044 idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model); 1045 break; 1046 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1047 idpf_rx_desc_rel_bufq(q->bufq, dev); 1048 break; 1049 case VIRTCHNL2_QUEUE_TYPE_TX: 1050 idpf_tx_desc_rel(q->txq); 1051 1052 if (idpf_queue_has(XDP, q->txq)) { 1053 q->txq->pending = 0; 1054 q->txq->xdp_tx = 0; 1055 } else { 1056 q->txq->txq_grp->num_completions_pending = 0; 1057 } 1058 1059 writel(q->txq->next_to_use, q->txq->tail); 1060 break; 1061 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1062 idpf_compl_desc_rel(q->complq); 1063 q->complq->num_completions = 0; 1064 break; 1065 default: 1066 break; 1067 } 1068 } 1069 } 1070 1071 static void idpf_qvec_ena_irq(struct idpf_q_vector *qv) 1072 { 1073 if (qv->num_txq) { 1074 u32 itr; 1075 1076 if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode)) 1077 itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix]; 1078 else 1079 itr = qv->tx_itr_value; 1080 1081 idpf_vport_intr_write_itr(qv, itr, true); 1082 } 1083 1084 if (qv->num_rxq) { 1085 u32 itr; 1086 1087 if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode)) 1088 itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix]; 1089 else 1090 itr = qv->rx_itr_value; 1091 1092 idpf_vport_intr_write_itr(qv, itr, false); 1093 } 1094 1095 if (qv->num_txq || qv->num_rxq) 1096 idpf_vport_intr_update_itr_ena_irq(qv); 1097 } 1098 1099 /** 1100 * idpf_vector_to_queue_set - create a queue set associated with the given 1101 * queue vector 1102 * @qv: queue vector corresponding to the queue pair 1103 * 1104 * Returns a pointer to a dynamically allocated array of pointers to all 1105 * queues associated with a given queue vector (@qv). 1106 * Please note that the caller is responsible to free the memory allocated 1107 * by this function using kfree(). 1108 * 1109 * Return: &idpf_queue_set on success, %NULL in case of error. 1110 */ 1111 static struct idpf_queue_set * 1112 idpf_vector_to_queue_set(struct idpf_q_vector *qv) 1113 { 1114 bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq; 1115 struct idpf_vport *vport = qv->vport; 1116 struct idpf_queue_set *qs; 1117 u32 num; 1118 1119 num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq; 1120 num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2; 1121 if (!num) 1122 return NULL; 1123 1124 qs = idpf_alloc_queue_set(vport, num); 1125 if (!qs) 1126 return NULL; 1127 1128 num = 0; 1129 1130 for (u32 i = 0; i < qv->num_bufq; i++) { 1131 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1132 qs->qs[num++].bufq = qv->bufq[i]; 1133 } 1134 1135 for (u32 i = 0; i < qv->num_rxq; i++) { 1136 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX; 1137 qs->qs[num++].rxq = qv->rx[i]; 1138 } 1139 1140 for (u32 i = 0; i < qv->num_txq; i++) { 1141 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1142 qs->qs[num++].txq = qv->tx[i]; 1143 } 1144 1145 for (u32 i = 0; i < qv->num_complq; i++) { 1146 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1147 qs->qs[num++].complq = qv->complq[i]; 1148 } 1149 1150 if (!vport->xdp_txq_offset) 1151 goto finalize; 1152 1153 if (xdp) { 1154 for (u32 i = 0; i < qv->num_rxq; i++) { 1155 u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx; 1156 1157 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1158 qs->qs[num++].txq = vport->txqs[idx]; 1159 1160 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1161 qs->qs[num++].complq = vport->txqs[idx]->complq; 1162 } 1163 } else { 1164 for (u32 i = 0; i < qv->num_xsksq; i++) { 1165 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1166 qs->qs[num++].txq = qv->xsksq[i]; 1167 1168 qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1169 qs->qs[num++].complq = qv->xsksq[i]->complq; 1170 } 1171 } 1172 1173 finalize: 1174 if (num != qs->num) { 1175 kfree(qs); 1176 return NULL; 1177 } 1178 1179 return qs; 1180 } 1181 1182 static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid) 1183 { 1184 struct idpf_vport *vport = qs->vport; 1185 struct idpf_q_vector *q_vector; 1186 int err; 1187 1188 q_vector = idpf_find_rxq_vec(vport, qid); 1189 1190 err = idpf_init_queue_set(qs); 1191 if (err) { 1192 netdev_err(vport->netdev, "Could not initialize queues in pair %u: %pe\n", 1193 qid, ERR_PTR(err)); 1194 return err; 1195 } 1196 1197 if (!vport->xdp_txq_offset) 1198 goto config; 1199 1200 q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp, 1201 vport->num_q_vectors), 1202 sizeof(*q_vector->xsksq), GFP_KERNEL); 1203 if (!q_vector->xsksq) 1204 return -ENOMEM; 1205 1206 for (u32 i = 0; i < qs->num; i++) { 1207 const struct idpf_queue_ptr *q = &qs->qs[i]; 1208 1209 if (q->type != VIRTCHNL2_QUEUE_TYPE_TX) 1210 continue; 1211 1212 if (!idpf_queue_has(XSK, q->txq)) 1213 continue; 1214 1215 idpf_xsk_init_wakeup(q_vector); 1216 1217 q->txq->q_vector = q_vector; 1218 q_vector->xsksq[q_vector->num_xsksq++] = q->txq; 1219 } 1220 1221 config: 1222 err = idpf_send_config_queue_set_msg(qs); 1223 if (err) { 1224 netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n", 1225 qid, ERR_PTR(err)); 1226 return err; 1227 } 1228 1229 err = idpf_send_enable_queue_set_msg(qs); 1230 if (err) { 1231 netdev_err(vport->netdev, "Could not enable queues in pair %u: %pe\n", 1232 qid, ERR_PTR(err)); 1233 return err; 1234 } 1235 1236 napi_enable(&q_vector->napi); 1237 idpf_qvec_ena_irq(q_vector); 1238 1239 netif_start_subqueue(vport->netdev, qid); 1240 1241 return 0; 1242 } 1243 1244 static int idpf_qp_disable(const struct idpf_queue_set *qs, u32 qid) 1245 { 1246 struct idpf_vport *vport = qs->vport; 1247 struct idpf_q_vector *q_vector; 1248 int err; 1249 1250 q_vector = idpf_find_rxq_vec(vport, qid); 1251 netif_stop_subqueue(vport->netdev, qid); 1252 1253 writel(0, q_vector->intr_reg.dyn_ctl); 1254 napi_disable(&q_vector->napi); 1255 1256 err = idpf_send_disable_queue_set_msg(qs); 1257 if (err) { 1258 netdev_err(vport->netdev, "Could not disable queues in pair %u: %pe\n", 1259 qid, ERR_PTR(err)); 1260 return err; 1261 } 1262 1263 idpf_clean_queue_set(qs); 1264 1265 kfree(q_vector->xsksq); 1266 q_vector->num_xsksq = 0; 1267 1268 return 0; 1269 } 1270 1271 /** 1272 * idpf_qp_switch - enable or disable queues associated with queue pair 1273 * @vport: vport to switch the pair for 1274 * @qid: index of the queue pair to switch 1275 * @en: whether to enable or disable the pair 1276 * 1277 * Return: 0 on success, -errno on failure. 1278 */ 1279 int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en) 1280 { 1281 struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid); 1282 struct idpf_queue_set *qs __free(kfree) = NULL; 1283 1284 if (idpf_find_txq_vec(vport, qid) != q_vector) 1285 return -EINVAL; 1286 1287 qs = idpf_vector_to_queue_set(q_vector); 1288 if (!qs) 1289 return -ENOMEM; 1290 1291 return en ? idpf_qp_enable(qs, qid) : idpf_qp_disable(qs, qid); 1292 } 1293 1294 /** 1295 * idpf_txq_group_rel - Release all resources for txq groups 1296 * @vport: vport to release txq groups on 1297 */ 1298 static void idpf_txq_group_rel(struct idpf_vport *vport) 1299 { 1300 bool split, flow_sch_en; 1301 int i, j; 1302 1303 if (!vport->txq_grps) 1304 return; 1305 1306 split = idpf_is_queue_model_split(vport->txq_model); 1307 flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS, 1308 VIRTCHNL2_CAP_SPLITQ_QSCHED); 1309 1310 for (i = 0; i < vport->num_txq_grp; i++) { 1311 struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; 1312 1313 for (j = 0; j < txq_grp->num_txq; j++) { 1314 if (flow_sch_en) { 1315 kfree(txq_grp->txqs[j]->refillq); 1316 txq_grp->txqs[j]->refillq = NULL; 1317 } 1318 1319 kfree(txq_grp->txqs[j]); 1320 txq_grp->txqs[j] = NULL; 1321 } 1322 1323 if (!split) 1324 continue; 1325 1326 kfree(txq_grp->complq); 1327 txq_grp->complq = NULL; 1328 } 1329 kfree(vport->txq_grps); 1330 vport->txq_grps = NULL; 1331 } 1332 1333 /** 1334 * idpf_rxq_sw_queue_rel - Release software queue resources 1335 * @rx_qgrp: rx queue group with software queues 1336 */ 1337 static void idpf_rxq_sw_queue_rel(struct idpf_rxq_group *rx_qgrp) 1338 { 1339 int i, j; 1340 1341 for (i = 0; i < rx_qgrp->vport->num_bufqs_per_qgrp; i++) { 1342 struct idpf_bufq_set *bufq_set = &rx_qgrp->splitq.bufq_sets[i]; 1343 1344 for (j = 0; j < bufq_set->num_refillqs; j++) { 1345 kfree(bufq_set->refillqs[j].ring); 1346 bufq_set->refillqs[j].ring = NULL; 1347 } 1348 kfree(bufq_set->refillqs); 1349 bufq_set->refillqs = NULL; 1350 } 1351 } 1352 1353 /** 1354 * idpf_rxq_group_rel - Release all resources for rxq groups 1355 * @vport: vport to release rxq groups on 1356 */ 1357 static void idpf_rxq_group_rel(struct idpf_vport *vport) 1358 { 1359 int i; 1360 1361 if (!vport->rxq_grps) 1362 return; 1363 1364 for (i = 0; i < vport->num_rxq_grp; i++) { 1365 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 1366 u16 num_rxq; 1367 int j; 1368 1369 if (idpf_is_queue_model_split(vport->rxq_model)) { 1370 num_rxq = rx_qgrp->splitq.num_rxq_sets; 1371 for (j = 0; j < num_rxq; j++) { 1372 kfree(rx_qgrp->splitq.rxq_sets[j]); 1373 rx_qgrp->splitq.rxq_sets[j] = NULL; 1374 } 1375 1376 idpf_rxq_sw_queue_rel(rx_qgrp); 1377 kfree(rx_qgrp->splitq.bufq_sets); 1378 rx_qgrp->splitq.bufq_sets = NULL; 1379 } else { 1380 num_rxq = rx_qgrp->singleq.num_rxq; 1381 for (j = 0; j < num_rxq; j++) { 1382 kfree(rx_qgrp->singleq.rxqs[j]); 1383 rx_qgrp->singleq.rxqs[j] = NULL; 1384 } 1385 } 1386 } 1387 kfree(vport->rxq_grps); 1388 vport->rxq_grps = NULL; 1389 } 1390 1391 /** 1392 * idpf_vport_queue_grp_rel_all - Release all queue groups 1393 * @vport: vport to release queue groups for 1394 */ 1395 static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport) 1396 { 1397 idpf_txq_group_rel(vport); 1398 idpf_rxq_group_rel(vport); 1399 } 1400 1401 /** 1402 * idpf_vport_queues_rel - Free memory for all queues 1403 * @vport: virtual port 1404 * 1405 * Free the memory allocated for queues associated to a vport 1406 */ 1407 void idpf_vport_queues_rel(struct idpf_vport *vport) 1408 { 1409 idpf_xdp_copy_prog_to_rqs(vport, NULL); 1410 1411 idpf_tx_desc_rel_all(vport); 1412 idpf_rx_desc_rel_all(vport); 1413 1414 idpf_xdpsqs_put(vport); 1415 idpf_vport_queue_grp_rel_all(vport); 1416 1417 kfree(vport->txqs); 1418 vport->txqs = NULL; 1419 } 1420 1421 /** 1422 * idpf_vport_init_fast_path_txqs - Initialize fast path txq array 1423 * @vport: vport to init txqs on 1424 * 1425 * We get a queue index from skb->queue_mapping and we need a fast way to 1426 * dereference the queue from queue groups. This allows us to quickly pull a 1427 * txq based on a queue index. 1428 * 1429 * Returns 0 on success, negative on failure 1430 */ 1431 static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport) 1432 { 1433 struct idpf_ptp_vport_tx_tstamp_caps *caps = vport->tx_tstamp_caps; 1434 struct work_struct *tstamp_task = &vport->tstamp_task; 1435 int i, j, k = 0; 1436 1437 vport->txqs = kcalloc(vport->num_txq, sizeof(*vport->txqs), 1438 GFP_KERNEL); 1439 1440 if (!vport->txqs) 1441 return -ENOMEM; 1442 1443 for (i = 0; i < vport->num_txq_grp; i++) { 1444 struct idpf_txq_group *tx_grp = &vport->txq_grps[i]; 1445 1446 for (j = 0; j < tx_grp->num_txq; j++, k++) { 1447 vport->txqs[k] = tx_grp->txqs[j]; 1448 vport->txqs[k]->idx = k; 1449 1450 if (!caps) 1451 continue; 1452 1453 vport->txqs[k]->cached_tstamp_caps = caps; 1454 vport->txqs[k]->tstamp_task = tstamp_task; 1455 } 1456 } 1457 1458 return 0; 1459 } 1460 1461 /** 1462 * idpf_vport_init_num_qs - Initialize number of queues 1463 * @vport: vport to initialize queues 1464 * @vport_msg: data to be filled into vport 1465 */ 1466 void idpf_vport_init_num_qs(struct idpf_vport *vport, 1467 struct virtchnl2_create_vport *vport_msg) 1468 { 1469 struct idpf_vport_user_config_data *config_data; 1470 u16 idx = vport->idx; 1471 1472 config_data = &vport->adapter->vport_config[idx]->user_config; 1473 vport->num_txq = le16_to_cpu(vport_msg->num_tx_q); 1474 vport->num_rxq = le16_to_cpu(vport_msg->num_rx_q); 1475 /* number of txqs and rxqs in config data will be zeros only in the 1476 * driver load path and we dont update them there after 1477 */ 1478 if (!config_data->num_req_tx_qs && !config_data->num_req_rx_qs) { 1479 config_data->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q); 1480 config_data->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q); 1481 } 1482 1483 if (idpf_is_queue_model_split(vport->txq_model)) 1484 vport->num_complq = le16_to_cpu(vport_msg->num_tx_complq); 1485 if (idpf_is_queue_model_split(vport->rxq_model)) 1486 vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq); 1487 1488 vport->xdp_prog = config_data->xdp_prog; 1489 if (idpf_xdp_enabled(vport)) { 1490 vport->xdp_txq_offset = config_data->num_req_tx_qs; 1491 vport->num_xdp_txq = le16_to_cpu(vport_msg->num_tx_q) - 1492 vport->xdp_txq_offset; 1493 vport->xdpsq_share = libeth_xdpsq_shared(vport->num_xdp_txq); 1494 } else { 1495 vport->xdp_txq_offset = 0; 1496 vport->num_xdp_txq = 0; 1497 vport->xdpsq_share = false; 1498 } 1499 1500 /* Adjust number of buffer queues per Rx queue group. */ 1501 if (!idpf_is_queue_model_split(vport->rxq_model)) { 1502 vport->num_bufqs_per_qgrp = 0; 1503 1504 return; 1505 } 1506 1507 vport->num_bufqs_per_qgrp = IDPF_MAX_BUFQS_PER_RXQ_GRP; 1508 } 1509 1510 /** 1511 * idpf_vport_calc_num_q_desc - Calculate number of queue groups 1512 * @vport: vport to calculate q groups for 1513 */ 1514 void idpf_vport_calc_num_q_desc(struct idpf_vport *vport) 1515 { 1516 struct idpf_vport_user_config_data *config_data; 1517 int num_bufqs = vport->num_bufqs_per_qgrp; 1518 u32 num_req_txq_desc, num_req_rxq_desc; 1519 u16 idx = vport->idx; 1520 int i; 1521 1522 config_data = &vport->adapter->vport_config[idx]->user_config; 1523 num_req_txq_desc = config_data->num_req_txq_desc; 1524 num_req_rxq_desc = config_data->num_req_rxq_desc; 1525 1526 vport->complq_desc_count = 0; 1527 if (num_req_txq_desc) { 1528 vport->txq_desc_count = num_req_txq_desc; 1529 if (idpf_is_queue_model_split(vport->txq_model)) { 1530 vport->complq_desc_count = num_req_txq_desc; 1531 if (vport->complq_desc_count < IDPF_MIN_TXQ_COMPLQ_DESC) 1532 vport->complq_desc_count = 1533 IDPF_MIN_TXQ_COMPLQ_DESC; 1534 } 1535 } else { 1536 vport->txq_desc_count = IDPF_DFLT_TX_Q_DESC_COUNT; 1537 if (idpf_is_queue_model_split(vport->txq_model)) 1538 vport->complq_desc_count = 1539 IDPF_DFLT_TX_COMPLQ_DESC_COUNT; 1540 } 1541 1542 if (num_req_rxq_desc) 1543 vport->rxq_desc_count = num_req_rxq_desc; 1544 else 1545 vport->rxq_desc_count = IDPF_DFLT_RX_Q_DESC_COUNT; 1546 1547 for (i = 0; i < num_bufqs; i++) { 1548 if (!vport->bufq_desc_count[i]) 1549 vport->bufq_desc_count[i] = 1550 IDPF_RX_BUFQ_DESC_COUNT(vport->rxq_desc_count, 1551 num_bufqs); 1552 } 1553 } 1554 1555 /** 1556 * idpf_vport_calc_total_qs - Calculate total number of queues 1557 * @adapter: private data struct 1558 * @vport_idx: vport idx to retrieve vport pointer 1559 * @vport_msg: message to fill with data 1560 * @max_q: vport max queue info 1561 * 1562 * Return 0 on success, error value on failure. 1563 */ 1564 int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, 1565 struct virtchnl2_create_vport *vport_msg, 1566 struct idpf_vport_max_q *max_q) 1567 { 1568 int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0; 1569 int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0; 1570 u16 num_req_tx_qs = 0, num_req_rx_qs = 0; 1571 struct idpf_vport_user_config_data *user; 1572 struct idpf_vport_config *vport_config; 1573 u16 num_txq_grps, num_rxq_grps; 1574 u32 num_qs, num_xdpsq; 1575 1576 vport_config = adapter->vport_config[vport_idx]; 1577 if (vport_config) { 1578 num_req_tx_qs = vport_config->user_config.num_req_tx_qs; 1579 num_req_rx_qs = vport_config->user_config.num_req_rx_qs; 1580 } else { 1581 u32 num_cpus = netif_get_num_default_rss_queues(); 1582 1583 dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus); 1584 dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus); 1585 dflt_splitq_rxq_grps = min_t(int, max_q->max_rxq, num_cpus); 1586 dflt_singleq_rxqs = min_t(int, max_q->max_rxq, num_cpus); 1587 } 1588 1589 if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) { 1590 num_txq_grps = num_req_tx_qs ? num_req_tx_qs : dflt_splitq_txq_grps; 1591 vport_msg->num_tx_complq = cpu_to_le16(num_txq_grps * 1592 IDPF_COMPLQ_PER_GROUP); 1593 vport_msg->num_tx_q = cpu_to_le16(num_txq_grps * 1594 IDPF_DFLT_SPLITQ_TXQ_PER_GROUP); 1595 } else { 1596 num_txq_grps = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS; 1597 num_qs = num_txq_grps * (num_req_tx_qs ? num_req_tx_qs : 1598 dflt_singleq_txqs); 1599 vport_msg->num_tx_q = cpu_to_le16(num_qs); 1600 vport_msg->num_tx_complq = 0; 1601 } 1602 if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->rxq_model))) { 1603 num_rxq_grps = num_req_rx_qs ? num_req_rx_qs : dflt_splitq_rxq_grps; 1604 vport_msg->num_rx_bufq = cpu_to_le16(num_rxq_grps * 1605 IDPF_MAX_BUFQS_PER_RXQ_GRP); 1606 vport_msg->num_rx_q = cpu_to_le16(num_rxq_grps * 1607 IDPF_DFLT_SPLITQ_RXQ_PER_GROUP); 1608 } else { 1609 num_rxq_grps = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS; 1610 num_qs = num_rxq_grps * (num_req_rx_qs ? num_req_rx_qs : 1611 dflt_singleq_rxqs); 1612 vport_msg->num_rx_q = cpu_to_le16(num_qs); 1613 vport_msg->num_rx_bufq = 0; 1614 } 1615 1616 if (!vport_config) 1617 return 0; 1618 1619 user = &vport_config->user_config; 1620 user->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q); 1621 user->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q); 1622 1623 if (vport_config->user_config.xdp_prog) 1624 num_xdpsq = libeth_xdpsq_num(user->num_req_rx_qs, 1625 user->num_req_tx_qs, 1626 vport_config->max_q.max_txq); 1627 else 1628 num_xdpsq = 0; 1629 1630 vport_msg->num_tx_q = cpu_to_le16(user->num_req_tx_qs + num_xdpsq); 1631 if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) 1632 vport_msg->num_tx_complq = vport_msg->num_tx_q; 1633 1634 return 0; 1635 } 1636 1637 /** 1638 * idpf_vport_calc_num_q_groups - Calculate number of queue groups 1639 * @vport: vport to calculate q groups for 1640 */ 1641 void idpf_vport_calc_num_q_groups(struct idpf_vport *vport) 1642 { 1643 if (idpf_is_queue_model_split(vport->txq_model)) 1644 vport->num_txq_grp = vport->num_txq; 1645 else 1646 vport->num_txq_grp = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS; 1647 1648 if (idpf_is_queue_model_split(vport->rxq_model)) 1649 vport->num_rxq_grp = vport->num_rxq; 1650 else 1651 vport->num_rxq_grp = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS; 1652 } 1653 1654 /** 1655 * idpf_vport_calc_numq_per_grp - Calculate number of queues per group 1656 * @vport: vport to calculate queues for 1657 * @num_txq: return parameter for number of TX queues 1658 * @num_rxq: return parameter for number of RX queues 1659 */ 1660 static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport, 1661 u16 *num_txq, u16 *num_rxq) 1662 { 1663 if (idpf_is_queue_model_split(vport->txq_model)) 1664 *num_txq = IDPF_DFLT_SPLITQ_TXQ_PER_GROUP; 1665 else 1666 *num_txq = vport->num_txq; 1667 1668 if (idpf_is_queue_model_split(vport->rxq_model)) 1669 *num_rxq = IDPF_DFLT_SPLITQ_RXQ_PER_GROUP; 1670 else 1671 *num_rxq = vport->num_rxq; 1672 } 1673 1674 /** 1675 * idpf_rxq_set_descids - set the descids supported by this queue 1676 * @vport: virtual port data structure 1677 * @q: rx queue for which descids are set 1678 * 1679 */ 1680 static void idpf_rxq_set_descids(const struct idpf_vport *vport, 1681 struct idpf_rx_queue *q) 1682 { 1683 if (idpf_is_queue_model_split(vport->rxq_model)) 1684 return; 1685 1686 if (vport->base_rxd) 1687 q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; 1688 else 1689 q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; 1690 } 1691 1692 /** 1693 * idpf_txq_group_alloc - Allocate all txq group resources 1694 * @vport: vport to allocate txq groups for 1695 * @num_txq: number of txqs to allocate for each group 1696 * 1697 * Returns 0 on success, negative on failure 1698 */ 1699 static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) 1700 { 1701 bool split, flow_sch_en; 1702 int i; 1703 1704 vport->txq_grps = kcalloc(vport->num_txq_grp, 1705 sizeof(*vport->txq_grps), GFP_KERNEL); 1706 if (!vport->txq_grps) 1707 return -ENOMEM; 1708 1709 split = idpf_is_queue_model_split(vport->txq_model); 1710 flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS, 1711 VIRTCHNL2_CAP_SPLITQ_QSCHED); 1712 1713 for (i = 0; i < vport->num_txq_grp; i++) { 1714 struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 1715 struct idpf_adapter *adapter = vport->adapter; 1716 int j; 1717 1718 tx_qgrp->vport = vport; 1719 tx_qgrp->num_txq = num_txq; 1720 1721 for (j = 0; j < tx_qgrp->num_txq; j++) { 1722 tx_qgrp->txqs[j] = kzalloc(sizeof(*tx_qgrp->txqs[j]), 1723 GFP_KERNEL); 1724 if (!tx_qgrp->txqs[j]) 1725 goto err_alloc; 1726 } 1727 1728 for (j = 0; j < tx_qgrp->num_txq; j++) { 1729 struct idpf_tx_queue *q = tx_qgrp->txqs[j]; 1730 1731 q->dev = &adapter->pdev->dev; 1732 q->desc_count = vport->txq_desc_count; 1733 q->tx_max_bufs = idpf_get_max_tx_bufs(adapter); 1734 q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); 1735 q->netdev = vport->netdev; 1736 q->txq_grp = tx_qgrp; 1737 q->rel_q_id = j; 1738 1739 if (!split) { 1740 q->clean_budget = vport->compln_clean_budget; 1741 idpf_queue_assign(CRC_EN, q, 1742 vport->crc_enable); 1743 } 1744 1745 if (!flow_sch_en) 1746 continue; 1747 1748 idpf_queue_set(FLOW_SCH_EN, q); 1749 1750 q->refillq = kzalloc(sizeof(*q->refillq), GFP_KERNEL); 1751 if (!q->refillq) 1752 goto err_alloc; 1753 1754 idpf_queue_set(GEN_CHK, q->refillq); 1755 idpf_queue_set(RFL_GEN_CHK, q->refillq); 1756 } 1757 1758 if (!split) 1759 continue; 1760 1761 tx_qgrp->complq = kcalloc(IDPF_COMPLQ_PER_GROUP, 1762 sizeof(*tx_qgrp->complq), 1763 GFP_KERNEL); 1764 if (!tx_qgrp->complq) 1765 goto err_alloc; 1766 1767 tx_qgrp->complq->desc_count = vport->complq_desc_count; 1768 tx_qgrp->complq->txq_grp = tx_qgrp; 1769 tx_qgrp->complq->netdev = vport->netdev; 1770 tx_qgrp->complq->clean_budget = vport->compln_clean_budget; 1771 1772 if (flow_sch_en) 1773 idpf_queue_set(FLOW_SCH_EN, tx_qgrp->complq); 1774 } 1775 1776 return 0; 1777 1778 err_alloc: 1779 idpf_txq_group_rel(vport); 1780 1781 return -ENOMEM; 1782 } 1783 1784 /** 1785 * idpf_rxq_group_alloc - Allocate all rxq group resources 1786 * @vport: vport to allocate rxq groups for 1787 * @num_rxq: number of rxqs to allocate for each group 1788 * 1789 * Returns 0 on success, negative on failure 1790 */ 1791 static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq) 1792 { 1793 int i, k, err = 0; 1794 bool hs; 1795 1796 vport->rxq_grps = kcalloc(vport->num_rxq_grp, 1797 sizeof(struct idpf_rxq_group), GFP_KERNEL); 1798 if (!vport->rxq_grps) 1799 return -ENOMEM; 1800 1801 hs = idpf_vport_get_hsplit(vport) == ETHTOOL_TCP_DATA_SPLIT_ENABLED; 1802 1803 for (i = 0; i < vport->num_rxq_grp; i++) { 1804 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 1805 int j; 1806 1807 rx_qgrp->vport = vport; 1808 if (!idpf_is_queue_model_split(vport->rxq_model)) { 1809 rx_qgrp->singleq.num_rxq = num_rxq; 1810 for (j = 0; j < num_rxq; j++) { 1811 rx_qgrp->singleq.rxqs[j] = 1812 kzalloc(sizeof(*rx_qgrp->singleq.rxqs[j]), 1813 GFP_KERNEL); 1814 if (!rx_qgrp->singleq.rxqs[j]) { 1815 err = -ENOMEM; 1816 goto err_alloc; 1817 } 1818 } 1819 goto skip_splitq_rx_init; 1820 } 1821 rx_qgrp->splitq.num_rxq_sets = num_rxq; 1822 1823 for (j = 0; j < num_rxq; j++) { 1824 rx_qgrp->splitq.rxq_sets[j] = 1825 kzalloc(sizeof(struct idpf_rxq_set), 1826 GFP_KERNEL); 1827 if (!rx_qgrp->splitq.rxq_sets[j]) { 1828 err = -ENOMEM; 1829 goto err_alloc; 1830 } 1831 } 1832 1833 rx_qgrp->splitq.bufq_sets = kcalloc(vport->num_bufqs_per_qgrp, 1834 sizeof(struct idpf_bufq_set), 1835 GFP_KERNEL); 1836 if (!rx_qgrp->splitq.bufq_sets) { 1837 err = -ENOMEM; 1838 goto err_alloc; 1839 } 1840 1841 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) { 1842 struct idpf_bufq_set *bufq_set = 1843 &rx_qgrp->splitq.bufq_sets[j]; 1844 int swq_size = sizeof(struct idpf_sw_queue); 1845 struct idpf_buf_queue *q; 1846 1847 q = &rx_qgrp->splitq.bufq_sets[j].bufq; 1848 q->desc_count = vport->bufq_desc_count[j]; 1849 q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK; 1850 1851 idpf_queue_assign(HSPLIT_EN, q, hs); 1852 1853 bufq_set->num_refillqs = num_rxq; 1854 bufq_set->refillqs = kcalloc(num_rxq, swq_size, 1855 GFP_KERNEL); 1856 if (!bufq_set->refillqs) { 1857 err = -ENOMEM; 1858 goto err_alloc; 1859 } 1860 for (k = 0; k < bufq_set->num_refillqs; k++) { 1861 struct idpf_sw_queue *refillq = 1862 &bufq_set->refillqs[k]; 1863 1864 refillq->desc_count = 1865 vport->bufq_desc_count[j]; 1866 idpf_queue_set(GEN_CHK, refillq); 1867 idpf_queue_set(RFL_GEN_CHK, refillq); 1868 refillq->ring = kcalloc(refillq->desc_count, 1869 sizeof(*refillq->ring), 1870 GFP_KERNEL); 1871 if (!refillq->ring) { 1872 err = -ENOMEM; 1873 goto err_alloc; 1874 } 1875 } 1876 } 1877 1878 skip_splitq_rx_init: 1879 for (j = 0; j < num_rxq; j++) { 1880 struct idpf_rx_queue *q; 1881 1882 if (!idpf_is_queue_model_split(vport->rxq_model)) { 1883 q = rx_qgrp->singleq.rxqs[j]; 1884 goto setup_rxq; 1885 } 1886 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 1887 rx_qgrp->splitq.rxq_sets[j]->refillq[0] = 1888 &rx_qgrp->splitq.bufq_sets[0].refillqs[j]; 1889 if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) 1890 rx_qgrp->splitq.rxq_sets[j]->refillq[1] = 1891 &rx_qgrp->splitq.bufq_sets[1].refillqs[j]; 1892 1893 idpf_queue_assign(HSPLIT_EN, q, hs); 1894 1895 setup_rxq: 1896 q->desc_count = vport->rxq_desc_count; 1897 q->rx_ptype_lkup = vport->rx_ptype_lkup; 1898 q->bufq_sets = rx_qgrp->splitq.bufq_sets; 1899 q->idx = (i * num_rxq) + j; 1900 q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK; 1901 q->rx_max_pkt_size = vport->netdev->mtu + 1902 LIBETH_RX_LL_LEN; 1903 idpf_rxq_set_descids(vport, q); 1904 } 1905 } 1906 1907 err_alloc: 1908 if (err) 1909 idpf_rxq_group_rel(vport); 1910 1911 return err; 1912 } 1913 1914 /** 1915 * idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources 1916 * @vport: vport with qgrps to allocate 1917 * 1918 * Returns 0 on success, negative on failure 1919 */ 1920 static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport) 1921 { 1922 u16 num_txq, num_rxq; 1923 int err; 1924 1925 idpf_vport_calc_numq_per_grp(vport, &num_txq, &num_rxq); 1926 1927 err = idpf_txq_group_alloc(vport, num_txq); 1928 if (err) 1929 goto err_out; 1930 1931 err = idpf_rxq_group_alloc(vport, num_rxq); 1932 if (err) 1933 goto err_out; 1934 1935 return 0; 1936 1937 err_out: 1938 idpf_vport_queue_grp_rel_all(vport); 1939 1940 return err; 1941 } 1942 1943 /** 1944 * idpf_vport_queues_alloc - Allocate memory for all queues 1945 * @vport: virtual port 1946 * 1947 * Allocate memory for queues associated with a vport. Returns 0 on success, 1948 * negative on failure. 1949 */ 1950 int idpf_vport_queues_alloc(struct idpf_vport *vport) 1951 { 1952 int err; 1953 1954 err = idpf_vport_queue_grp_alloc_all(vport); 1955 if (err) 1956 goto err_out; 1957 1958 err = idpf_vport_init_fast_path_txqs(vport); 1959 if (err) 1960 goto err_out; 1961 1962 err = idpf_xdpsqs_get(vport); 1963 if (err) 1964 goto err_out; 1965 1966 err = idpf_tx_desc_alloc_all(vport); 1967 if (err) 1968 goto err_out; 1969 1970 err = idpf_rx_desc_alloc_all(vport); 1971 if (err) 1972 goto err_out; 1973 1974 return 0; 1975 1976 err_out: 1977 idpf_vport_queues_rel(vport); 1978 1979 return err; 1980 } 1981 1982 /** 1983 * idpf_tx_read_tstamp - schedule a work to read Tx timestamp value 1984 * @txq: queue to read the timestamp from 1985 * @skb: socket buffer to provide Tx timestamp value 1986 * 1987 * Schedule a work to read Tx timestamp value generated once the packet is 1988 * transmitted. 1989 */ 1990 static void idpf_tx_read_tstamp(struct idpf_tx_queue *txq, struct sk_buff *skb) 1991 { 1992 struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps; 1993 struct idpf_ptp_tx_tstamp_status *tx_tstamp_status; 1994 1995 tx_tstamp_caps = txq->cached_tstamp_caps; 1996 spin_lock_bh(&tx_tstamp_caps->status_lock); 1997 1998 for (u32 i = 0; i < tx_tstamp_caps->num_entries; i++) { 1999 tx_tstamp_status = &tx_tstamp_caps->tx_tstamp_status[i]; 2000 if (tx_tstamp_status->state != IDPF_PTP_FREE) 2001 continue; 2002 2003 tx_tstamp_status->skb = skb; 2004 tx_tstamp_status->state = IDPF_PTP_REQUEST; 2005 2006 /* Fetch timestamp from completion descriptor through 2007 * virtchnl msg to report to stack. 2008 */ 2009 queue_work(system_unbound_wq, txq->tstamp_task); 2010 break; 2011 } 2012 2013 spin_unlock_bh(&tx_tstamp_caps->status_lock); 2014 } 2015 2016 #define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \ 2017 do { \ 2018 if (unlikely(++(ntc) == (txq)->desc_count)) { \ 2019 ntc = 0; \ 2020 buf = (txq)->tx_buf; \ 2021 desc = &(txq)->flex_tx[0]; \ 2022 } else { \ 2023 (buf)++; \ 2024 (desc)++; \ 2025 } \ 2026 } while (0) 2027 2028 /** 2029 * idpf_tx_splitq_clean - Reclaim resources from buffer queue 2030 * @tx_q: Tx queue to clean 2031 * @end: queue index until which it should be cleaned 2032 * @napi_budget: Used to determine if we are in netpoll 2033 * @cleaned: pointer to stats struct to track cleaned packets/bytes 2034 * @descs_only: true if queue is using flow-based scheduling and should 2035 * not clean buffers at this time 2036 * 2037 * Cleans the queue descriptor ring. If the queue is using queue-based 2038 * scheduling, the buffers will be cleaned as well. If the queue is using 2039 * flow-based scheduling, only the descriptors are cleaned at this time. 2040 * Separate packet completion events will be reported on the completion queue, 2041 * and the buffers will be cleaned separately. The stats are not updated from 2042 * this function when using flow-based scheduling. 2043 */ 2044 static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, 2045 int napi_budget, 2046 struct libeth_sq_napi_stats *cleaned, 2047 bool descs_only) 2048 { 2049 union idpf_tx_flex_desc *next_pending_desc = NULL; 2050 union idpf_tx_flex_desc *tx_desc; 2051 u32 ntc = tx_q->next_to_clean; 2052 struct libeth_cq_pp cp = { 2053 .dev = tx_q->dev, 2054 .ss = cleaned, 2055 .napi = napi_budget, 2056 }; 2057 struct idpf_tx_buf *tx_buf; 2058 2059 if (descs_only) { 2060 /* Bump ring index to mark as cleaned. */ 2061 tx_q->next_to_clean = end; 2062 return; 2063 } 2064 2065 tx_desc = &tx_q->flex_tx[ntc]; 2066 next_pending_desc = &tx_q->flex_tx[end]; 2067 tx_buf = &tx_q->tx_buf[ntc]; 2068 2069 while (tx_desc != next_pending_desc) { 2070 u32 eop_idx; 2071 2072 /* If this entry in the ring was used as a context descriptor, 2073 * it's corresponding entry in the buffer ring is reserved. We 2074 * can skip this descriptor since there is no buffer to clean. 2075 */ 2076 if (tx_buf->type <= LIBETH_SQE_CTX) 2077 goto fetch_next_txq_desc; 2078 2079 if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) 2080 break; 2081 2082 eop_idx = tx_buf->rs_idx; 2083 libeth_tx_complete(tx_buf, &cp); 2084 2085 /* unmap remaining buffers */ 2086 while (ntc != eop_idx) { 2087 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, 2088 tx_desc, tx_buf); 2089 2090 /* unmap any remaining paged data */ 2091 libeth_tx_complete(tx_buf, &cp); 2092 } 2093 2094 fetch_next_txq_desc: 2095 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf); 2096 } 2097 2098 tx_q->next_to_clean = ntc; 2099 } 2100 2101 /** 2102 * idpf_tx_clean_bufs - clean flow scheduling TX queue buffers 2103 * @txq: queue to clean 2104 * @buf_id: packet's starting buffer ID, from completion descriptor 2105 * @cleaned: pointer to stats struct to track cleaned packets/bytes 2106 * @budget: Used to determine if we are in netpoll 2107 * 2108 * Clean all buffers associated with the packet starting at buf_id. Returns the 2109 * byte/segment count for the cleaned packet. 2110 */ 2111 static void idpf_tx_clean_bufs(struct idpf_tx_queue *txq, u32 buf_id, 2112 struct libeth_sq_napi_stats *cleaned, 2113 int budget) 2114 { 2115 struct idpf_tx_buf *tx_buf = NULL; 2116 struct libeth_cq_pp cp = { 2117 .dev = txq->dev, 2118 .ss = cleaned, 2119 .napi = budget, 2120 }; 2121 2122 tx_buf = &txq->tx_buf[buf_id]; 2123 if (tx_buf->type == LIBETH_SQE_SKB) { 2124 if (skb_shinfo(tx_buf->skb)->tx_flags & SKBTX_IN_PROGRESS) 2125 idpf_tx_read_tstamp(txq, tx_buf->skb); 2126 2127 libeth_tx_complete(tx_buf, &cp); 2128 idpf_post_buf_refill(txq->refillq, buf_id); 2129 } 2130 2131 while (idpf_tx_buf_next(tx_buf) != IDPF_TXBUF_NULL) { 2132 buf_id = idpf_tx_buf_next(tx_buf); 2133 2134 tx_buf = &txq->tx_buf[buf_id]; 2135 libeth_tx_complete(tx_buf, &cp); 2136 idpf_post_buf_refill(txq->refillq, buf_id); 2137 } 2138 } 2139 2140 /** 2141 * idpf_tx_handle_rs_completion - clean a single packet and all of its buffers 2142 * whether on the buffer ring or in the hash table 2143 * @txq: Tx ring to clean 2144 * @desc: pointer to completion queue descriptor to extract completion 2145 * information from 2146 * @cleaned: pointer to stats struct to track cleaned packets/bytes 2147 * @budget: Used to determine if we are in netpoll 2148 * 2149 * Returns bytes/packets cleaned 2150 */ 2151 static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, 2152 struct idpf_splitq_tx_compl_desc *desc, 2153 struct libeth_sq_napi_stats *cleaned, 2154 int budget) 2155 { 2156 /* RS completion contains queue head for queue based scheduling or 2157 * completion tag for flow based scheduling. 2158 */ 2159 u16 rs_compl_val = le16_to_cpu(desc->common.q_head_compl_tag.q_head); 2160 2161 if (!idpf_queue_has(FLOW_SCH_EN, txq)) { 2162 idpf_tx_splitq_clean(txq, rs_compl_val, budget, cleaned, false); 2163 return; 2164 } 2165 2166 idpf_tx_clean_bufs(txq, rs_compl_val, cleaned, budget); 2167 } 2168 2169 /** 2170 * idpf_tx_clean_complq - Reclaim resources on completion queue 2171 * @complq: Tx ring to clean 2172 * @budget: Used to determine if we are in netpoll 2173 * @cleaned: returns number of packets cleaned 2174 * 2175 * Returns true if there's any budget left (e.g. the clean is finished) 2176 */ 2177 static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, 2178 int *cleaned) 2179 { 2180 struct idpf_splitq_tx_compl_desc *tx_desc; 2181 s16 ntc = complq->next_to_clean; 2182 struct idpf_netdev_priv *np; 2183 unsigned int complq_budget; 2184 bool complq_ok = true; 2185 int i; 2186 2187 complq_budget = complq->clean_budget; 2188 tx_desc = &complq->comp[ntc]; 2189 ntc -= complq->desc_count; 2190 2191 do { 2192 struct libeth_sq_napi_stats cleaned_stats = { }; 2193 struct idpf_tx_queue *tx_q; 2194 __le16 hw_head; 2195 int rel_tx_qid; 2196 u8 ctype; /* completion type */ 2197 u16 gen; 2198 2199 /* if the descriptor isn't done, no work yet to do */ 2200 gen = le16_get_bits(tx_desc->common.qid_comptype_gen, 2201 IDPF_TXD_COMPLQ_GEN_M); 2202 if (idpf_queue_has(GEN_CHK, complq) != gen) 2203 break; 2204 2205 /* Find necessary info of TX queue to clean buffers */ 2206 rel_tx_qid = le16_get_bits(tx_desc->common.qid_comptype_gen, 2207 IDPF_TXD_COMPLQ_QID_M); 2208 if (rel_tx_qid >= complq->txq_grp->num_txq || 2209 !complq->txq_grp->txqs[rel_tx_qid]) { 2210 netdev_err(complq->netdev, "TxQ not found\n"); 2211 goto fetch_next_desc; 2212 } 2213 tx_q = complq->txq_grp->txqs[rel_tx_qid]; 2214 2215 /* Determine completion type */ 2216 ctype = le16_get_bits(tx_desc->common.qid_comptype_gen, 2217 IDPF_TXD_COMPLQ_COMPL_TYPE_M); 2218 switch (ctype) { 2219 case IDPF_TXD_COMPLT_RE: 2220 hw_head = tx_desc->common.q_head_compl_tag.q_head; 2221 2222 idpf_tx_splitq_clean(tx_q, le16_to_cpu(hw_head), 2223 budget, &cleaned_stats, true); 2224 break; 2225 case IDPF_TXD_COMPLT_RS: 2226 idpf_tx_handle_rs_completion(tx_q, tx_desc, 2227 &cleaned_stats, budget); 2228 break; 2229 default: 2230 netdev_err(tx_q->netdev, 2231 "Unknown TX completion type: %d\n", ctype); 2232 goto fetch_next_desc; 2233 } 2234 2235 u64_stats_update_begin(&tx_q->stats_sync); 2236 u64_stats_add(&tx_q->q_stats.packets, cleaned_stats.packets); 2237 u64_stats_add(&tx_q->q_stats.bytes, cleaned_stats.bytes); 2238 tx_q->cleaned_pkts += cleaned_stats.packets; 2239 tx_q->cleaned_bytes += cleaned_stats.bytes; 2240 complq->num_completions++; 2241 u64_stats_update_end(&tx_q->stats_sync); 2242 2243 fetch_next_desc: 2244 tx_desc++; 2245 ntc++; 2246 if (unlikely(!ntc)) { 2247 ntc -= complq->desc_count; 2248 tx_desc = &complq->comp[0]; 2249 idpf_queue_change(GEN_CHK, complq); 2250 } 2251 2252 prefetch(tx_desc); 2253 2254 /* update budget accounting */ 2255 complq_budget--; 2256 } while (likely(complq_budget)); 2257 2258 /* Store the state of the complq to be used later in deciding if a 2259 * TXQ can be started again 2260 */ 2261 if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) > 2262 IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq))) 2263 complq_ok = false; 2264 2265 np = netdev_priv(complq->netdev); 2266 for (i = 0; i < complq->txq_grp->num_txq; ++i) { 2267 struct idpf_tx_queue *tx_q = complq->txq_grp->txqs[i]; 2268 struct netdev_queue *nq; 2269 bool dont_wake; 2270 2271 /* We didn't clean anything on this queue, move along */ 2272 if (!tx_q->cleaned_bytes) 2273 continue; 2274 2275 *cleaned += tx_q->cleaned_pkts; 2276 2277 /* Update BQL */ 2278 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2279 2280 dont_wake = !complq_ok || !test_bit(IDPF_VPORT_UP, np->state) || 2281 !netif_carrier_ok(tx_q->netdev); 2282 /* Check if the TXQ needs to and can be restarted */ 2283 __netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes, 2284 IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, 2285 dont_wake); 2286 2287 /* Reset cleaned stats for the next time this queue is 2288 * cleaned 2289 */ 2290 tx_q->cleaned_bytes = 0; 2291 tx_q->cleaned_pkts = 0; 2292 } 2293 2294 ntc += complq->desc_count; 2295 complq->next_to_clean = ntc; 2296 2297 return !!complq_budget; 2298 } 2299 2300 /** 2301 * idpf_wait_for_sw_marker_completion - wait for SW marker of disabled Tx queue 2302 * @txq: disabled Tx queue 2303 * 2304 * When Tx queue is requested for disabling, the CP sends a special completion 2305 * descriptor called "SW marker", meaning the queue is ready to be destroyed. 2306 * If, for some reason, the marker is not received within 500 ms, break the 2307 * polling to not hang the driver. 2308 */ 2309 void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) 2310 { 2311 struct idpf_compl_queue *complq; 2312 unsigned long timeout; 2313 bool flow, gen_flag; 2314 u32 ntc; 2315 2316 if (!idpf_queue_has(SW_MARKER, txq)) 2317 return; 2318 2319 complq = idpf_queue_has(XDP, txq) ? txq->complq : txq->txq_grp->complq; 2320 ntc = complq->next_to_clean; 2321 2322 flow = idpf_queue_has(FLOW_SCH_EN, complq); 2323 gen_flag = idpf_queue_has(GEN_CHK, complq); 2324 2325 timeout = jiffies + msecs_to_jiffies(IDPF_WAIT_FOR_MARKER_TIMEO); 2326 2327 do { 2328 struct idpf_splitq_4b_tx_compl_desc *tx_desc; 2329 struct idpf_tx_queue *target; 2330 u32 ctype_gen, id; 2331 2332 tx_desc = flow ? &complq->comp[ntc].common : 2333 &complq->comp_4b[ntc]; 2334 ctype_gen = le16_to_cpu(tx_desc->qid_comptype_gen); 2335 2336 if (!!(ctype_gen & IDPF_TXD_COMPLQ_GEN_M) != gen_flag) { 2337 usleep_range(500, 1000); 2338 continue; 2339 } 2340 2341 if (FIELD_GET(IDPF_TXD_COMPLQ_COMPL_TYPE_M, ctype_gen) != 2342 IDPF_TXD_COMPLT_SW_MARKER) 2343 goto next; 2344 2345 id = FIELD_GET(IDPF_TXD_COMPLQ_QID_M, ctype_gen); 2346 target = complq->txq_grp->txqs[id]; 2347 2348 idpf_queue_clear(SW_MARKER, target); 2349 if (target == txq) 2350 break; 2351 2352 next: 2353 if (unlikely(++ntc == complq->desc_count)) { 2354 ntc = 0; 2355 gen_flag = !gen_flag; 2356 } 2357 } while (time_before(jiffies, timeout)); 2358 2359 idpf_queue_assign(GEN_CHK, complq, gen_flag); 2360 complq->next_to_clean = ntc; 2361 } 2362 2363 /** 2364 * idpf_tx_splitq_build_ctb - populate command tag and size for queue 2365 * based scheduling descriptors 2366 * @desc: descriptor to populate 2367 * @params: pointer to tx params struct 2368 * @td_cmd: command to be filled in desc 2369 * @size: size of buffer 2370 */ 2371 void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc, 2372 struct idpf_tx_splitq_params *params, 2373 u16 td_cmd, u16 size) 2374 { 2375 desc->q.qw1.cmd_dtype = 2376 le16_encode_bits(params->dtype, IDPF_FLEX_TXD_QW1_DTYPE_M); 2377 desc->q.qw1.cmd_dtype |= 2378 le16_encode_bits(td_cmd, IDPF_FLEX_TXD_QW1_CMD_M); 2379 desc->q.qw1.buf_size = cpu_to_le16(size); 2380 desc->q.qw1.l2tags.l2tag1 = cpu_to_le16(params->td_tag); 2381 } 2382 2383 /** 2384 * idpf_tx_splitq_build_flow_desc - populate command tag and size for flow 2385 * scheduling descriptors 2386 * @desc: descriptor to populate 2387 * @params: pointer to tx params struct 2388 * @td_cmd: command to be filled in desc 2389 * @size: size of buffer 2390 */ 2391 void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, 2392 struct idpf_tx_splitq_params *params, 2393 u16 td_cmd, u16 size) 2394 { 2395 *(u32 *)&desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); 2396 desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); 2397 desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); 2398 } 2399 2400 /** 2401 * idpf_tx_splitq_has_room - check if enough Tx splitq resources are available 2402 * @tx_q: the queue to be checked 2403 * @descs_needed: number of descriptors required for this packet 2404 * @bufs_needed: number of Tx buffers required for this packet 2405 * 2406 * Return: 0 if no room available, 1 otherwise 2407 */ 2408 static int idpf_txq_has_room(struct idpf_tx_queue *tx_q, u32 descs_needed, 2409 u32 bufs_needed) 2410 { 2411 if (IDPF_DESC_UNUSED(tx_q) < descs_needed || 2412 IDPF_TX_COMPLQ_PENDING(tx_q->txq_grp) > 2413 IDPF_TX_COMPLQ_OVERFLOW_THRESH(tx_q->txq_grp->complq) || 2414 idpf_tx_splitq_get_free_bufs(tx_q->refillq) < bufs_needed) 2415 return 0; 2416 return 1; 2417 } 2418 2419 /** 2420 * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions 2421 * @tx_q: the queue to be checked 2422 * @descs_needed: number of descriptors required for this packet 2423 * @bufs_needed: number of buffers needed for this packet 2424 * 2425 * Return: 0 if stop is not needed 2426 */ 2427 static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, 2428 u32 descs_needed, 2429 u32 bufs_needed) 2430 { 2431 /* Since we have multiple resources to check for splitq, our 2432 * start,stop_thrs becomes a boolean check instead of a count 2433 * threshold. 2434 */ 2435 if (netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx, 2436 idpf_txq_has_room(tx_q, descs_needed, 2437 bufs_needed), 2438 1, 1)) 2439 return 0; 2440 2441 u64_stats_update_begin(&tx_q->stats_sync); 2442 u64_stats_inc(&tx_q->q_stats.q_busy); 2443 u64_stats_update_end(&tx_q->stats_sync); 2444 2445 return -EBUSY; 2446 } 2447 2448 /** 2449 * idpf_tx_buf_hw_update - Store the new tail value 2450 * @tx_q: queue to bump 2451 * @val: new tail index 2452 * @xmit_more: more skb's pending 2453 * 2454 * The naming here is special in that 'hw' signals that this function is about 2455 * to do a register write to update our queue status. We know this can only 2456 * mean tail here as HW should be owning head for TX. 2457 */ 2458 void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, 2459 bool xmit_more) 2460 { 2461 struct netdev_queue *nq; 2462 2463 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2464 tx_q->next_to_use = val; 2465 2466 /* Force memory writes to complete before letting h/w 2467 * know there are new descriptors to fetch. (Only 2468 * applicable for weak-ordered memory model archs, 2469 * such as IA-64). 2470 */ 2471 wmb(); 2472 2473 /* notify HW of packet */ 2474 if (netif_xmit_stopped(nq) || !xmit_more) 2475 writel(val, tx_q->tail); 2476 } 2477 2478 /** 2479 * idpf_tx_res_count_required - get number of Tx resources needed for this pkt 2480 * @txq: queue to send buffer on 2481 * @skb: send buffer 2482 * @bufs_needed: (output) number of buffers needed for this skb. 2483 * 2484 * Return: number of data descriptors and buffers needed for this skb. 2485 */ 2486 unsigned int idpf_tx_res_count_required(struct idpf_tx_queue *txq, 2487 struct sk_buff *skb, 2488 u32 *bufs_needed) 2489 { 2490 const struct skb_shared_info *shinfo; 2491 unsigned int count = 0, i; 2492 2493 count += !!skb_headlen(skb); 2494 2495 if (!skb_is_nonlinear(skb)) 2496 return count; 2497 2498 shinfo = skb_shinfo(skb); 2499 *bufs_needed += shinfo->nr_frags; 2500 for (i = 0; i < shinfo->nr_frags; i++) { 2501 unsigned int size; 2502 2503 size = skb_frag_size(&shinfo->frags[i]); 2504 2505 /* We only need to use the idpf_size_to_txd_count check if the 2506 * fragment is going to span multiple descriptors, 2507 * i.e. size >= 16K. 2508 */ 2509 if (size >= SZ_16K) 2510 count += idpf_size_to_txd_count(size); 2511 else 2512 count++; 2513 } 2514 2515 if (idpf_chk_linearize(skb, txq->tx_max_bufs, count)) { 2516 if (__skb_linearize(skb)) 2517 return 0; 2518 2519 count = idpf_size_to_txd_count(skb->len); 2520 u64_stats_update_begin(&txq->stats_sync); 2521 u64_stats_inc(&txq->q_stats.linearize); 2522 u64_stats_update_end(&txq->stats_sync); 2523 } 2524 2525 return count; 2526 } 2527 2528 /** 2529 * idpf_tx_splitq_bump_ntu - adjust NTU and generation 2530 * @txq: the tx ring to wrap 2531 * @ntu: ring index to bump 2532 */ 2533 static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_tx_queue *txq, u16 ntu) 2534 { 2535 ntu++; 2536 2537 if (ntu == txq->desc_count) 2538 ntu = 0; 2539 2540 return ntu; 2541 } 2542 2543 /** 2544 * idpf_tx_get_free_buf_id - get a free buffer ID from the refill queue 2545 * @refillq: refill queue to get buffer ID from 2546 * @buf_id: return buffer ID 2547 * 2548 * Return: true if a buffer ID was found, false if not 2549 */ 2550 static bool idpf_tx_get_free_buf_id(struct idpf_sw_queue *refillq, 2551 u32 *buf_id) 2552 { 2553 u32 ntc = refillq->next_to_clean; 2554 u32 refill_desc; 2555 2556 refill_desc = refillq->ring[ntc]; 2557 2558 if (unlikely(idpf_queue_has(RFL_GEN_CHK, refillq) != 2559 !!(refill_desc & IDPF_RFL_BI_GEN_M))) 2560 return false; 2561 2562 *buf_id = FIELD_GET(IDPF_RFL_BI_BUFID_M, refill_desc); 2563 2564 if (unlikely(++ntc == refillq->desc_count)) { 2565 idpf_queue_change(RFL_GEN_CHK, refillq); 2566 ntc = 0; 2567 } 2568 2569 refillq->next_to_clean = ntc; 2570 2571 return true; 2572 } 2573 2574 /** 2575 * idpf_tx_splitq_pkt_err_unmap - Unmap buffers and bump tail in case of error 2576 * @txq: Tx queue to unwind 2577 * @params: pointer to splitq params struct 2578 * @first: starting buffer for packet to unmap 2579 */ 2580 static void idpf_tx_splitq_pkt_err_unmap(struct idpf_tx_queue *txq, 2581 struct idpf_tx_splitq_params *params, 2582 struct idpf_tx_buf *first) 2583 { 2584 struct idpf_sw_queue *refillq = txq->refillq; 2585 struct libeth_sq_napi_stats ss = { }; 2586 struct idpf_tx_buf *tx_buf = first; 2587 struct libeth_cq_pp cp = { 2588 .dev = txq->dev, 2589 .ss = &ss, 2590 }; 2591 2592 u64_stats_update_begin(&txq->stats_sync); 2593 u64_stats_inc(&txq->q_stats.dma_map_errs); 2594 u64_stats_update_end(&txq->stats_sync); 2595 2596 libeth_tx_complete(tx_buf, &cp); 2597 while (idpf_tx_buf_next(tx_buf) != IDPF_TXBUF_NULL) { 2598 tx_buf = &txq->tx_buf[idpf_tx_buf_next(tx_buf)]; 2599 libeth_tx_complete(tx_buf, &cp); 2600 } 2601 2602 /* Update tail in case netdev_xmit_more was previously true. */ 2603 idpf_tx_buf_hw_update(txq, params->prev_ntu, false); 2604 2605 if (!refillq) 2606 return; 2607 2608 /* Restore refillq state to avoid leaking tags. */ 2609 if (params->prev_refill_gen != idpf_queue_has(RFL_GEN_CHK, refillq)) 2610 idpf_queue_change(RFL_GEN_CHK, refillq); 2611 refillq->next_to_clean = params->prev_refill_ntc; 2612 } 2613 2614 /** 2615 * idpf_tx_splitq_map - Build the Tx flex descriptor 2616 * @tx_q: queue to send buffer on 2617 * @params: pointer to splitq params struct 2618 * @first: first buffer info buffer to use 2619 * 2620 * This function loops over the skb data pointed to by *first 2621 * and gets a physical address for each memory location and programs 2622 * it and the length into the transmit flex descriptor. 2623 */ 2624 static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, 2625 struct idpf_tx_splitq_params *params, 2626 struct idpf_tx_buf *first) 2627 { 2628 union idpf_tx_flex_desc *tx_desc; 2629 unsigned int data_len, size; 2630 struct idpf_tx_buf *tx_buf; 2631 u16 i = tx_q->next_to_use; 2632 struct netdev_queue *nq; 2633 struct sk_buff *skb; 2634 skb_frag_t *frag; 2635 u32 next_buf_id; 2636 u16 td_cmd = 0; 2637 dma_addr_t dma; 2638 2639 skb = first->skb; 2640 2641 td_cmd = params->offload.td_cmd; 2642 2643 data_len = skb->data_len; 2644 size = skb_headlen(skb); 2645 2646 tx_desc = &tx_q->flex_tx[i]; 2647 2648 dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE); 2649 2650 tx_buf = first; 2651 first->nr_frags = 0; 2652 2653 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 2654 unsigned int max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED; 2655 2656 if (unlikely(dma_mapping_error(tx_q->dev, dma))) { 2657 idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL; 2658 return idpf_tx_splitq_pkt_err_unmap(tx_q, params, 2659 first); 2660 } 2661 2662 first->nr_frags++; 2663 tx_buf->type = LIBETH_SQE_FRAG; 2664 2665 /* record length, and DMA address */ 2666 dma_unmap_len_set(tx_buf, len, size); 2667 dma_unmap_addr_set(tx_buf, dma, dma); 2668 2669 /* buf_addr is in same location for both desc types */ 2670 tx_desc->q.buf_addr = cpu_to_le64(dma); 2671 2672 /* The stack can send us fragments that are too large for a 2673 * single descriptor i.e. frag size > 16K-1. We will need to 2674 * split the fragment across multiple descriptors in this case. 2675 * To adhere to HW alignment restrictions, the fragment needs 2676 * to be split such that the first chunk ends on a 4K boundary 2677 * and all subsequent chunks start on a 4K boundary. We still 2678 * want to send as much data as possible though, so our 2679 * intermediate descriptor chunk size will be 12K. 2680 * 2681 * For example, consider a 32K fragment mapped to DMA addr 2600. 2682 * ------------------------------------------------------------ 2683 * | frag_size = 32K | 2684 * ------------------------------------------------------------ 2685 * |2600 |16384 |28672 2686 * 2687 * 3 descriptors will be used for this fragment. The HW expects 2688 * the descriptors to contain the following: 2689 * ------------------------------------------------------------ 2690 * | size = 13784 | size = 12K | size = 6696 | 2691 * | dma = 2600 | dma = 16384 | dma = 28672 | 2692 * ------------------------------------------------------------ 2693 * 2694 * We need to first adjust the max_data for the first chunk so 2695 * that it ends on a 4K boundary. By negating the value of the 2696 * DMA address and taking only the low order bits, we're 2697 * effectively calculating 2698 * 4K - (DMA addr lower order bits) = 2699 * bytes to next boundary. 2700 * 2701 * Add that to our base aligned max_data (12K) and we have 2702 * our first chunk size. In the example above, 2703 * 13784 = 12K + (4096-2600) 2704 * 2705 * After guaranteeing the first chunk ends on a 4K boundary, we 2706 * will give the intermediate descriptors 12K chunks and 2707 * whatever is left to the final descriptor. This ensures that 2708 * all descriptors used for the remaining chunks of the 2709 * fragment start on a 4K boundary and we use as few 2710 * descriptors as possible. 2711 */ 2712 max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1); 2713 while (unlikely(size > IDPF_TX_MAX_DESC_DATA)) { 2714 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, 2715 max_data); 2716 2717 if (unlikely(++i == tx_q->desc_count)) { 2718 tx_desc = &tx_q->flex_tx[0]; 2719 i = 0; 2720 } else { 2721 tx_desc++; 2722 } 2723 2724 /* Adjust the DMA offset and the remaining size of the 2725 * fragment. On the first iteration of this loop, 2726 * max_data will be >= 12K and <= 16K-1. On any 2727 * subsequent iteration of this loop, max_data will 2728 * always be 12K. 2729 */ 2730 dma += max_data; 2731 size -= max_data; 2732 2733 /* Reset max_data since remaining chunks will be 12K 2734 * at most 2735 */ 2736 max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED; 2737 2738 /* buf_addr is in same location for both desc types */ 2739 tx_desc->q.buf_addr = cpu_to_le64(dma); 2740 } 2741 2742 if (!data_len) 2743 break; 2744 2745 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); 2746 2747 if (unlikely(++i == tx_q->desc_count)) { 2748 tx_desc = &tx_q->flex_tx[0]; 2749 i = 0; 2750 } else { 2751 tx_desc++; 2752 } 2753 2754 if (idpf_queue_has(FLOW_SCH_EN, tx_q)) { 2755 if (unlikely(!idpf_tx_get_free_buf_id(tx_q->refillq, 2756 &next_buf_id))) { 2757 idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL; 2758 return idpf_tx_splitq_pkt_err_unmap(tx_q, params, 2759 first); 2760 } 2761 } else { 2762 next_buf_id = i; 2763 } 2764 idpf_tx_buf_next(tx_buf) = next_buf_id; 2765 tx_buf = &tx_q->tx_buf[next_buf_id]; 2766 2767 size = skb_frag_size(frag); 2768 data_len -= size; 2769 2770 dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, 2771 DMA_TO_DEVICE); 2772 } 2773 2774 /* record SW timestamp if HW timestamp is not available */ 2775 skb_tx_timestamp(skb); 2776 2777 first->type = LIBETH_SQE_SKB; 2778 2779 /* write last descriptor with RS and EOP bits */ 2780 first->rs_idx = i; 2781 idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL; 2782 td_cmd |= params->eop_cmd; 2783 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); 2784 i = idpf_tx_splitq_bump_ntu(tx_q, i); 2785 2786 tx_q->txq_grp->num_completions_pending++; 2787 2788 /* record bytecount for BQL */ 2789 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2790 netdev_tx_sent_queue(nq, first->bytes); 2791 2792 idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); 2793 } 2794 2795 /** 2796 * idpf_tso - computes mss and TSO length to prepare for TSO 2797 * @skb: pointer to skb 2798 * @off: pointer to struct that holds offload parameters 2799 * 2800 * Returns error (negative) if TSO was requested but cannot be applied to the 2801 * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise. 2802 */ 2803 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) 2804 { 2805 const struct skb_shared_info *shinfo; 2806 union { 2807 struct iphdr *v4; 2808 struct ipv6hdr *v6; 2809 unsigned char *hdr; 2810 } ip; 2811 union { 2812 struct tcphdr *tcp; 2813 struct udphdr *udp; 2814 unsigned char *hdr; 2815 } l4; 2816 u32 paylen, l4_start; 2817 int err; 2818 2819 if (!skb_is_gso(skb)) 2820 return 0; 2821 2822 err = skb_cow_head(skb, 0); 2823 if (err < 0) 2824 return err; 2825 2826 shinfo = skb_shinfo(skb); 2827 2828 ip.hdr = skb_network_header(skb); 2829 l4.hdr = skb_transport_header(skb); 2830 2831 /* initialize outer IP header fields */ 2832 if (ip.v4->version == 4) { 2833 ip.v4->tot_len = 0; 2834 ip.v4->check = 0; 2835 } else if (ip.v6->version == 6) { 2836 ip.v6->payload_len = 0; 2837 } 2838 2839 l4_start = skb_transport_offset(skb); 2840 2841 /* remove payload length from checksum */ 2842 paylen = skb->len - l4_start; 2843 2844 switch (shinfo->gso_type & ~SKB_GSO_DODGY) { 2845 case SKB_GSO_TCPV4: 2846 case SKB_GSO_TCPV6: 2847 csum_replace_by_diff(&l4.tcp->check, 2848 (__force __wsum)htonl(paylen)); 2849 off->tso_hdr_len = __tcp_hdrlen(l4.tcp) + l4_start; 2850 break; 2851 case SKB_GSO_UDP_L4: 2852 csum_replace_by_diff(&l4.udp->check, 2853 (__force __wsum)htonl(paylen)); 2854 /* compute length of segmentation header */ 2855 off->tso_hdr_len = sizeof(struct udphdr) + l4_start; 2856 l4.udp->len = htons(shinfo->gso_size + sizeof(struct udphdr)); 2857 break; 2858 default: 2859 return -EINVAL; 2860 } 2861 2862 off->tso_len = skb->len - off->tso_hdr_len; 2863 off->mss = shinfo->gso_size; 2864 off->tso_segs = shinfo->gso_segs; 2865 2866 off->tx_flags |= IDPF_TX_FLAGS_TSO; 2867 2868 return 1; 2869 } 2870 2871 2872 /** 2873 * idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring 2874 * @txq: queue to put context descriptor on 2875 * 2876 * Since the TX buffer rings mimics the descriptor ring, update the tx buffer 2877 * ring entry to reflect that this index is a context descriptor 2878 */ 2879 static union idpf_flex_tx_ctx_desc * 2880 idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) 2881 { 2882 union idpf_flex_tx_ctx_desc *desc; 2883 int i = txq->next_to_use; 2884 2885 /* grab the next descriptor */ 2886 desc = &txq->flex_ctx[i]; 2887 txq->next_to_use = idpf_tx_splitq_bump_ntu(txq, i); 2888 2889 return desc; 2890 } 2891 2892 /** 2893 * idpf_tx_drop_skb - free the SKB and bump tail if necessary 2894 * @tx_q: queue to send buffer on 2895 * @skb: pointer to skb 2896 */ 2897 netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb) 2898 { 2899 u64_stats_update_begin(&tx_q->stats_sync); 2900 u64_stats_inc(&tx_q->q_stats.skb_drops); 2901 u64_stats_update_end(&tx_q->stats_sync); 2902 2903 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); 2904 2905 dev_kfree_skb(skb); 2906 2907 return NETDEV_TX_OK; 2908 } 2909 2910 #if (IS_ENABLED(CONFIG_PTP_1588_CLOCK)) 2911 /** 2912 * idpf_tx_tstamp - set up context descriptor for hardware timestamp 2913 * @tx_q: queue to send buffer on 2914 * @skb: pointer to the SKB we're sending 2915 * @off: pointer to the offload struct 2916 * 2917 * Return: Positive index number on success, negative otherwise. 2918 */ 2919 static int idpf_tx_tstamp(struct idpf_tx_queue *tx_q, struct sk_buff *skb, 2920 struct idpf_tx_offload_params *off) 2921 { 2922 int err, idx; 2923 2924 /* only timestamp the outbound packet if the user has requested it */ 2925 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 2926 return -1; 2927 2928 if (!idpf_ptp_get_txq_tstamp_capability(tx_q)) 2929 return -1; 2930 2931 /* Tx timestamps cannot be sampled when doing TSO */ 2932 if (off->tx_flags & IDPF_TX_FLAGS_TSO) 2933 return -1; 2934 2935 /* Grab an open timestamp slot */ 2936 err = idpf_ptp_request_ts(tx_q, skb, &idx); 2937 if (err) { 2938 u64_stats_update_begin(&tx_q->stats_sync); 2939 u64_stats_inc(&tx_q->q_stats.tstamp_skipped); 2940 u64_stats_update_end(&tx_q->stats_sync); 2941 2942 return -1; 2943 } 2944 2945 off->tx_flags |= IDPF_TX_FLAGS_TSYN; 2946 2947 return idx; 2948 } 2949 2950 /** 2951 * idpf_tx_set_tstamp_desc - Set the Tx descriptor fields needed to generate 2952 * PHY Tx timestamp 2953 * @ctx_desc: Context descriptor 2954 * @idx: Index of the Tx timestamp latch 2955 */ 2956 static void idpf_tx_set_tstamp_desc(union idpf_flex_tx_ctx_desc *ctx_desc, 2957 u32 idx) 2958 { 2959 ctx_desc->tsyn.qw1 = le64_encode_bits(IDPF_TX_DESC_DTYPE_CTX, 2960 IDPF_TX_CTX_DTYPE_M) | 2961 le64_encode_bits(IDPF_TX_CTX_DESC_TSYN, 2962 IDPF_TX_CTX_CMD_M) | 2963 le64_encode_bits(idx, IDPF_TX_CTX_TSYN_REG_M); 2964 } 2965 #else /* CONFIG_PTP_1588_CLOCK */ 2966 static int idpf_tx_tstamp(struct idpf_tx_queue *tx_q, struct sk_buff *skb, 2967 struct idpf_tx_offload_params *off) 2968 { 2969 return -1; 2970 } 2971 2972 static void idpf_tx_set_tstamp_desc(union idpf_flex_tx_ctx_desc *ctx_desc, 2973 u32 idx) 2974 { } 2975 #endif /* CONFIG_PTP_1588_CLOCK */ 2976 2977 /** 2978 * idpf_tx_splitq_need_re - check whether RE bit needs to be set 2979 * @tx_q: pointer to Tx queue 2980 * 2981 * Return: true if RE bit needs to be set, false otherwise 2982 */ 2983 static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q) 2984 { 2985 int gap = tx_q->next_to_use - tx_q->last_re; 2986 2987 gap += (gap < 0) ? tx_q->desc_count : 0; 2988 2989 return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP; 2990 } 2991 2992 /** 2993 * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors 2994 * @skb: send buffer 2995 * @tx_q: queue to send buffer on 2996 * 2997 * Returns NETDEV_TX_OK if sent, else an error code 2998 */ 2999 static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, 3000 struct idpf_tx_queue *tx_q) 3001 { 3002 struct idpf_tx_splitq_params tx_params = { 3003 .prev_ntu = tx_q->next_to_use, 3004 }; 3005 union idpf_flex_tx_ctx_desc *ctx_desc; 3006 struct idpf_tx_buf *first; 3007 u32 count, buf_count = 1; 3008 int tso, idx; 3009 u32 buf_id; 3010 3011 count = idpf_tx_res_count_required(tx_q, skb, &buf_count); 3012 if (unlikely(!count)) 3013 return idpf_tx_drop_skb(tx_q, skb); 3014 3015 tso = idpf_tso(skb, &tx_params.offload); 3016 if (unlikely(tso < 0)) 3017 return idpf_tx_drop_skb(tx_q, skb); 3018 3019 /* Check for splitq specific TX resources */ 3020 count += (IDPF_TX_DESCS_PER_CACHE_LINE + tso); 3021 if (idpf_tx_maybe_stop_splitq(tx_q, count, buf_count)) { 3022 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); 3023 3024 return NETDEV_TX_BUSY; 3025 } 3026 3027 if (tso) { 3028 /* If tso is needed, set up context desc */ 3029 ctx_desc = idpf_tx_splitq_get_ctx_desc(tx_q); 3030 3031 ctx_desc->tso.qw1.cmd_dtype = 3032 cpu_to_le16(IDPF_TX_DESC_DTYPE_FLEX_TSO_CTX | 3033 IDPF_TX_FLEX_CTX_DESC_CMD_TSO); 3034 ctx_desc->tso.qw0.flex_tlen = 3035 cpu_to_le32(tx_params.offload.tso_len & 3036 IDPF_TXD_FLEX_CTX_TLEN_M); 3037 ctx_desc->tso.qw0.mss_rt = 3038 cpu_to_le16(tx_params.offload.mss & 3039 IDPF_TXD_FLEX_CTX_MSS_RT_M); 3040 ctx_desc->tso.qw0.hdr_len = tx_params.offload.tso_hdr_len; 3041 3042 u64_stats_update_begin(&tx_q->stats_sync); 3043 u64_stats_inc(&tx_q->q_stats.lso_pkts); 3044 u64_stats_update_end(&tx_q->stats_sync); 3045 } 3046 3047 idx = idpf_tx_tstamp(tx_q, skb, &tx_params.offload); 3048 if (idx != -1) { 3049 ctx_desc = idpf_tx_splitq_get_ctx_desc(tx_q); 3050 idpf_tx_set_tstamp_desc(ctx_desc, idx); 3051 } 3052 3053 if (idpf_queue_has(FLOW_SCH_EN, tx_q)) { 3054 struct idpf_sw_queue *refillq = tx_q->refillq; 3055 3056 /* Save refillq state in case of a packet rollback. Otherwise, 3057 * the tags will be leaked since they will be popped from the 3058 * refillq but never reposted during cleaning. 3059 */ 3060 tx_params.prev_refill_gen = 3061 idpf_queue_has(RFL_GEN_CHK, refillq); 3062 tx_params.prev_refill_ntc = refillq->next_to_clean; 3063 3064 if (unlikely(!idpf_tx_get_free_buf_id(tx_q->refillq, 3065 &buf_id))) { 3066 if (tx_params.prev_refill_gen != 3067 idpf_queue_has(RFL_GEN_CHK, refillq)) 3068 idpf_queue_change(RFL_GEN_CHK, refillq); 3069 refillq->next_to_clean = tx_params.prev_refill_ntc; 3070 3071 tx_q->next_to_use = tx_params.prev_ntu; 3072 return idpf_tx_drop_skb(tx_q, skb); 3073 } 3074 tx_params.compl_tag = buf_id; 3075 3076 tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE; 3077 tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP; 3078 /* Set the RE bit to periodically "clean" the descriptor ring. 3079 * MIN_GAP is set to MIN_RING size to ensure it will be set at 3080 * least once each time around the ring. 3081 */ 3082 if (idpf_tx_splitq_need_re(tx_q)) { 3083 tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE; 3084 tx_q->txq_grp->num_completions_pending++; 3085 tx_q->last_re = tx_q->next_to_use; 3086 } 3087 3088 if (skb->ip_summed == CHECKSUM_PARTIAL) 3089 tx_params.offload.td_cmd |= IDPF_TXD_FLEX_FLOW_CMD_CS_EN; 3090 3091 } else { 3092 buf_id = tx_q->next_to_use; 3093 3094 tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2; 3095 tx_params.eop_cmd = IDPF_TXD_LAST_DESC_CMD; 3096 3097 if (skb->ip_summed == CHECKSUM_PARTIAL) 3098 tx_params.offload.td_cmd |= IDPF_TX_FLEX_DESC_CMD_CS_EN; 3099 } 3100 3101 first = &tx_q->tx_buf[buf_id]; 3102 first->skb = skb; 3103 3104 if (tso) { 3105 first->packets = tx_params.offload.tso_segs; 3106 first->bytes = skb->len + 3107 ((first->packets - 1) * tx_params.offload.tso_hdr_len); 3108 } else { 3109 first->packets = 1; 3110 first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); 3111 } 3112 3113 idpf_tx_splitq_map(tx_q, &tx_params, first); 3114 3115 return NETDEV_TX_OK; 3116 } 3117 3118 /** 3119 * idpf_tx_start - Selects the right Tx queue to send buffer 3120 * @skb: send buffer 3121 * @netdev: network interface device structure 3122 * 3123 * Returns NETDEV_TX_OK if sent, else an error code 3124 */ 3125 netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) 3126 { 3127 const struct idpf_vport *vport = idpf_netdev_to_vport(netdev); 3128 struct idpf_tx_queue *tx_q; 3129 3130 if (unlikely(skb_get_queue_mapping(skb) >= 3131 vport->num_txq - vport->num_xdp_txq)) { 3132 dev_kfree_skb_any(skb); 3133 3134 return NETDEV_TX_OK; 3135 } 3136 3137 tx_q = vport->txqs[skb_get_queue_mapping(skb)]; 3138 3139 /* hardware can't handle really short frames, hardware padding works 3140 * beyond this point 3141 */ 3142 if (skb_put_padto(skb, tx_q->tx_min_pkt_len)) { 3143 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); 3144 3145 return NETDEV_TX_OK; 3146 } 3147 3148 if (idpf_is_queue_model_split(vport->txq_model)) 3149 return idpf_tx_splitq_frame(skb, tx_q); 3150 else 3151 return idpf_tx_singleq_frame(skb, tx_q); 3152 } 3153 3154 /** 3155 * idpf_rx_hash - set the hash value in the skb 3156 * @rxq: Rx descriptor ring packet is being transacted on 3157 * @skb: pointer to current skb being populated 3158 * @rx_desc: Receive descriptor 3159 * @decoded: Decoded Rx packet type related fields 3160 */ 3161 static void 3162 idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb, 3163 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, 3164 struct libeth_rx_pt decoded) 3165 { 3166 u32 hash; 3167 3168 if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, decoded)) 3169 return; 3170 3171 hash = le16_to_cpu(rx_desc->hash1) | 3172 (rx_desc->ff2_mirrid_hash2.hash2 << 16) | 3173 (rx_desc->hash3 << 24); 3174 3175 libeth_rx_pt_set_hash(skb, hash, decoded); 3176 } 3177 3178 /** 3179 * idpf_rx_csum - Indicate in skb if checksum is good 3180 * @rxq: Rx descriptor ring packet is being transacted on 3181 * @skb: pointer to current skb being populated 3182 * @csum_bits: checksum fields extracted from the descriptor 3183 * @decoded: Decoded Rx packet type related fields 3184 * 3185 * skb->protocol must be set before this function is called 3186 */ 3187 static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, 3188 struct libeth_rx_csum csum_bits, 3189 struct libeth_rx_pt decoded) 3190 { 3191 bool ipv4, ipv6; 3192 3193 /* check if Rx checksum is enabled */ 3194 if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) 3195 return; 3196 3197 /* check if HW has decoded the packet and checksum */ 3198 if (unlikely(!csum_bits.l3l4p)) 3199 return; 3200 3201 ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4; 3202 ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6; 3203 3204 if (unlikely(ipv4 && (csum_bits.ipe || csum_bits.eipe))) 3205 goto checksum_fail; 3206 3207 if (unlikely(ipv6 && csum_bits.ipv6exadd)) 3208 return; 3209 3210 /* check for L4 errors and handle packets that were not able to be 3211 * checksummed 3212 */ 3213 if (unlikely(csum_bits.l4e)) 3214 goto checksum_fail; 3215 3216 if (!csum_bits.raw_csum_valid || 3217 decoded.inner_prot == LIBETH_RX_PT_INNER_SCTP) { 3218 skb->ip_summed = CHECKSUM_UNNECESSARY; 3219 return; 3220 } 3221 3222 skb->csum = csum_unfold((__force __sum16)~swab16(csum_bits.raw_csum)); 3223 skb->ip_summed = CHECKSUM_COMPLETE; 3224 3225 return; 3226 3227 checksum_fail: 3228 u64_stats_update_begin(&rxq->stats_sync); 3229 u64_stats_inc(&rxq->q_stats.hw_csum_err); 3230 u64_stats_update_end(&rxq->stats_sync); 3231 } 3232 3233 /** 3234 * idpf_rx_splitq_extract_csum_bits - Extract checksum bits from descriptor 3235 * @rx_desc: receive descriptor 3236 * 3237 * Return: parsed checksum status. 3238 **/ 3239 static struct libeth_rx_csum 3240 idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) 3241 { 3242 struct libeth_rx_csum csum = { }; 3243 u8 qword0, qword1; 3244 3245 qword0 = rx_desc->status_err0_qw0; 3246 qword1 = rx_desc->status_err0_qw1; 3247 3248 csum.ipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_M, 3249 qword1); 3250 csum.eipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_M, 3251 qword1); 3252 csum.l4e = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_M, 3253 qword1); 3254 csum.l3l4p = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_M, 3255 qword1); 3256 csum.ipv6exadd = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_IPV6EXADD_M, 3257 qword0); 3258 csum.raw_csum_valid = 3259 !le16_get_bits(rx_desc->ptype_err_fflags0, 3260 VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_M); 3261 csum.raw_csum = le16_to_cpu(rx_desc->misc.raw_cs); 3262 3263 return csum; 3264 } 3265 3266 /** 3267 * idpf_rx_rsc - Set the RSC fields in the skb 3268 * @rxq : Rx descriptor ring packet is being transacted on 3269 * @skb : pointer to current skb being populated 3270 * @rx_desc: Receive descriptor 3271 * @decoded: Decoded Rx packet type related fields 3272 * 3273 * Return 0 on success and error code on failure 3274 * 3275 * Populate the skb fields with the total number of RSC segments, RSC payload 3276 * length and packet type. 3277 */ 3278 static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, 3279 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, 3280 struct libeth_rx_pt decoded) 3281 { 3282 u16 rsc_segments, rsc_seg_len; 3283 bool ipv4, ipv6; 3284 int len; 3285 3286 if (unlikely(libeth_rx_pt_get_ip_ver(decoded) == 3287 LIBETH_RX_PT_OUTER_L2)) 3288 return -EINVAL; 3289 3290 rsc_seg_len = le16_to_cpu(rx_desc->misc.rscseglen); 3291 if (unlikely(!rsc_seg_len)) 3292 return -EINVAL; 3293 3294 ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4; 3295 ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6; 3296 3297 if (unlikely(!(ipv4 ^ ipv6))) 3298 return -EINVAL; 3299 3300 rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len); 3301 3302 NAPI_GRO_CB(skb)->count = rsc_segments; 3303 skb_shinfo(skb)->gso_size = rsc_seg_len; 3304 3305 skb_reset_network_header(skb); 3306 3307 if (ipv4) { 3308 struct iphdr *ipv4h = ip_hdr(skb); 3309 3310 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 3311 3312 /* Reset and set transport header offset in skb */ 3313 skb_set_transport_header(skb, sizeof(struct iphdr)); 3314 len = skb->len - skb_transport_offset(skb); 3315 3316 /* Compute the TCP pseudo header checksum*/ 3317 tcp_hdr(skb)->check = 3318 ~tcp_v4_check(len, ipv4h->saddr, ipv4h->daddr, 0); 3319 } else { 3320 struct ipv6hdr *ipv6h = ipv6_hdr(skb); 3321 3322 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 3323 skb_set_transport_header(skb, sizeof(struct ipv6hdr)); 3324 len = skb->len - skb_transport_offset(skb); 3325 tcp_hdr(skb)->check = 3326 ~tcp_v6_check(len, &ipv6h->saddr, &ipv6h->daddr, 0); 3327 } 3328 3329 tcp_gro_complete(skb); 3330 3331 u64_stats_update_begin(&rxq->stats_sync); 3332 u64_stats_inc(&rxq->q_stats.rsc_pkts); 3333 u64_stats_update_end(&rxq->stats_sync); 3334 3335 return 0; 3336 } 3337 3338 /** 3339 * idpf_rx_hwtstamp - check for an RX timestamp and pass up the stack 3340 * @rxq: pointer to the rx queue that receives the timestamp 3341 * @rx_desc: pointer to rx descriptor containing timestamp 3342 * @skb: skb to put timestamp in 3343 */ 3344 static void 3345 idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq, 3346 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, 3347 struct sk_buff *skb) 3348 { 3349 u64 cached_time, ts_ns; 3350 u32 ts_high; 3351 3352 if (!(rx_desc->ts_low & VIRTCHNL2_RX_FLEX_TSTAMP_VALID)) 3353 return; 3354 3355 cached_time = READ_ONCE(rxq->cached_phc_time); 3356 3357 ts_high = le32_to_cpu(rx_desc->ts_high); 3358 ts_ns = idpf_ptp_tstamp_extend_32b_to_64b(cached_time, ts_high); 3359 3360 *skb_hwtstamps(skb) = (struct skb_shared_hwtstamps) { 3361 .hwtstamp = ns_to_ktime(ts_ns), 3362 }; 3363 } 3364 3365 /** 3366 * __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor 3367 * @rxq: Rx descriptor ring packet is being transacted on 3368 * @skb: pointer to current skb being populated 3369 * @rx_desc: Receive descriptor 3370 * 3371 * This function checks the ring, descriptor, and packet information in 3372 * order to populate the hash, checksum, protocol, and 3373 * other fields within the skb. 3374 */ 3375 static int 3376 __idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, 3377 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) 3378 { 3379 struct libeth_rx_csum csum_bits; 3380 struct libeth_rx_pt decoded; 3381 u16 rx_ptype; 3382 3383 rx_ptype = le16_get_bits(rx_desc->ptype_err_fflags0, 3384 VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M); 3385 decoded = rxq->rx_ptype_lkup[rx_ptype]; 3386 3387 /* process RSS/hash */ 3388 idpf_rx_hash(rxq, skb, rx_desc, decoded); 3389 3390 if (idpf_queue_has(PTP, rxq)) 3391 idpf_rx_hwtstamp(rxq, rx_desc, skb); 3392 3393 if (le16_get_bits(rx_desc->hdrlen_flags, 3394 VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) 3395 return idpf_rx_rsc(rxq, skb, rx_desc, decoded); 3396 3397 csum_bits = idpf_rx_splitq_extract_csum_bits(rx_desc); 3398 idpf_rx_csum(rxq, skb, csum_bits, decoded); 3399 3400 return 0; 3401 } 3402 3403 bool idpf_rx_process_skb_fields(struct sk_buff *skb, 3404 const struct libeth_xdp_buff *xdp, 3405 struct libeth_rq_napi_stats *rs) 3406 { 3407 struct idpf_rx_queue *rxq; 3408 3409 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 3410 3411 return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc); 3412 } 3413 3414 LIBETH_XDP_DEFINE_START(); 3415 LIBETH_XDP_DEFINE_RUN(static idpf_xdp_run_pass, idpf_xdp_run_prog, 3416 idpf_xdp_tx_flush_bulk, idpf_rx_process_skb_fields); 3417 LIBETH_XDP_DEFINE_FINALIZE(static idpf_xdp_finalize_rx, idpf_xdp_tx_flush_bulk, 3418 idpf_xdp_tx_finalize); 3419 LIBETH_XDP_DEFINE_END(); 3420 3421 /** 3422 * idpf_rx_hsplit_wa - handle header buffer overflows and split errors 3423 * @hdr: Rx buffer for the headers 3424 * @buf: Rx buffer for the payload 3425 * @data_len: number of bytes received to the payload buffer 3426 * 3427 * When a header buffer overflow occurs or the HW was unable do parse the 3428 * packet type to perform header split, the whole frame gets placed to the 3429 * payload buffer. We can't build a valid skb around a payload buffer when 3430 * the header split is active since it doesn't reserve any head- or tailroom. 3431 * In that case, copy either the whole frame when it's short or just the 3432 * Ethernet header to the header buffer to be able to build an skb and adjust 3433 * the data offset in the payload buffer, IOW emulate the header split. 3434 * 3435 * Return: number of bytes copied to the header buffer. 3436 */ 3437 static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr, 3438 struct libeth_fqe *buf, u32 data_len) 3439 { 3440 u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; 3441 struct page *hdr_page, *buf_page; 3442 const void *src; 3443 void *dst; 3444 3445 if (unlikely(netmem_is_net_iov(buf->netmem)) || 3446 !libeth_rx_sync_for_cpu(buf, copy)) 3447 return 0; 3448 3449 hdr_page = __netmem_to_page(hdr->netmem); 3450 buf_page = __netmem_to_page(buf->netmem); 3451 dst = page_address(hdr_page) + hdr->offset + 3452 pp_page_to_nmdesc(hdr_page)->pp->p.offset; 3453 src = page_address(buf_page) + buf->offset + 3454 pp_page_to_nmdesc(buf_page)->pp->p.offset; 3455 3456 memcpy(dst, src, LARGEST_ALIGN(copy)); 3457 buf->offset += copy; 3458 3459 return copy; 3460 } 3461 3462 /** 3463 * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor 3464 * status and error fields 3465 * @stat_err_field: field from descriptor to test bits in 3466 * @stat_err_bits: value to mask 3467 * 3468 */ 3469 static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field, 3470 const u8 stat_err_bits) 3471 { 3472 return !!(stat_err_field & stat_err_bits); 3473 } 3474 3475 /** 3476 * idpf_rx_splitq_is_eop - process handling of EOP buffers 3477 * @rx_desc: Rx descriptor for current buffer 3478 * 3479 * If the buffer is an EOP buffer, this function exits returning true, 3480 * otherwise return false indicating that this is in fact a non-EOP buffer. 3481 */ 3482 static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) 3483 { 3484 /* if we are the last buffer then there is nothing else to do */ 3485 return likely(idpf_rx_splitq_test_staterr(rx_desc->status_err0_qw1, 3486 IDPF_RXD_EOF_SPLITQ)); 3487 } 3488 3489 /** 3490 * idpf_rx_splitq_clean - Clean completed descriptors from Rx queue 3491 * @rxq: Rx descriptor queue to retrieve receive buffer queue 3492 * @budget: Total limit on number of packets to process 3493 * 3494 * This function provides a "bounce buffer" approach to Rx interrupt 3495 * processing. The advantage to this is that on systems that have 3496 * expensive overhead for IOMMU access this provides a means of avoiding 3497 * it by maintaining the mapping of the page to the system. 3498 * 3499 * Returns amount of work completed 3500 */ 3501 static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) 3502 { 3503 struct idpf_buf_queue *rx_bufq = NULL; 3504 struct libeth_rq_napi_stats rs = { }; 3505 u16 ntc = rxq->next_to_clean; 3506 LIBETH_XDP_ONSTACK_BUFF(xdp); 3507 LIBETH_XDP_ONSTACK_BULK(bq); 3508 3509 libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, 3510 rxq->xdpsqs, rxq->num_xdp_txq); 3511 libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq); 3512 3513 /* Process Rx packets bounded by budget */ 3514 while (likely(rs.packets < budget)) { 3515 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; 3516 struct libeth_fqe *hdr, *rx_buf = NULL; 3517 struct idpf_sw_queue *refillq = NULL; 3518 struct idpf_rxq_set *rxq_set = NULL; 3519 unsigned int pkt_len = 0; 3520 unsigned int hdr_len = 0; 3521 u16 gen_id, buf_id = 0; 3522 int bufq_id; 3523 u8 rxdid; 3524 3525 /* get the Rx desc from Rx queue based on 'next_to_clean' */ 3526 rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; 3527 3528 /* if the descriptor isn't done, no work yet to do */ 3529 gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id, 3530 VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M); 3531 if (idpf_queue_has(GEN_CHK, rxq) != gen_id) 3532 break; 3533 3534 dma_rmb(); 3535 3536 rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M, 3537 rx_desc->rxdid_ucast); 3538 if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) { 3539 IDPF_RX_BUMP_NTC(rxq, ntc); 3540 u64_stats_update_begin(&rxq->stats_sync); 3541 u64_stats_inc(&rxq->q_stats.bad_descs); 3542 u64_stats_update_end(&rxq->stats_sync); 3543 continue; 3544 } 3545 3546 pkt_len = le16_get_bits(rx_desc->pktlen_gen_bufq_id, 3547 VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M); 3548 3549 bufq_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id, 3550 VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M); 3551 3552 rxq_set = container_of(rxq, struct idpf_rxq_set, rxq); 3553 refillq = rxq_set->refillq[bufq_id]; 3554 3555 /* retrieve buffer from the rxq */ 3556 rx_bufq = &rxq->bufq_sets[bufq_id].bufq; 3557 3558 buf_id = le16_to_cpu(rx_desc->buf_id); 3559 3560 rx_buf = &rx_bufq->buf[buf_id]; 3561 3562 if (!rx_bufq->hdr_pp) 3563 goto payload; 3564 3565 #define __HBO_BIT VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_HBO_M 3566 #define __HDR_LEN_MASK VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_HDR_M 3567 if (likely(!(rx_desc->status_err0_qw1 & __HBO_BIT))) 3568 /* If a header buffer overflow, occurs, i.e. header is 3569 * too large to fit in the header split buffer, HW will 3570 * put the entire packet, including headers, in the 3571 * data/payload buffer. 3572 */ 3573 hdr_len = le16_get_bits(rx_desc->hdrlen_flags, 3574 __HDR_LEN_MASK); 3575 #undef __HDR_LEN_MASK 3576 #undef __HBO_BIT 3577 3578 hdr = &rx_bufq->hdr_buf[buf_id]; 3579 3580 if (unlikely(!hdr_len && !xdp->data)) { 3581 hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len); 3582 /* If failed, drop both buffers by setting len to 0 */ 3583 pkt_len -= hdr_len ? : pkt_len; 3584 3585 u64_stats_update_begin(&rxq->stats_sync); 3586 u64_stats_inc(&rxq->q_stats.hsplit_buf_ovf); 3587 u64_stats_update_end(&rxq->stats_sync); 3588 } 3589 3590 if (libeth_xdp_process_buff(xdp, hdr, hdr_len)) 3591 rs.hsplit++; 3592 3593 hdr->netmem = 0; 3594 3595 payload: 3596 libeth_xdp_process_buff(xdp, rx_buf, pkt_len); 3597 rx_buf->netmem = 0; 3598 3599 idpf_post_buf_refill(refillq, buf_id); 3600 IDPF_RX_BUMP_NTC(rxq, ntc); 3601 3602 /* skip if it is non EOP desc */ 3603 if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data)) 3604 continue; 3605 3606 idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); 3607 } 3608 3609 idpf_xdp_finalize_rx(&bq); 3610 3611 rxq->next_to_clean = ntc; 3612 libeth_xdp_save_buff(&rxq->xdp, xdp); 3613 3614 u64_stats_update_begin(&rxq->stats_sync); 3615 u64_stats_add(&rxq->q_stats.packets, rs.packets); 3616 u64_stats_add(&rxq->q_stats.bytes, rs.bytes); 3617 u64_stats_add(&rxq->q_stats.hsplit_pkts, rs.hsplit); 3618 u64_stats_update_end(&rxq->stats_sync); 3619 3620 return rs.packets; 3621 } 3622 3623 /** 3624 * idpf_rx_update_bufq_desc - Update buffer queue descriptor 3625 * @bufq: Pointer to the buffer queue 3626 * @buf_id: buffer ID 3627 * @buf_desc: Buffer queue descriptor 3628 * 3629 * Return 0 on success and negative on failure. 3630 */ 3631 static int idpf_rx_update_bufq_desc(struct idpf_buf_queue *bufq, u32 buf_id, 3632 struct virtchnl2_splitq_rx_buf_desc *buf_desc) 3633 { 3634 struct libeth_fq_fp fq = { 3635 .pp = bufq->pp, 3636 .fqes = bufq->buf, 3637 .truesize = bufq->truesize, 3638 .count = bufq->desc_count, 3639 }; 3640 dma_addr_t addr; 3641 3642 addr = libeth_rx_alloc(&fq, buf_id); 3643 if (addr == DMA_MAPPING_ERROR) 3644 return -ENOMEM; 3645 3646 buf_desc->pkt_addr = cpu_to_le64(addr); 3647 buf_desc->qword0.buf_id = cpu_to_le16(buf_id); 3648 3649 if (!idpf_queue_has(HSPLIT_EN, bufq)) 3650 return 0; 3651 3652 fq.pp = bufq->hdr_pp; 3653 fq.fqes = bufq->hdr_buf; 3654 fq.truesize = bufq->hdr_truesize; 3655 3656 addr = libeth_rx_alloc(&fq, buf_id); 3657 if (addr == DMA_MAPPING_ERROR) 3658 return -ENOMEM; 3659 3660 buf_desc->hdr_addr = cpu_to_le64(addr); 3661 3662 return 0; 3663 } 3664 3665 /** 3666 * idpf_rx_clean_refillq - Clean refill queue buffers 3667 * @bufq: buffer queue to post buffers back to 3668 * @refillq: refill queue to clean 3669 * 3670 * This function takes care of the buffer refill management 3671 */ 3672 static void idpf_rx_clean_refillq(struct idpf_buf_queue *bufq, 3673 struct idpf_sw_queue *refillq) 3674 { 3675 struct virtchnl2_splitq_rx_buf_desc *buf_desc; 3676 u16 bufq_nta = bufq->next_to_alloc; 3677 u16 ntc = refillq->next_to_clean; 3678 int cleaned = 0; 3679 3680 buf_desc = &bufq->split_buf[bufq_nta]; 3681 3682 /* make sure we stop at ring wrap in the unlikely case ring is full */ 3683 while (likely(cleaned < refillq->desc_count)) { 3684 u32 buf_id, refill_desc = refillq->ring[ntc]; 3685 bool failure; 3686 3687 if (idpf_queue_has(RFL_GEN_CHK, refillq) != 3688 !!(refill_desc & IDPF_RFL_BI_GEN_M)) 3689 break; 3690 3691 buf_id = FIELD_GET(IDPF_RFL_BI_BUFID_M, refill_desc); 3692 failure = idpf_rx_update_bufq_desc(bufq, buf_id, buf_desc); 3693 if (failure) 3694 break; 3695 3696 if (unlikely(++ntc == refillq->desc_count)) { 3697 idpf_queue_change(RFL_GEN_CHK, refillq); 3698 ntc = 0; 3699 } 3700 3701 if (unlikely(++bufq_nta == bufq->desc_count)) { 3702 buf_desc = &bufq->split_buf[0]; 3703 bufq_nta = 0; 3704 } else { 3705 buf_desc++; 3706 } 3707 3708 cleaned++; 3709 } 3710 3711 if (!cleaned) 3712 return; 3713 3714 /* We want to limit how many transactions on the bus we trigger with 3715 * tail writes so we only do it in strides. It's also important we 3716 * align the write to a multiple of 8 as required by HW. 3717 */ 3718 if (((bufq->next_to_use <= bufq_nta ? 0 : bufq->desc_count) + 3719 bufq_nta - bufq->next_to_use) >= IDPF_RX_BUF_POST_STRIDE) 3720 idpf_rx_buf_hw_update(bufq, ALIGN_DOWN(bufq_nta, 3721 IDPF_RX_BUF_POST_STRIDE)); 3722 3723 /* update next to alloc since we have filled the ring */ 3724 refillq->next_to_clean = ntc; 3725 bufq->next_to_alloc = bufq_nta; 3726 } 3727 3728 /** 3729 * idpf_rx_clean_refillq_all - Clean all refill queues 3730 * @bufq: buffer queue with refill queues 3731 * @nid: ID of the closest NUMA node with memory 3732 * 3733 * Iterates through all refill queues assigned to the buffer queue assigned to 3734 * this vector. Returns true if clean is complete within budget, false 3735 * otherwise. 3736 */ 3737 static void idpf_rx_clean_refillq_all(struct idpf_buf_queue *bufq, int nid) 3738 { 3739 struct idpf_bufq_set *bufq_set; 3740 int i; 3741 3742 page_pool_nid_changed(bufq->pp, nid); 3743 if (bufq->hdr_pp) 3744 page_pool_nid_changed(bufq->hdr_pp, nid); 3745 3746 bufq_set = container_of(bufq, struct idpf_bufq_set, bufq); 3747 for (i = 0; i < bufq_set->num_refillqs; i++) 3748 idpf_rx_clean_refillq(bufq, &bufq_set->refillqs[i]); 3749 } 3750 3751 /** 3752 * idpf_vport_intr_clean_queues - MSIX mode Interrupt Handler 3753 * @irq: interrupt number 3754 * @data: pointer to a q_vector 3755 * 3756 */ 3757 static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, 3758 void *data) 3759 { 3760 struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data; 3761 3762 q_vector->total_events++; 3763 napi_schedule_irqoff(&q_vector->napi); 3764 3765 return IRQ_HANDLED; 3766 } 3767 3768 /** 3769 * idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport 3770 * @vport: virtual port structure 3771 * 3772 */ 3773 static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport) 3774 { 3775 u16 v_idx; 3776 3777 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) 3778 netif_napi_del(&vport->q_vectors[v_idx].napi); 3779 } 3780 3781 /** 3782 * idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport 3783 * @vport: main vport structure 3784 */ 3785 static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport) 3786 { 3787 int v_idx; 3788 3789 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) 3790 napi_disable(&vport->q_vectors[v_idx].napi); 3791 } 3792 3793 /** 3794 * idpf_vport_intr_rel - Free memory allocated for interrupt vectors 3795 * @vport: virtual port 3796 * 3797 * Free the memory allocated for interrupt vectors associated to a vport 3798 */ 3799 void idpf_vport_intr_rel(struct idpf_vport *vport) 3800 { 3801 for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { 3802 struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; 3803 3804 kfree(q_vector->xsksq); 3805 q_vector->xsksq = NULL; 3806 kfree(q_vector->complq); 3807 q_vector->complq = NULL; 3808 kfree(q_vector->bufq); 3809 q_vector->bufq = NULL; 3810 kfree(q_vector->tx); 3811 q_vector->tx = NULL; 3812 kfree(q_vector->rx); 3813 q_vector->rx = NULL; 3814 } 3815 3816 kfree(vport->q_vectors); 3817 vport->q_vectors = NULL; 3818 } 3819 3820 static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link) 3821 { 3822 struct napi_struct *napi = link ? &q_vector->napi : NULL; 3823 struct net_device *dev = q_vector->vport->netdev; 3824 3825 for (u32 i = 0; i < q_vector->num_rxq; i++) 3826 netif_queue_set_napi(dev, q_vector->rx[i]->idx, 3827 NETDEV_QUEUE_TYPE_RX, napi); 3828 3829 for (u32 i = 0; i < q_vector->num_txq; i++) 3830 netif_queue_set_napi(dev, q_vector->tx[i]->idx, 3831 NETDEV_QUEUE_TYPE_TX, napi); 3832 } 3833 3834 /** 3835 * idpf_vport_intr_rel_irq - Free the IRQ association with the OS 3836 * @vport: main vport structure 3837 */ 3838 static void idpf_vport_intr_rel_irq(struct idpf_vport *vport) 3839 { 3840 struct idpf_adapter *adapter = vport->adapter; 3841 int vector; 3842 3843 for (vector = 0; vector < vport->num_q_vectors; vector++) { 3844 struct idpf_q_vector *q_vector = &vport->q_vectors[vector]; 3845 int irq_num, vidx; 3846 3847 /* free only the irqs that were actually requested */ 3848 if (!q_vector) 3849 continue; 3850 3851 vidx = vport->q_vector_idxs[vector]; 3852 irq_num = adapter->msix_entries[vidx].vector; 3853 3854 idpf_q_vector_set_napi(q_vector, false); 3855 kfree(free_irq(irq_num, q_vector)); 3856 } 3857 } 3858 3859 /** 3860 * idpf_vport_intr_dis_irq_all - Disable all interrupt 3861 * @vport: main vport structure 3862 */ 3863 static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport) 3864 { 3865 struct idpf_q_vector *q_vector = vport->q_vectors; 3866 int q_idx; 3867 3868 writel(0, vport->noirq_dyn_ctl); 3869 3870 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) 3871 writel(0, q_vector[q_idx].intr_reg.dyn_ctl); 3872 } 3873 3874 /** 3875 * idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings 3876 * @q_vector: pointer to q_vector 3877 */ 3878 static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector) 3879 { 3880 u32 itr_val = q_vector->intr_reg.dyn_ctl_intena_m; 3881 int type = IDPF_NO_ITR_UPDATE_IDX; 3882 u16 itr = 0; 3883 3884 if (q_vector->wb_on_itr) { 3885 /* 3886 * Trigger a software interrupt when exiting wb_on_itr, to make 3887 * sure we catch any pending write backs that might have been 3888 * missed due to interrupt state transition. 3889 */ 3890 itr_val |= q_vector->intr_reg.dyn_ctl_swint_trig_m | 3891 q_vector->intr_reg.dyn_ctl_sw_itridx_ena_m; 3892 type = IDPF_SW_ITR_UPDATE_IDX; 3893 itr = IDPF_ITR_20K; 3894 } 3895 3896 itr &= IDPF_ITR_MASK; 3897 /* Don't clear PBA because that can cause lost interrupts that 3898 * came in while we were cleaning/polling 3899 */ 3900 itr_val |= (type << q_vector->intr_reg.dyn_ctl_itridx_s) | 3901 (itr << (q_vector->intr_reg.dyn_ctl_intrvl_s - 1)); 3902 3903 return itr_val; 3904 } 3905 3906 /** 3907 * idpf_update_dim_sample - Update dim sample with packets and bytes 3908 * @q_vector: the vector associated with the interrupt 3909 * @dim_sample: dim sample to update 3910 * @dim: dim instance structure 3911 * @packets: total packets 3912 * @bytes: total bytes 3913 * 3914 * Update the dim sample with the packets and bytes which are passed to this 3915 * function. Set the dim state appropriately if the dim settings gets stale. 3916 */ 3917 static void idpf_update_dim_sample(struct idpf_q_vector *q_vector, 3918 struct dim_sample *dim_sample, 3919 struct dim *dim, u64 packets, u64 bytes) 3920 { 3921 dim_update_sample(q_vector->total_events, packets, bytes, dim_sample); 3922 dim_sample->comp_ctr = 0; 3923 3924 /* if dim settings get stale, like when not updated for 1 second or 3925 * longer, force it to start again. This addresses the frequent case 3926 * of an idle queue being switched to by the scheduler. 3927 */ 3928 if (ktime_ms_delta(dim_sample->time, dim->start_sample.time) >= HZ) 3929 dim->state = DIM_START_MEASURE; 3930 } 3931 3932 /** 3933 * idpf_net_dim - Update net DIM algorithm 3934 * @q_vector: the vector associated with the interrupt 3935 * 3936 * Create a DIM sample and notify net_dim() so that it can possibly decide 3937 * a new ITR value based on incoming packets, bytes, and interrupts. 3938 * 3939 * This function is a no-op if the queue is not configured to dynamic ITR. 3940 */ 3941 static void idpf_net_dim(struct idpf_q_vector *q_vector) 3942 { 3943 struct dim_sample dim_sample = { }; 3944 u64 packets, bytes, pkts, bts; 3945 u32 i; 3946 3947 if (!IDPF_ITR_IS_DYNAMIC(q_vector->tx_intr_mode)) 3948 goto check_rx_itr; 3949 3950 for (i = 0, packets = 0, bytes = 0; i < q_vector->num_txq; i++) { 3951 struct idpf_tx_queue *txq = q_vector->tx[i]; 3952 unsigned int start; 3953 3954 do { 3955 start = u64_stats_fetch_begin(&txq->stats_sync); 3956 pkts = u64_stats_read(&txq->q_stats.packets); 3957 bts = u64_stats_read(&txq->q_stats.bytes); 3958 } while (u64_stats_fetch_retry(&txq->stats_sync, start)); 3959 3960 packets += pkts; 3961 bytes += bts; 3962 } 3963 3964 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim, 3965 packets, bytes); 3966 net_dim(&q_vector->tx_dim, &dim_sample); 3967 3968 check_rx_itr: 3969 if (!IDPF_ITR_IS_DYNAMIC(q_vector->rx_intr_mode)) 3970 return; 3971 3972 for (i = 0, packets = 0, bytes = 0; i < q_vector->num_rxq; i++) { 3973 struct idpf_rx_queue *rxq = q_vector->rx[i]; 3974 unsigned int start; 3975 3976 do { 3977 start = u64_stats_fetch_begin(&rxq->stats_sync); 3978 pkts = u64_stats_read(&rxq->q_stats.packets); 3979 bts = u64_stats_read(&rxq->q_stats.bytes); 3980 } while (u64_stats_fetch_retry(&rxq->stats_sync, start)); 3981 3982 packets += pkts; 3983 bytes += bts; 3984 } 3985 3986 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim, 3987 packets, bytes); 3988 net_dim(&q_vector->rx_dim, &dim_sample); 3989 } 3990 3991 /** 3992 * idpf_vport_intr_update_itr_ena_irq - Update itr and re-enable MSIX interrupt 3993 * @q_vector: q_vector for which itr is being updated and interrupt enabled 3994 * 3995 * Update the net_dim() algorithm and re-enable the interrupt associated with 3996 * this vector. 3997 */ 3998 void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) 3999 { 4000 u32 intval; 4001 4002 /* net_dim() updates ITR out-of-band using a work item */ 4003 idpf_net_dim(q_vector); 4004 4005 intval = idpf_vport_intr_buildreg_itr(q_vector); 4006 q_vector->wb_on_itr = false; 4007 4008 writel(intval, q_vector->intr_reg.dyn_ctl); 4009 } 4010 4011 /** 4012 * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport 4013 * @vport: main vport structure 4014 */ 4015 static int idpf_vport_intr_req_irq(struct idpf_vport *vport) 4016 { 4017 struct idpf_adapter *adapter = vport->adapter; 4018 const char *drv_name, *if_name, *vec_name; 4019 int vector, err, irq_num, vidx; 4020 4021 drv_name = dev_driver_string(&adapter->pdev->dev); 4022 if_name = netdev_name(vport->netdev); 4023 4024 for (vector = 0; vector < vport->num_q_vectors; vector++) { 4025 struct idpf_q_vector *q_vector = &vport->q_vectors[vector]; 4026 char *name; 4027 4028 vidx = vport->q_vector_idxs[vector]; 4029 irq_num = adapter->msix_entries[vidx].vector; 4030 4031 if (q_vector->num_rxq && q_vector->num_txq) 4032 vec_name = "TxRx"; 4033 else if (q_vector->num_rxq) 4034 vec_name = "Rx"; 4035 else if (q_vector->num_txq) 4036 vec_name = "Tx"; 4037 else 4038 continue; 4039 4040 name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name, 4041 vec_name, vidx); 4042 4043 err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0, 4044 name, q_vector); 4045 if (err) { 4046 netdev_err(vport->netdev, 4047 "Request_irq failed, error: %d\n", err); 4048 goto free_q_irqs; 4049 } 4050 4051 idpf_q_vector_set_napi(q_vector, true); 4052 } 4053 4054 return 0; 4055 4056 free_q_irqs: 4057 while (--vector >= 0) { 4058 vidx = vport->q_vector_idxs[vector]; 4059 irq_num = adapter->msix_entries[vidx].vector; 4060 kfree(free_irq(irq_num, &vport->q_vectors[vector])); 4061 } 4062 4063 return err; 4064 } 4065 4066 /** 4067 * idpf_vport_intr_write_itr - Write ITR value to the ITR register 4068 * @q_vector: q_vector structure 4069 * @itr: Interrupt throttling rate 4070 * @tx: Tx or Rx ITR 4071 */ 4072 void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx) 4073 { 4074 struct idpf_intr_reg *intr_reg; 4075 4076 if (tx && !q_vector->tx) 4077 return; 4078 else if (!tx && !q_vector->rx) 4079 return; 4080 4081 intr_reg = &q_vector->intr_reg; 4082 writel(ITR_REG_ALIGN(itr) >> IDPF_ITR_GRAN_S, 4083 tx ? intr_reg->tx_itr : intr_reg->rx_itr); 4084 } 4085 4086 /** 4087 * idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport 4088 * @vport: main vport structure 4089 */ 4090 static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport) 4091 { 4092 bool dynamic; 4093 int q_idx; 4094 u16 itr; 4095 4096 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) { 4097 struct idpf_q_vector *qv = &vport->q_vectors[q_idx]; 4098 4099 /* Set the initial ITR values */ 4100 if (qv->num_txq) { 4101 dynamic = IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode); 4102 itr = vport->tx_itr_profile[qv->tx_dim.profile_ix]; 4103 idpf_vport_intr_write_itr(qv, dynamic ? 4104 itr : qv->tx_itr_value, 4105 true); 4106 } 4107 4108 if (qv->num_rxq) { 4109 dynamic = IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode); 4110 itr = vport->rx_itr_profile[qv->rx_dim.profile_ix]; 4111 idpf_vport_intr_write_itr(qv, dynamic ? 4112 itr : qv->rx_itr_value, 4113 false); 4114 } 4115 4116 if (qv->num_txq || qv->num_rxq) 4117 idpf_vport_intr_update_itr_ena_irq(qv); 4118 } 4119 4120 writel(vport->noirq_dyn_ctl_ena, vport->noirq_dyn_ctl); 4121 } 4122 4123 /** 4124 * idpf_vport_intr_deinit - Release all vector associations for the vport 4125 * @vport: main vport structure 4126 */ 4127 void idpf_vport_intr_deinit(struct idpf_vport *vport) 4128 { 4129 idpf_vport_intr_dis_irq_all(vport); 4130 idpf_vport_intr_napi_dis_all(vport); 4131 idpf_vport_intr_napi_del_all(vport); 4132 idpf_vport_intr_rel_irq(vport); 4133 } 4134 4135 /** 4136 * idpf_tx_dim_work - Call back from the stack 4137 * @work: work queue structure 4138 */ 4139 static void idpf_tx_dim_work(struct work_struct *work) 4140 { 4141 struct idpf_q_vector *q_vector; 4142 struct idpf_vport *vport; 4143 struct dim *dim; 4144 u16 itr; 4145 4146 dim = container_of(work, struct dim, work); 4147 q_vector = container_of(dim, struct idpf_q_vector, tx_dim); 4148 vport = q_vector->vport; 4149 4150 if (dim->profile_ix >= ARRAY_SIZE(vport->tx_itr_profile)) 4151 dim->profile_ix = ARRAY_SIZE(vport->tx_itr_profile) - 1; 4152 4153 /* look up the values in our local table */ 4154 itr = vport->tx_itr_profile[dim->profile_ix]; 4155 4156 idpf_vport_intr_write_itr(q_vector, itr, true); 4157 4158 dim->state = DIM_START_MEASURE; 4159 } 4160 4161 /** 4162 * idpf_rx_dim_work - Call back from the stack 4163 * @work: work queue structure 4164 */ 4165 static void idpf_rx_dim_work(struct work_struct *work) 4166 { 4167 struct idpf_q_vector *q_vector; 4168 struct idpf_vport *vport; 4169 struct dim *dim; 4170 u16 itr; 4171 4172 dim = container_of(work, struct dim, work); 4173 q_vector = container_of(dim, struct idpf_q_vector, rx_dim); 4174 vport = q_vector->vport; 4175 4176 if (dim->profile_ix >= ARRAY_SIZE(vport->rx_itr_profile)) 4177 dim->profile_ix = ARRAY_SIZE(vport->rx_itr_profile) - 1; 4178 4179 /* look up the values in our local table */ 4180 itr = vport->rx_itr_profile[dim->profile_ix]; 4181 4182 idpf_vport_intr_write_itr(q_vector, itr, false); 4183 4184 dim->state = DIM_START_MEASURE; 4185 } 4186 4187 /** 4188 * idpf_init_dim - Set up dynamic interrupt moderation 4189 * @qv: q_vector structure 4190 */ 4191 static void idpf_init_dim(struct idpf_q_vector *qv) 4192 { 4193 INIT_WORK(&qv->tx_dim.work, idpf_tx_dim_work); 4194 qv->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 4195 qv->tx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX; 4196 4197 INIT_WORK(&qv->rx_dim.work, idpf_rx_dim_work); 4198 qv->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 4199 qv->rx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX; 4200 } 4201 4202 /** 4203 * idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport 4204 * @vport: main vport structure 4205 */ 4206 static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport) 4207 { 4208 int q_idx; 4209 4210 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) { 4211 struct idpf_q_vector *q_vector = &vport->q_vectors[q_idx]; 4212 4213 idpf_init_dim(q_vector); 4214 napi_enable(&q_vector->napi); 4215 } 4216 } 4217 4218 /** 4219 * idpf_tx_splitq_clean_all- Clean completion queues 4220 * @q_vec: queue vector 4221 * @budget: Used to determine if we are in netpoll 4222 * @cleaned: returns number of packets cleaned 4223 * 4224 * Returns false if clean is not complete else returns true 4225 */ 4226 static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec, 4227 int budget, int *cleaned) 4228 { 4229 u16 num_complq = q_vec->num_complq; 4230 bool clean_complete = true; 4231 int i, budget_per_q; 4232 4233 if (unlikely(!num_complq)) 4234 return true; 4235 4236 budget_per_q = DIV_ROUND_UP(budget, num_complq); 4237 4238 for (i = 0; i < num_complq; i++) 4239 clean_complete &= idpf_tx_clean_complq(q_vec->complq[i], 4240 budget_per_q, cleaned); 4241 4242 return clean_complete; 4243 } 4244 4245 /** 4246 * idpf_rx_splitq_clean_all- Clean completion queues 4247 * @q_vec: queue vector 4248 * @budget: Used to determine if we are in netpoll 4249 * @cleaned: returns number of packets cleaned 4250 * 4251 * Returns false if clean is not complete else returns true 4252 */ 4253 static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, 4254 int *cleaned) 4255 { 4256 u16 num_rxq = q_vec->num_rxq; 4257 bool clean_complete = true; 4258 int pkts_cleaned = 0; 4259 int i, budget_per_q; 4260 int nid; 4261 4262 /* We attempt to distribute budget to each Rx queue fairly, but don't 4263 * allow the budget to go below 1 because that would exit polling early. 4264 */ 4265 budget_per_q = num_rxq ? max(budget / num_rxq, 1) : 0; 4266 for (i = 0; i < num_rxq; i++) { 4267 struct idpf_rx_queue *rxq = q_vec->rx[i]; 4268 int pkts_cleaned_per_q; 4269 4270 pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ? 4271 idpf_xskrq_poll(rxq, budget_per_q) : 4272 idpf_rx_splitq_clean(rxq, budget_per_q); 4273 /* if we clean as many as budgeted, we must not be done */ 4274 if (pkts_cleaned_per_q >= budget_per_q) 4275 clean_complete = false; 4276 pkts_cleaned += pkts_cleaned_per_q; 4277 } 4278 *cleaned = pkts_cleaned; 4279 4280 nid = numa_mem_id(); 4281 4282 for (i = 0; i < q_vec->num_bufq; i++) { 4283 if (!idpf_queue_has(XSK, q_vec->bufq[i])) 4284 idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); 4285 } 4286 4287 return clean_complete; 4288 } 4289 4290 /** 4291 * idpf_vport_splitq_napi_poll - NAPI handler 4292 * @napi: struct from which you get q_vector 4293 * @budget: budget provided by stack 4294 */ 4295 static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) 4296 { 4297 struct idpf_q_vector *q_vector = 4298 container_of(napi, struct idpf_q_vector, napi); 4299 bool clean_complete = true; 4300 int work_done = 0; 4301 4302 /* Handle case where we are called by netpoll with a budget of 0 */ 4303 if (unlikely(!budget)) { 4304 idpf_tx_splitq_clean_all(q_vector, budget, &work_done); 4305 4306 return 0; 4307 } 4308 4309 for (u32 i = 0; i < q_vector->num_xsksq; i++) 4310 clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]); 4311 4312 clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, 4313 &work_done); 4314 clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget, 4315 &work_done); 4316 4317 /* If work not completed, return budget and polling will return */ 4318 if (!clean_complete) { 4319 idpf_vport_intr_set_wb_on_itr(q_vector); 4320 return budget; 4321 } 4322 4323 work_done = min_t(int, work_done, budget - 1); 4324 4325 /* Exit the polling mode, but don't re-enable interrupts if stack might 4326 * poll us due to busy-polling 4327 */ 4328 if (napi_complete_done(napi, work_done)) 4329 idpf_vport_intr_update_itr_ena_irq(q_vector); 4330 else 4331 idpf_vport_intr_set_wb_on_itr(q_vector); 4332 4333 return work_done; 4334 } 4335 4336 /** 4337 * idpf_vport_intr_map_vector_to_qs - Map vectors to queues 4338 * @vport: virtual port 4339 * 4340 * Mapping for vectors to queues 4341 */ 4342 static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) 4343 { 4344 u16 num_txq_grp = vport->num_txq_grp - vport->num_xdp_txq; 4345 bool split = idpf_is_queue_model_split(vport->rxq_model); 4346 struct idpf_rxq_group *rx_qgrp; 4347 struct idpf_txq_group *tx_qgrp; 4348 u32 i, qv_idx, q_index; 4349 4350 for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) { 4351 u16 num_rxq; 4352 4353 if (qv_idx >= vport->num_q_vectors) 4354 qv_idx = 0; 4355 4356 rx_qgrp = &vport->rxq_grps[i]; 4357 if (split) 4358 num_rxq = rx_qgrp->splitq.num_rxq_sets; 4359 else 4360 num_rxq = rx_qgrp->singleq.num_rxq; 4361 4362 for (u32 j = 0; j < num_rxq; j++) { 4363 struct idpf_rx_queue *q; 4364 4365 if (split) 4366 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 4367 else 4368 q = rx_qgrp->singleq.rxqs[j]; 4369 q->q_vector = &vport->q_vectors[qv_idx]; 4370 q_index = q->q_vector->num_rxq; 4371 q->q_vector->rx[q_index] = q; 4372 q->q_vector->num_rxq++; 4373 4374 if (split) 4375 q->napi = &q->q_vector->napi; 4376 } 4377 4378 if (split) { 4379 for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { 4380 struct idpf_buf_queue *bufq; 4381 4382 bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 4383 bufq->q_vector = &vport->q_vectors[qv_idx]; 4384 q_index = bufq->q_vector->num_bufq; 4385 bufq->q_vector->bufq[q_index] = bufq; 4386 bufq->q_vector->num_bufq++; 4387 } 4388 } 4389 4390 qv_idx++; 4391 } 4392 4393 split = idpf_is_queue_model_split(vport->txq_model); 4394 4395 for (i = 0, qv_idx = 0; i < num_txq_grp; i++) { 4396 u16 num_txq; 4397 4398 if (qv_idx >= vport->num_q_vectors) 4399 qv_idx = 0; 4400 4401 tx_qgrp = &vport->txq_grps[i]; 4402 num_txq = tx_qgrp->num_txq; 4403 4404 for (u32 j = 0; j < num_txq; j++) { 4405 struct idpf_tx_queue *q; 4406 4407 q = tx_qgrp->txqs[j]; 4408 q->q_vector = &vport->q_vectors[qv_idx]; 4409 q->q_vector->tx[q->q_vector->num_txq++] = q; 4410 } 4411 4412 if (split) { 4413 struct idpf_compl_queue *q = tx_qgrp->complq; 4414 4415 q->q_vector = &vport->q_vectors[qv_idx]; 4416 q->q_vector->complq[q->q_vector->num_complq++] = q; 4417 } 4418 4419 qv_idx++; 4420 } 4421 4422 for (i = 0; i < vport->num_xdp_txq; i++) { 4423 struct idpf_tx_queue *xdpsq; 4424 struct idpf_q_vector *qv; 4425 4426 xdpsq = vport->txqs[vport->xdp_txq_offset + i]; 4427 if (!idpf_queue_has(XSK, xdpsq)) 4428 continue; 4429 4430 qv = idpf_find_rxq_vec(vport, i); 4431 idpf_xsk_init_wakeup(qv); 4432 4433 xdpsq->q_vector = qv; 4434 qv->xsksq[qv->num_xsksq++] = xdpsq; 4435 } 4436 } 4437 4438 /** 4439 * idpf_vport_intr_init_vec_idx - Initialize the vector indexes 4440 * @vport: virtual port 4441 * 4442 * Initialize vector indexes with values returened over mailbox 4443 */ 4444 static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) 4445 { 4446 struct idpf_adapter *adapter = vport->adapter; 4447 struct virtchnl2_alloc_vectors *ac; 4448 u16 *vecids, total_vecs; 4449 int i; 4450 4451 ac = adapter->req_vec_chunks; 4452 if (!ac) { 4453 for (i = 0; i < vport->num_q_vectors; i++) 4454 vport->q_vectors[i].v_idx = vport->q_vector_idxs[i]; 4455 4456 vport->noirq_v_idx = vport->q_vector_idxs[i]; 4457 4458 return 0; 4459 } 4460 4461 total_vecs = idpf_get_reserved_vecs(adapter); 4462 vecids = kcalloc(total_vecs, sizeof(u16), GFP_KERNEL); 4463 if (!vecids) 4464 return -ENOMEM; 4465 4466 idpf_get_vec_ids(adapter, vecids, total_vecs, &ac->vchunks); 4467 4468 for (i = 0; i < vport->num_q_vectors; i++) 4469 vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]]; 4470 4471 vport->noirq_v_idx = vecids[vport->q_vector_idxs[i]]; 4472 4473 kfree(vecids); 4474 4475 return 0; 4476 } 4477 4478 /** 4479 * idpf_vport_intr_napi_add_all- Register napi handler for all qvectors 4480 * @vport: virtual port structure 4481 */ 4482 static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport) 4483 { 4484 int (*napi_poll)(struct napi_struct *napi, int budget); 4485 u16 v_idx, qv_idx; 4486 int irq_num; 4487 4488 if (idpf_is_queue_model_split(vport->txq_model)) 4489 napi_poll = idpf_vport_splitq_napi_poll; 4490 else 4491 napi_poll = idpf_vport_singleq_napi_poll; 4492 4493 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { 4494 struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; 4495 qv_idx = vport->q_vector_idxs[v_idx]; 4496 irq_num = vport->adapter->msix_entries[qv_idx].vector; 4497 4498 netif_napi_add_config(vport->netdev, &q_vector->napi, 4499 napi_poll, v_idx); 4500 netif_napi_set_irq(&q_vector->napi, irq_num); 4501 } 4502 } 4503 4504 /** 4505 * idpf_vport_intr_alloc - Allocate memory for interrupt vectors 4506 * @vport: virtual port 4507 * 4508 * We allocate one q_vector per queue interrupt. If allocation fails we 4509 * return -ENOMEM. 4510 */ 4511 int idpf_vport_intr_alloc(struct idpf_vport *vport) 4512 { 4513 u16 txqs_per_vector, rxqs_per_vector, bufqs_per_vector; 4514 struct idpf_vport_user_config_data *user_config; 4515 struct idpf_q_vector *q_vector; 4516 struct idpf_q_coalesce *q_coal; 4517 u32 complqs_per_vector, v_idx; 4518 u16 idx = vport->idx; 4519 4520 user_config = &vport->adapter->vport_config[idx]->user_config; 4521 vport->q_vectors = kcalloc(vport->num_q_vectors, 4522 sizeof(struct idpf_q_vector), GFP_KERNEL); 4523 if (!vport->q_vectors) 4524 return -ENOMEM; 4525 4526 txqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp, 4527 vport->num_q_vectors); 4528 rxqs_per_vector = DIV_ROUND_UP(vport->num_rxq_grp, 4529 vport->num_q_vectors); 4530 bufqs_per_vector = vport->num_bufqs_per_qgrp * 4531 DIV_ROUND_UP(vport->num_rxq_grp, 4532 vport->num_q_vectors); 4533 complqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp, 4534 vport->num_q_vectors); 4535 4536 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { 4537 q_vector = &vport->q_vectors[v_idx]; 4538 q_coal = &user_config->q_coalesce[v_idx]; 4539 q_vector->vport = vport; 4540 4541 q_vector->tx_itr_value = q_coal->tx_coalesce_usecs; 4542 q_vector->tx_intr_mode = q_coal->tx_intr_mode; 4543 q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1; 4544 4545 q_vector->rx_itr_value = q_coal->rx_coalesce_usecs; 4546 q_vector->rx_intr_mode = q_coal->rx_intr_mode; 4547 q_vector->rx_itr_idx = VIRTCHNL2_ITR_IDX_0; 4548 4549 q_vector->tx = kcalloc(txqs_per_vector, sizeof(*q_vector->tx), 4550 GFP_KERNEL); 4551 if (!q_vector->tx) 4552 goto error; 4553 4554 q_vector->rx = kcalloc(rxqs_per_vector, sizeof(*q_vector->rx), 4555 GFP_KERNEL); 4556 if (!q_vector->rx) 4557 goto error; 4558 4559 if (!idpf_is_queue_model_split(vport->rxq_model)) 4560 continue; 4561 4562 q_vector->bufq = kcalloc(bufqs_per_vector, 4563 sizeof(*q_vector->bufq), 4564 GFP_KERNEL); 4565 if (!q_vector->bufq) 4566 goto error; 4567 4568 q_vector->complq = kcalloc(complqs_per_vector, 4569 sizeof(*q_vector->complq), 4570 GFP_KERNEL); 4571 if (!q_vector->complq) 4572 goto error; 4573 4574 if (!vport->xdp_txq_offset) 4575 continue; 4576 4577 q_vector->xsksq = kcalloc(rxqs_per_vector, 4578 sizeof(*q_vector->xsksq), 4579 GFP_KERNEL); 4580 if (!q_vector->xsksq) 4581 goto error; 4582 } 4583 4584 return 0; 4585 4586 error: 4587 idpf_vport_intr_rel(vport); 4588 4589 return -ENOMEM; 4590 } 4591 4592 /** 4593 * idpf_vport_intr_init - Setup all vectors for the given vport 4594 * @vport: virtual port 4595 * 4596 * Returns 0 on success or negative on failure 4597 */ 4598 int idpf_vport_intr_init(struct idpf_vport *vport) 4599 { 4600 int err; 4601 4602 err = idpf_vport_intr_init_vec_idx(vport); 4603 if (err) 4604 return err; 4605 4606 idpf_vport_intr_map_vector_to_qs(vport); 4607 idpf_vport_intr_napi_add_all(vport); 4608 4609 err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport); 4610 if (err) 4611 goto unroll_vectors_alloc; 4612 4613 err = idpf_vport_intr_req_irq(vport); 4614 if (err) 4615 goto unroll_vectors_alloc; 4616 4617 return 0; 4618 4619 unroll_vectors_alloc: 4620 idpf_vport_intr_napi_del_all(vport); 4621 4622 return err; 4623 } 4624 4625 void idpf_vport_intr_ena(struct idpf_vport *vport) 4626 { 4627 idpf_vport_intr_napi_ena_all(vport); 4628 idpf_vport_intr_ena_irq_all(vport); 4629 } 4630 4631 /** 4632 * idpf_config_rss - Send virtchnl messages to configure RSS 4633 * @vport: virtual port 4634 * 4635 * Return 0 on success, negative on failure 4636 */ 4637 int idpf_config_rss(struct idpf_vport *vport) 4638 { 4639 int err; 4640 4641 err = idpf_send_get_set_rss_key_msg(vport, false); 4642 if (err) 4643 return err; 4644 4645 return idpf_send_get_set_rss_lut_msg(vport, false); 4646 } 4647 4648 /** 4649 * idpf_fill_dflt_rss_lut - Fill the indirection table with the default values 4650 * @vport: virtual port structure 4651 */ 4652 void idpf_fill_dflt_rss_lut(struct idpf_vport *vport) 4653 { 4654 struct idpf_adapter *adapter = vport->adapter; 4655 u16 num_active_rxq = vport->num_rxq; 4656 struct idpf_rss_data *rss_data; 4657 int i; 4658 4659 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; 4660 4661 for (i = 0; i < rss_data->rss_lut_size; i++) 4662 rss_data->rss_lut[i] = i % num_active_rxq; 4663 } 4664 4665 /** 4666 * idpf_init_rss_lut - Allocate and initialize RSS LUT 4667 * @vport: virtual port 4668 * 4669 * Return: 0 on success, negative on failure 4670 */ 4671 int idpf_init_rss_lut(struct idpf_vport *vport) 4672 { 4673 struct idpf_adapter *adapter = vport->adapter; 4674 struct idpf_rss_data *rss_data; 4675 4676 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; 4677 if (!rss_data->rss_lut) { 4678 u32 lut_size; 4679 4680 lut_size = rss_data->rss_lut_size * sizeof(u32); 4681 rss_data->rss_lut = kzalloc(lut_size, GFP_KERNEL); 4682 if (!rss_data->rss_lut) 4683 return -ENOMEM; 4684 } 4685 4686 /* Fill the default RSS lut values */ 4687 idpf_fill_dflt_rss_lut(vport); 4688 4689 return 0; 4690 } 4691 4692 /** 4693 * idpf_deinit_rss_lut - Release RSS LUT 4694 * @vport: virtual port 4695 */ 4696 void idpf_deinit_rss_lut(struct idpf_vport *vport) 4697 { 4698 struct idpf_adapter *adapter = vport->adapter; 4699 struct idpf_rss_data *rss_data; 4700 4701 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data; 4702 kfree(rss_data->rss_lut); 4703 rss_data->rss_lut = NULL; 4704 } 4705