1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #include <net/libeth/xsk.h> 5 6 #include "idpf.h" 7 #include "xdp.h" 8 #include "xsk.h" 9 10 static void idpf_xsk_tx_timer(struct work_struct *work); 11 12 static void idpf_xsk_setup_rxq(const struct idpf_vport *vport, 13 struct idpf_rx_queue *rxq) 14 { 15 struct xsk_buff_pool *pool; 16 17 pool = xsk_get_pool_from_qid(vport->netdev, rxq->idx); 18 if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) 19 return; 20 21 rxq->pool = pool; 22 23 idpf_queue_set(XSK, rxq); 24 } 25 26 static void idpf_xsk_setup_bufq(const struct idpf_vport *vport, 27 struct idpf_buf_queue *bufq) 28 { 29 struct xsk_buff_pool *pool; 30 u32 qid = U32_MAX; 31 32 for (u32 i = 0; i < vport->num_rxq_grp; i++) { 33 const struct idpf_rxq_group *grp = &vport->rxq_grps[i]; 34 35 for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { 36 if (&grp->splitq.bufq_sets[j].bufq == bufq) { 37 qid = grp->splitq.rxq_sets[0]->rxq.idx; 38 goto setup; 39 } 40 } 41 } 42 43 setup: 44 pool = xsk_get_pool_from_qid(vport->netdev, qid); 45 if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) 46 return; 47 48 bufq->pool = pool; 49 50 idpf_queue_set(XSK, bufq); 51 } 52 53 static void idpf_xsk_setup_txq(const struct idpf_vport *vport, 54 struct idpf_tx_queue *txq) 55 { 56 struct xsk_buff_pool *pool; 57 u32 qid; 58 59 idpf_queue_clear(XSK, txq); 60 61 if (!idpf_queue_has(XDP, txq)) 62 return; 63 64 qid = txq->idx - vport->xdp_txq_offset; 65 66 pool = xsk_get_pool_from_qid(vport->netdev, qid); 67 if (!pool || !pool->dev) 68 return; 69 70 txq->pool = pool; 71 libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock, 72 idpf_xsk_tx_timer); 73 74 idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool)); 75 idpf_queue_set(XSK, txq); 76 } 77 78 static void idpf_xsk_setup_complq(const struct idpf_vport *vport, 79 struct idpf_compl_queue *complq) 80 { 81 const struct xsk_buff_pool *pool; 82 u32 qid; 83 84 idpf_queue_clear(XSK, complq); 85 86 if (!idpf_queue_has(XDP, complq)) 87 return; 88 89 qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset; 90 91 pool = xsk_get_pool_from_qid(vport->netdev, qid); 92 if (!pool || !pool->dev) 93 return; 94 95 idpf_queue_set(XSK, complq); 96 } 97 98 void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, 99 enum virtchnl2_queue_type type) 100 { 101 if (!idpf_xdp_enabled(vport)) 102 return; 103 104 switch (type) { 105 case VIRTCHNL2_QUEUE_TYPE_RX: 106 idpf_xsk_setup_rxq(vport, q); 107 break; 108 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 109 idpf_xsk_setup_bufq(vport, q); 110 break; 111 case VIRTCHNL2_QUEUE_TYPE_TX: 112 idpf_xsk_setup_txq(vport, q); 113 break; 114 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 115 idpf_xsk_setup_complq(vport, q); 116 break; 117 default: 118 break; 119 } 120 } 121 122 void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) 123 { 124 struct idpf_compl_queue *complq; 125 struct idpf_buf_queue *bufq; 126 struct idpf_rx_queue *rxq; 127 struct idpf_tx_queue *txq; 128 129 switch (type) { 130 case VIRTCHNL2_QUEUE_TYPE_RX: 131 rxq = q; 132 if (!idpf_queue_has_clear(XSK, rxq)) 133 return; 134 135 rxq->pool = NULL; 136 break; 137 case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 138 bufq = q; 139 if (!idpf_queue_has_clear(XSK, bufq)) 140 return; 141 142 bufq->pool = NULL; 143 break; 144 case VIRTCHNL2_QUEUE_TYPE_TX: 145 txq = q; 146 if (!idpf_queue_has_clear(XSK, txq)) 147 return; 148 149 idpf_queue_set(NOIRQ, txq); 150 txq->dev = txq->netdev->dev.parent; 151 break; 152 case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 153 complq = q; 154 idpf_queue_clear(XSK, complq); 155 break; 156 default: 157 break; 158 } 159 } 160 161 void idpf_xsk_init_wakeup(struct idpf_q_vector *qv) 162 { 163 libeth_xsk_init_wakeup(&qv->csd, &qv->napi); 164 } 165 166 void idpf_xsksq_clean(struct idpf_tx_queue *xdpsq) 167 { 168 struct libeth_xdpsq_napi_stats ss = { }; 169 u32 ntc = xdpsq->next_to_clean; 170 struct xdp_frame_bulk bq; 171 struct libeth_cq_pp cp = { 172 .dev = xdpsq->pool->dev, 173 .bq = &bq, 174 .xss = &ss, 175 }; 176 u32 xsk_frames = 0; 177 178 xdp_frame_bulk_init(&bq); 179 180 while (ntc != xdpsq->next_to_use) { 181 struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; 182 183 if (sqe->type) 184 libeth_xdp_complete_tx(sqe, &cp); 185 else 186 xsk_frames++; 187 188 if (unlikely(++ntc == xdpsq->desc_count)) 189 ntc = 0; 190 } 191 192 xdp_flush_frame_bulk(&bq); 193 194 if (xsk_frames) 195 xsk_tx_completed(xdpsq->pool, xsk_frames); 196 } 197 198 static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq, 199 u32 done) 200 { 201 struct libeth_xdpsq_napi_stats ss = { }; 202 u32 ntc = xdpsq->next_to_clean; 203 u32 cnt = xdpsq->desc_count; 204 struct xdp_frame_bulk bq; 205 struct libeth_cq_pp cp = { 206 .dev = xdpsq->pool->dev, 207 .bq = &bq, 208 .xss = &ss, 209 .napi = true, 210 }; 211 u32 xsk_frames = 0; 212 213 xdp_frame_bulk_init(&bq); 214 215 for (u32 i = 0; likely(i < done); i++) { 216 struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; 217 218 if (sqe->type) 219 libeth_xdp_complete_tx(sqe, &cp); 220 else 221 xsk_frames++; 222 223 if (unlikely(++ntc == cnt)) 224 ntc = 0; 225 } 226 227 xdp_flush_frame_bulk(&bq); 228 229 xdpsq->next_to_clean = ntc; 230 xdpsq->xdp_tx -= cp.xdp_tx; 231 232 return xsk_frames; 233 } 234 235 static __always_inline u32 idpf_xsksq_complete(void *_xdpsq, u32 budget) 236 { 237 struct idpf_tx_queue *xdpsq = _xdpsq; 238 u32 tx_ntc = xdpsq->next_to_clean; 239 u32 tx_cnt = xdpsq->desc_count; 240 u32 done_frames; 241 u32 xsk_frames; 242 243 done_frames = idpf_xdpsq_poll(xdpsq, budget); 244 if (unlikely(!done_frames)) 245 return 0; 246 247 if (likely(!xdpsq->xdp_tx)) { 248 tx_ntc += done_frames; 249 if (tx_ntc >= tx_cnt) 250 tx_ntc -= tx_cnt; 251 252 xdpsq->next_to_clean = tx_ntc; 253 xsk_frames = done_frames; 254 255 goto finalize; 256 } 257 258 xsk_frames = idpf_xsksq_complete_slow(xdpsq, done_frames); 259 if (xsk_frames) 260 finalize: 261 xsk_tx_completed(xdpsq->pool, xsk_frames); 262 263 xdpsq->pending -= done_frames; 264 265 return done_frames; 266 } 267 268 static u32 idpf_xsk_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 269 { 270 struct idpf_tx_queue *xdpsq = _xdpsq; 271 u32 free; 272 273 libeth_xdpsq_lock(&xdpsq->xdp_lock); 274 275 free = xdpsq->desc_count - xdpsq->pending; 276 if (free < xdpsq->thresh) 277 free += idpf_xsksq_complete(xdpsq, xdpsq->thresh); 278 279 *sq = (struct libeth_xdpsq){ 280 .pool = xdpsq->pool, 281 .sqes = xdpsq->tx_buf, 282 .descs = xdpsq->desc_ring, 283 .count = xdpsq->desc_count, 284 .lock = &xdpsq->xdp_lock, 285 .ntu = &xdpsq->next_to_use, 286 .pending = &xdpsq->pending, 287 .xdp_tx = &xdpsq->xdp_tx, 288 }; 289 290 return free; 291 } 292 293 static u32 idpf_xsk_xmit_prep(void *_xdpsq, struct libeth_xdpsq *sq) 294 { 295 struct idpf_tx_queue *xdpsq = _xdpsq; 296 297 *sq = (struct libeth_xdpsq){ 298 .pool = xdpsq->pool, 299 .sqes = xdpsq->tx_buf, 300 .descs = xdpsq->desc_ring, 301 .count = xdpsq->desc_count, 302 .lock = &xdpsq->xdp_lock, 303 .ntu = &xdpsq->next_to_use, 304 .pending = &xdpsq->pending, 305 }; 306 307 /* 308 * The queue is cleaned, the budget is already known, optimize out 309 * the second min() by passing the type limit. 310 */ 311 return U32_MAX; 312 } 313 314 bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq) 315 { 316 u32 free; 317 318 libeth_xdpsq_lock(&xsksq->xdp_lock); 319 320 free = xsksq->desc_count - xsksq->pending; 321 if (free < xsksq->thresh) 322 free += idpf_xsksq_complete(xsksq, xsksq->thresh); 323 324 return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq, 325 min(free - 1, xsksq->thresh), 326 libeth_xsktmo, idpf_xsk_xmit_prep, 327 idpf_xdp_tx_xmit, idpf_xdp_tx_finalize); 328 } 329 330 LIBETH_XDP_DEFINE_START(); 331 LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_xsksq_complete); 332 LIBETH_XSK_DEFINE_FLUSH_TX(static idpf_xsk_tx_flush_bulk, idpf_xsk_tx_prep, 333 idpf_xdp_tx_xmit); 334 LIBETH_XSK_DEFINE_RUN(static idpf_xsk_run_pass, idpf_xsk_run_prog, 335 idpf_xsk_tx_flush_bulk, idpf_rx_process_skb_fields); 336 LIBETH_XSK_DEFINE_FINALIZE(static idpf_xsk_finalize_rx, idpf_xsk_tx_flush_bulk, 337 idpf_xdp_tx_finalize); 338 LIBETH_XDP_DEFINE_END(); 339 340 static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i) 341 { 342 struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs; 343 344 desc = &desc[i]; 345 #ifdef __LIBETH_WORD_ACCESS 346 *(u64 *)&desc->qword0 = i; 347 #else 348 desc->qword0.buf_id = cpu_to_le16(i); 349 #endif 350 desc->pkt_addr = cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); 351 } 352 353 static bool idpf_xskfq_refill_thresh(struct idpf_buf_queue *bufq, u32 count) 354 { 355 struct libeth_xskfq_fp fq = { 356 .pool = bufq->pool, 357 .fqes = bufq->xsk_buf, 358 .descs = bufq->split_buf, 359 .ntu = bufq->next_to_use, 360 .count = bufq->desc_count, 361 }; 362 u32 done; 363 364 done = libeth_xskfqe_alloc(&fq, count, idpf_xskfqe_init); 365 writel(fq.ntu, bufq->tail); 366 367 bufq->next_to_use = fq.ntu; 368 bufq->pending -= done; 369 370 return done == count; 371 } 372 373 static bool idpf_xskfq_refill(struct idpf_buf_queue *bufq) 374 { 375 u32 count, rx_thresh = bufq->thresh; 376 377 count = ALIGN_DOWN(bufq->pending - 1, rx_thresh); 378 379 for (u32 i = 0; i < count; i += rx_thresh) { 380 if (unlikely(!idpf_xskfq_refill_thresh(bufq, rx_thresh))) 381 return false; 382 } 383 384 return true; 385 } 386 387 int idpf_xskfq_init(struct idpf_buf_queue *bufq) 388 { 389 struct libeth_xskfq fq = { 390 .pool = bufq->pool, 391 .count = bufq->desc_count, 392 .nid = idpf_q_vector_to_mem(bufq->q_vector), 393 }; 394 int ret; 395 396 ret = libeth_xskfq_create(&fq); 397 if (ret) 398 return ret; 399 400 bufq->xsk_buf = fq.fqes; 401 bufq->pending = fq.pending; 402 bufq->thresh = fq.thresh; 403 bufq->rx_buf_size = fq.buf_len; 404 405 if (!idpf_xskfq_refill(bufq)) 406 netdev_err(bufq->pool->netdev, 407 "failed to allocate XSk buffers for qid %d\n", 408 bufq->pool->queue_id); 409 410 bufq->next_to_alloc = bufq->next_to_use; 411 412 idpf_queue_clear(HSPLIT_EN, bufq); 413 bufq->rx_hbuf_size = 0; 414 415 return 0; 416 } 417 418 void idpf_xskfq_rel(struct idpf_buf_queue *bufq) 419 { 420 struct libeth_xskfq fq = { 421 .fqes = bufq->xsk_buf, 422 }; 423 424 libeth_xskfq_destroy(&fq); 425 426 bufq->rx_buf_size = fq.buf_len; 427 bufq->thresh = fq.thresh; 428 bufq->pending = fq.pending; 429 } 430 431 struct idpf_xskfq_refill_set { 432 struct { 433 struct idpf_buf_queue *q; 434 u32 buf_id; 435 u32 pending; 436 } bufqs[IDPF_MAX_BUFQS_PER_RXQ_GRP]; 437 }; 438 439 static bool idpf_xskfq_refill_set(const struct idpf_xskfq_refill_set *set) 440 { 441 bool ret = true; 442 443 for (u32 i = 0; i < ARRAY_SIZE(set->bufqs); i++) { 444 struct idpf_buf_queue *bufq = set->bufqs[i].q; 445 u32 ntc; 446 447 if (!bufq) 448 continue; 449 450 ntc = set->bufqs[i].buf_id; 451 if (unlikely(++ntc == bufq->desc_count)) 452 ntc = 0; 453 454 bufq->next_to_clean = ntc; 455 bufq->pending += set->bufqs[i].pending; 456 457 if (bufq->pending > bufq->thresh) 458 ret &= idpf_xskfq_refill(bufq); 459 } 460 461 return ret; 462 } 463 464 int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget) 465 { 466 struct idpf_xskfq_refill_set set = { }; 467 struct libeth_rq_napi_stats rs = { }; 468 bool wake, gen, fail = false; 469 u32 ntc = rxq->next_to_clean; 470 struct libeth_xdp_buff *xdp; 471 LIBETH_XDP_ONSTACK_BULK(bq); 472 u32 cnt = rxq->desc_count; 473 474 wake = xsk_uses_need_wakeup(rxq->pool); 475 if (wake) 476 xsk_clear_rx_need_wakeup(rxq->pool); 477 478 gen = idpf_queue_has(GEN_CHK, rxq); 479 480 libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, 481 rxq->xdpsqs, rxq->num_xdp_txq); 482 xdp = rxq->xsk; 483 484 while (likely(rs.packets < budget)) { 485 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; 486 struct idpf_xdp_rx_desc desc __uninitialized; 487 struct idpf_buf_queue *bufq; 488 u32 bufq_id, buf_id; 489 490 rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; 491 492 idpf_xdp_get_qw0(&desc, rx_desc); 493 if (idpf_xdp_rx_gen(&desc) != gen) 494 break; 495 496 dma_rmb(); 497 498 bufq_id = idpf_xdp_rx_bufq(&desc); 499 bufq = set.bufqs[bufq_id].q; 500 if (!bufq) { 501 bufq = &rxq->bufq_sets[bufq_id].bufq; 502 set.bufqs[bufq_id].q = bufq; 503 } 504 505 idpf_xdp_get_qw1(&desc, rx_desc); 506 buf_id = idpf_xdp_rx_buf(&desc); 507 508 set.bufqs[bufq_id].buf_id = buf_id; 509 set.bufqs[bufq_id].pending++; 510 511 xdp = libeth_xsk_process_buff(xdp, bufq->xsk_buf[buf_id], 512 idpf_xdp_rx_len(&desc)); 513 514 if (unlikely(++ntc == cnt)) { 515 ntc = 0; 516 gen = !gen; 517 idpf_queue_change(GEN_CHK, rxq); 518 } 519 520 if (!idpf_xdp_rx_eop(&desc) || unlikely(!xdp)) 521 continue; 522 523 fail = !idpf_xsk_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); 524 xdp = NULL; 525 526 if (fail) 527 break; 528 } 529 530 idpf_xsk_finalize_rx(&bq); 531 532 rxq->next_to_clean = ntc; 533 rxq->xsk = xdp; 534 535 fail |= !idpf_xskfq_refill_set(&set); 536 537 u64_stats_update_begin(&rxq->stats_sync); 538 u64_stats_add(&rxq->q_stats.packets, rs.packets); 539 u64_stats_add(&rxq->q_stats.bytes, rs.bytes); 540 u64_stats_update_end(&rxq->stats_sync); 541 542 if (!wake) 543 return unlikely(fail) ? budget : rs.packets; 544 545 if (unlikely(fail)) 546 xsk_set_rx_need_wakeup(rxq->pool); 547 548 return rs.packets; 549 } 550 551 int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) 552 { 553 struct xsk_buff_pool *pool = bpf->xsk.pool; 554 u32 qid = bpf->xsk.queue_id; 555 bool restart; 556 int ret; 557 558 if (pool && !IS_ALIGNED(xsk_pool_get_rx_frame_size(pool), 559 LIBETH_RX_BUF_STRIDE)) { 560 NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 561 "%s: HW doesn't support frames sizes not aligned to %u (qid %u: %u)", 562 netdev_name(vport->netdev), 563 LIBETH_RX_BUF_STRIDE, qid, 564 xsk_pool_get_rx_frame_size(pool)); 565 return -EINVAL; 566 } 567 568 restart = idpf_xdp_enabled(vport) && netif_running(vport->netdev); 569 if (!restart) 570 goto pool; 571 572 ret = idpf_qp_switch(vport, qid, false); 573 if (ret) { 574 NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 575 "%s: failed to disable queue pair %u: %pe", 576 netdev_name(vport->netdev), qid, 577 ERR_PTR(ret)); 578 return ret; 579 } 580 581 pool: 582 ret = libeth_xsk_setup_pool(vport->netdev, qid, pool); 583 if (ret) { 584 NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 585 "%s: failed to configure XSk pool for pair %u: %pe", 586 netdev_name(vport->netdev), qid, 587 ERR_PTR(ret)); 588 return ret; 589 } 590 591 if (!restart) 592 return 0; 593 594 ret = idpf_qp_switch(vport, qid, true); 595 if (ret) { 596 NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 597 "%s: failed to enable queue pair %u: %pe", 598 netdev_name(vport->netdev), qid, 599 ERR_PTR(ret)); 600 goto err_dis; 601 } 602 603 return 0; 604 605 err_dis: 606 libeth_xsk_setup_pool(vport->netdev, qid, false); 607 608 return ret; 609 } 610 611 int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) 612 { 613 const struct idpf_netdev_priv *np = netdev_priv(dev); 614 const struct idpf_vport *vport = np->vport; 615 struct idpf_q_vector *q_vector; 616 617 if (unlikely(idpf_vport_ctrl_is_locked(dev))) 618 return -EBUSY; 619 620 if (unlikely(!vport->link_up)) 621 return -ENETDOWN; 622 623 if (unlikely(!vport->num_xdp_txq)) 624 return -ENXIO; 625 626 q_vector = idpf_find_rxq_vec(vport, qid); 627 if (unlikely(!q_vector->xsksq)) 628 return -ENXIO; 629 630 libeth_xsk_wakeup(&q_vector->csd, qid); 631 632 return 0; 633 } 634