1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #include "idpf.h" 5 #include "idpf_ptp.h" 6 #include "idpf_virtchnl.h" 7 #include "xdp.h" 8 #include "xsk.h" 9 10 static int idpf_rxq_for_each(const struct idpf_q_vec_rsrc *rsrc, 11 int (*fn)(struct idpf_rx_queue *rxq, void *arg), 12 void *arg) 13 { 14 bool splitq = idpf_is_queue_model_split(rsrc->rxq_model); 15 16 if (!rsrc->rxq_grps) 17 return -ENETDOWN; 18 19 for (u32 i = 0; i < rsrc->num_rxq_grp; i++) { 20 const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i]; 21 u32 num_rxq; 22 23 if (splitq) 24 num_rxq = rx_qgrp->splitq.num_rxq_sets; 25 else 26 num_rxq = rx_qgrp->singleq.num_rxq; 27 28 for (u32 j = 0; j < num_rxq; j++) { 29 struct idpf_rx_queue *q; 30 int err; 31 32 if (splitq) 33 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 34 else 35 q = rx_qgrp->singleq.rxqs[j]; 36 37 err = fn(q, arg); 38 if (err) 39 return err; 40 } 41 } 42 43 return 0; 44 } 45 46 static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) 47 { 48 const struct idpf_vport *vport = rxq->q_vector->vport; 49 const struct idpf_q_vec_rsrc *rsrc; 50 bool split; 51 int err; 52 53 err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, 54 rxq->q_vector->napi.napi_id, 55 rxq->rx_buf_size); 56 if (err) 57 return err; 58 59 rsrc = &vport->dflt_qv_rsrc; 60 split = idpf_is_queue_model_split(rsrc->rxq_model); 61 62 if (idpf_queue_has(XSK, rxq)) { 63 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, 64 MEM_TYPE_XSK_BUFF_POOL, 65 rxq->pool); 66 if (err) 67 goto unreg; 68 } else { 69 const struct page_pool *pp; 70 71 pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 72 xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 73 } 74 75 if (!split) 76 return 0; 77 78 rxq->xdpsqs = &vport->txqs[rsrc->xdp_txq_offset]; 79 rxq->num_xdp_txq = vport->num_xdp_txq; 80 81 return 0; 82 83 unreg: 84 xdp_rxq_info_unreg(&rxq->xdp_rxq); 85 86 return err; 87 } 88 89 int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) 90 { 91 return __idpf_xdp_rxq_info_init(rxq, NULL); 92 } 93 94 int idpf_xdp_rxq_info_init_all(const struct idpf_q_vec_rsrc *rsrc) 95 { 96 return idpf_rxq_for_each(rsrc, __idpf_xdp_rxq_info_init, NULL); 97 } 98 99 static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) 100 { 101 if (idpf_is_queue_model_split((size_t)arg)) { 102 rxq->xdpsqs = NULL; 103 rxq->num_xdp_txq = 0; 104 } 105 106 if (!idpf_queue_has(XSK, rxq)) 107 xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 108 109 xdp_rxq_info_unreg(&rxq->xdp_rxq); 110 111 return 0; 112 } 113 114 void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model) 115 { 116 __idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model); 117 } 118 119 void idpf_xdp_rxq_info_deinit_all(const struct idpf_q_vec_rsrc *rsrc) 120 { 121 idpf_rxq_for_each(rsrc, __idpf_xdp_rxq_info_deinit, 122 (void *)(size_t)rsrc->rxq_model); 123 } 124 125 static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) 126 { 127 struct bpf_prog *prog = arg; 128 struct bpf_prog *old; 129 130 if (prog) 131 bpf_prog_inc(prog); 132 133 old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held()); 134 if (old) 135 bpf_prog_put(old); 136 137 return 0; 138 } 139 140 void idpf_xdp_copy_prog_to_rqs(const struct idpf_q_vec_rsrc *rsrc, 141 struct bpf_prog *xdp_prog) 142 { 143 idpf_rxq_for_each(rsrc, idpf_xdp_rxq_assign_prog, xdp_prog); 144 } 145 146 static void idpf_xdp_tx_timer(struct work_struct *work); 147 148 int idpf_xdpsqs_get(const struct idpf_vport *vport) 149 { 150 struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; 151 struct net_device *dev; 152 u32 sqs; 153 154 if (!idpf_xdp_enabled(vport)) 155 return 0; 156 157 timers = kvzalloc_objs(*timers, vport->num_xdp_txq); 158 if (!timers) 159 return -ENOMEM; 160 161 for (u32 i = 0; i < vport->num_xdp_txq; i++) { 162 timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, 163 cpu_to_mem(i)); 164 if (!timers[i]) { 165 for (int j = i - 1; j >= 0; j--) 166 kfree(timers[j]); 167 168 return -ENOMEM; 169 } 170 } 171 172 dev = vport->netdev; 173 sqs = vport->dflt_qv_rsrc.xdp_txq_offset; 174 175 for (u32 i = sqs; i < vport->num_txq; i++) { 176 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 177 178 xdpsq->complq = xdpsq->txq_grp->complq; 179 kfree(xdpsq->refillq); 180 xdpsq->refillq = NULL; 181 182 idpf_queue_clear(FLOW_SCH_EN, xdpsq); 183 idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq); 184 idpf_queue_set(NOIRQ, xdpsq); 185 idpf_queue_set(XDP, xdpsq); 186 idpf_queue_set(XDP, xdpsq->complq); 187 188 xdpsq->timer = timers[i - sqs]; 189 libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share); 190 libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock, 191 idpf_xdp_tx_timer); 192 193 xdpsq->pending = 0; 194 xdpsq->xdp_tx = 0; 195 xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count); 196 } 197 198 return 0; 199 } 200 201 void idpf_xdpsqs_put(const struct idpf_vport *vport) 202 { 203 struct net_device *dev; 204 u32 sqs; 205 206 if (!idpf_xdp_enabled(vport)) 207 return; 208 209 dev = vport->netdev; 210 sqs = vport->dflt_qv_rsrc.xdp_txq_offset; 211 212 for (u32 i = sqs; i < vport->num_txq; i++) { 213 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 214 215 if (!idpf_queue_has_clear(XDP, xdpsq)) 216 continue; 217 218 libeth_xdpsq_deinit_timer(xdpsq->timer); 219 libeth_xdpsq_put(&xdpsq->xdp_lock, dev); 220 221 kfree(xdpsq->timer); 222 xdpsq->refillq = NULL; 223 idpf_queue_clear(NOIRQ, xdpsq); 224 } 225 } 226 227 static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc, 228 bool gen) 229 { 230 u32 val; 231 232 #ifdef __LIBETH_WORD_ACCESS 233 val = *(const u32 *)desc; 234 #else 235 val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | 236 le16_to_cpu(desc->qid_comptype_gen); 237 #endif 238 if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) 239 return -ENODATA; 240 241 if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != 242 FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, 243 IDPF_TXD_COMPLT_RS))) 244 return -EINVAL; 245 246 return upper_16_bits(val); 247 } 248 249 u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 250 { 251 struct idpf_compl_queue *cq = xdpsq->complq; 252 u32 tx_ntc = xdpsq->next_to_clean; 253 u32 tx_cnt = xdpsq->desc_count; 254 u32 ntc = cq->next_to_clean; 255 u32 cnt = cq->desc_count; 256 u32 done_frames; 257 bool gen; 258 259 gen = idpf_queue_has(GEN_CHK, cq); 260 261 for (done_frames = 0; done_frames < budget; ) { 262 int ret; 263 264 ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen); 265 if (ret >= 0) { 266 done_frames = ret > tx_ntc ? ret - tx_ntc : 267 ret + tx_cnt - tx_ntc; 268 goto next; 269 } 270 271 switch (ret) { 272 case -ENODATA: 273 goto out; 274 case -EINVAL: 275 break; 276 } 277 278 next: 279 if (unlikely(++ntc == cnt)) { 280 ntc = 0; 281 gen = !gen; 282 idpf_queue_change(GEN_CHK, cq); 283 } 284 } 285 286 out: 287 cq->next_to_clean = ntc; 288 289 return done_frames; 290 } 291 292 static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget) 293 { 294 struct libeth_xdpsq_napi_stats ss = { }; 295 struct idpf_tx_queue *xdpsq = _xdpsq; 296 u32 tx_ntc = xdpsq->next_to_clean; 297 u32 tx_cnt = xdpsq->desc_count; 298 struct xdp_frame_bulk bq; 299 struct libeth_cq_pp cp = { 300 .dev = xdpsq->dev, 301 .bq = &bq, 302 .xss = &ss, 303 .napi = true, 304 }; 305 u32 done_frames; 306 307 done_frames = idpf_xdpsq_poll(xdpsq, budget); 308 if (unlikely(!done_frames)) 309 return 0; 310 311 xdp_frame_bulk_init(&bq); 312 313 for (u32 i = 0; likely(i < done_frames); i++) { 314 libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp); 315 316 if (unlikely(++tx_ntc == tx_cnt)) 317 tx_ntc = 0; 318 } 319 320 xdp_flush_frame_bulk(&bq); 321 322 xdpsq->next_to_clean = tx_ntc; 323 xdpsq->pending -= done_frames; 324 xdpsq->xdp_tx -= cp.xdp_tx; 325 326 return done_frames; 327 } 328 329 static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 330 { 331 struct idpf_tx_queue *xdpsq = _xdpsq; 332 u32 free; 333 334 libeth_xdpsq_lock(&xdpsq->xdp_lock); 335 336 free = xdpsq->desc_count - xdpsq->pending; 337 if (free < xdpsq->thresh) 338 free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh); 339 340 *sq = (struct libeth_xdpsq){ 341 .sqes = xdpsq->tx_buf, 342 .descs = xdpsq->desc_ring, 343 .count = xdpsq->desc_count, 344 .lock = &xdpsq->xdp_lock, 345 .ntu = &xdpsq->next_to_use, 346 .pending = &xdpsq->pending, 347 .xdp_tx = &xdpsq->xdp_tx, 348 }; 349 350 return free; 351 } 352 353 LIBETH_XDP_DEFINE_START(); 354 LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete); 355 LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, 356 idpf_xdp_tx_xmit); 357 LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep, 358 idpf_xdp_tx_xmit); 359 LIBETH_XDP_DEFINE_END(); 360 361 int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 362 u32 flags) 363 { 364 const struct idpf_netdev_priv *np = netdev_priv(dev); 365 const struct idpf_vport *vport = np->vport; 366 u32 xdp_txq_offset; 367 368 if (unlikely(!netif_carrier_ok(dev) || !vport->link_up)) 369 return -ENETDOWN; 370 371 xdp_txq_offset = vport->dflt_qv_rsrc.xdp_txq_offset; 372 373 return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, 374 &vport->txqs[xdp_txq_offset], 375 vport->num_xdp_txq, 376 idpf_xdp_xmit_flush_bulk, 377 idpf_xdp_tx_finalize); 378 } 379 380 static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash, 381 enum xdp_rss_hash_type *rss_type) 382 { 383 const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; 384 struct idpf_xdp_rx_desc desc __uninitialized; 385 const struct idpf_rx_queue *rxq; 386 struct libeth_rx_pt pt; 387 388 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 389 390 idpf_xdp_get_qw0(&desc, xdp->desc); 391 392 pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)]; 393 if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt)) 394 return -ENODATA; 395 396 idpf_xdp_get_qw2(&desc, xdp->desc); 397 398 return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc), 399 pt); 400 } 401 402 static int idpf_xdpmo_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp) 403 { 404 const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; 405 struct idpf_xdp_rx_desc desc __uninitialized; 406 const struct idpf_rx_queue *rxq; 407 u64 cached_time, ts_ns; 408 u32 ts_high; 409 410 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 411 412 if (!idpf_queue_has(PTP, rxq)) 413 return -ENODATA; 414 415 idpf_xdp_get_qw1(&desc, xdp->desc); 416 417 if (!(idpf_xdp_rx_ts_low(&desc) & VIRTCHNL2_RX_FLEX_TSTAMP_VALID)) 418 return -ENODATA; 419 420 cached_time = READ_ONCE(rxq->cached_phc_time); 421 422 idpf_xdp_get_qw3(&desc, xdp->desc); 423 424 ts_high = idpf_xdp_rx_ts_high(&desc); 425 ts_ns = idpf_ptp_tstamp_extend_32b_to_64b(cached_time, ts_high); 426 427 *timestamp = ts_ns; 428 return 0; 429 } 430 431 static const struct xdp_metadata_ops idpf_xdpmo = { 432 .xmo_rx_hash = idpf_xdpmo_rx_hash, 433 .xmo_rx_timestamp = idpf_xdpmo_rx_timestamp, 434 }; 435 436 void idpf_xdp_set_features(const struct idpf_vport *vport) 437 { 438 if (!idpf_is_queue_model_split(vport->dflt_qv_rsrc.rxq_model)) 439 return; 440 441 libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo, 442 idpf_get_max_tx_bufs(vport->adapter), 443 libeth_xsktmo); 444 } 445 446 static int idpf_xdp_setup_prog(struct idpf_vport *vport, 447 const struct netdev_bpf *xdp) 448 { 449 const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 450 const struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; 451 struct bpf_prog *old, *prog = xdp->prog; 452 struct idpf_vport_config *cfg; 453 int ret; 454 455 cfg = vport->adapter->vport_config[vport->idx]; 456 457 if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || 458 !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || 459 !!vport->xdp_prog == !!prog) { 460 if (test_bit(IDPF_VPORT_UP, np->state)) 461 idpf_xdp_copy_prog_to_rqs(rsrc, prog); 462 463 old = xchg(&vport->xdp_prog, prog); 464 if (old) 465 bpf_prog_put(old); 466 467 cfg->user_config.xdp_prog = prog; 468 469 return 0; 470 } 471 472 if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) { 473 NL_SET_ERR_MSG_MOD(xdp->extack, 474 "No Tx queues available for XDP, please decrease the number of regular SQs"); 475 return -ENOSPC; 476 } 477 478 old = cfg->user_config.xdp_prog; 479 cfg->user_config.xdp_prog = prog; 480 481 ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); 482 if (ret) { 483 NL_SET_ERR_MSG_MOD(xdp->extack, 484 "Could not reopen the vport after XDP setup"); 485 486 cfg->user_config.xdp_prog = old; 487 old = prog; 488 } 489 490 if (old) 491 bpf_prog_put(old); 492 493 libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog); 494 495 return ret; 496 } 497 498 int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 499 { 500 struct idpf_vport *vport; 501 int ret; 502 503 idpf_vport_ctrl_lock(dev); 504 vport = idpf_netdev_to_vport(dev); 505 506 if (!idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model)) 507 goto notsupp; 508 509 switch (xdp->command) { 510 case XDP_SETUP_PROG: 511 ret = idpf_xdp_setup_prog(vport, xdp); 512 break; 513 case XDP_SETUP_XSK_POOL: 514 ret = idpf_xsk_pool_setup(vport, xdp); 515 break; 516 default: 517 notsupp: 518 ret = -EOPNOTSUPP; 519 break; 520 } 521 522 idpf_vport_ctrl_unlock(dev); 523 524 return ret; 525 } 526