1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #include "idpf.h" 5 #include "idpf_virtchnl.h" 6 #include "xdp.h" 7 #include "xsk.h" 8 9 static int idpf_rxq_for_each(const struct idpf_vport *vport, 10 int (*fn)(struct idpf_rx_queue *rxq, void *arg), 11 void *arg) 12 { 13 bool splitq = idpf_is_queue_model_split(vport->rxq_model); 14 15 if (!vport->rxq_grps) 16 return -ENETDOWN; 17 18 for (u32 i = 0; i < vport->num_rxq_grp; i++) { 19 const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 20 u32 num_rxq; 21 22 if (splitq) 23 num_rxq = rx_qgrp->splitq.num_rxq_sets; 24 else 25 num_rxq = rx_qgrp->singleq.num_rxq; 26 27 for (u32 j = 0; j < num_rxq; j++) { 28 struct idpf_rx_queue *q; 29 int err; 30 31 if (splitq) 32 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 33 else 34 q = rx_qgrp->singleq.rxqs[j]; 35 36 err = fn(q, arg); 37 if (err) 38 return err; 39 } 40 } 41 42 return 0; 43 } 44 45 static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) 46 { 47 const struct idpf_vport *vport = rxq->q_vector->vport; 48 bool split = idpf_is_queue_model_split(vport->rxq_model); 49 const struct page_pool *pp; 50 int err; 51 52 err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, 53 rxq->q_vector->napi.napi_id, 54 rxq->rx_buf_size); 55 if (err) 56 return err; 57 58 pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 59 xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 60 61 if (!split) 62 return 0; 63 64 rxq->xdpsqs = &vport->txqs[vport->xdp_txq_offset]; 65 rxq->num_xdp_txq = vport->num_xdp_txq; 66 67 return 0; 68 } 69 70 int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) 71 { 72 return __idpf_xdp_rxq_info_init(rxq, NULL); 73 } 74 75 int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) 76 { 77 return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL); 78 } 79 80 static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) 81 { 82 if (idpf_is_queue_model_split((size_t)arg)) { 83 rxq->xdpsqs = NULL; 84 rxq->num_xdp_txq = 0; 85 } 86 87 xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 88 xdp_rxq_info_unreg(&rxq->xdp_rxq); 89 90 return 0; 91 } 92 93 void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model) 94 { 95 __idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model); 96 } 97 98 void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) 99 { 100 idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit, 101 (void *)(size_t)vport->rxq_model); 102 } 103 104 static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) 105 { 106 struct bpf_prog *prog = arg; 107 struct bpf_prog *old; 108 109 if (prog) 110 bpf_prog_inc(prog); 111 112 old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held()); 113 if (old) 114 bpf_prog_put(old); 115 116 return 0; 117 } 118 119 void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, 120 struct bpf_prog *xdp_prog) 121 { 122 idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); 123 } 124 125 static void idpf_xdp_tx_timer(struct work_struct *work); 126 127 int idpf_xdpsqs_get(const struct idpf_vport *vport) 128 { 129 struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; 130 struct net_device *dev; 131 u32 sqs; 132 133 if (!idpf_xdp_enabled(vport)) 134 return 0; 135 136 timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL); 137 if (!timers) 138 return -ENOMEM; 139 140 for (u32 i = 0; i < vport->num_xdp_txq; i++) { 141 timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, 142 cpu_to_mem(i)); 143 if (!timers[i]) { 144 for (int j = i - 1; j >= 0; j--) 145 kfree(timers[j]); 146 147 return -ENOMEM; 148 } 149 } 150 151 dev = vport->netdev; 152 sqs = vport->xdp_txq_offset; 153 154 for (u32 i = sqs; i < vport->num_txq; i++) { 155 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 156 157 xdpsq->complq = xdpsq->txq_grp->complq; 158 kfree(xdpsq->refillq); 159 xdpsq->refillq = NULL; 160 161 idpf_queue_clear(FLOW_SCH_EN, xdpsq); 162 idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq); 163 idpf_queue_set(NOIRQ, xdpsq); 164 idpf_queue_set(XDP, xdpsq); 165 idpf_queue_set(XDP, xdpsq->complq); 166 167 xdpsq->timer = timers[i - sqs]; 168 libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share); 169 libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock, 170 idpf_xdp_tx_timer); 171 172 xdpsq->pending = 0; 173 xdpsq->xdp_tx = 0; 174 xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count); 175 } 176 177 return 0; 178 } 179 180 void idpf_xdpsqs_put(const struct idpf_vport *vport) 181 { 182 struct net_device *dev; 183 u32 sqs; 184 185 if (!idpf_xdp_enabled(vport)) 186 return; 187 188 dev = vport->netdev; 189 sqs = vport->xdp_txq_offset; 190 191 for (u32 i = sqs; i < vport->num_txq; i++) { 192 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 193 194 if (!idpf_queue_has_clear(XDP, xdpsq)) 195 continue; 196 197 libeth_xdpsq_deinit_timer(xdpsq->timer); 198 libeth_xdpsq_put(&xdpsq->xdp_lock, dev); 199 200 kfree(xdpsq->timer); 201 xdpsq->refillq = NULL; 202 idpf_queue_clear(NOIRQ, xdpsq); 203 } 204 } 205 206 static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc, 207 bool gen) 208 { 209 u32 val; 210 211 #ifdef __LIBETH_WORD_ACCESS 212 val = *(const u32 *)desc; 213 #else 214 val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | 215 le16_to_cpu(desc->qid_comptype_gen); 216 #endif 217 if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) 218 return -ENODATA; 219 220 if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != 221 FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, 222 IDPF_TXD_COMPLT_RS))) 223 return -EINVAL; 224 225 return upper_16_bits(val); 226 } 227 228 static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 229 { 230 struct idpf_compl_queue *cq = xdpsq->complq; 231 u32 tx_ntc = xdpsq->next_to_clean; 232 u32 tx_cnt = xdpsq->desc_count; 233 u32 ntc = cq->next_to_clean; 234 u32 cnt = cq->desc_count; 235 u32 done_frames; 236 bool gen; 237 238 gen = idpf_queue_has(GEN_CHK, cq); 239 240 for (done_frames = 0; done_frames < budget; ) { 241 int ret; 242 243 ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen); 244 if (ret >= 0) { 245 done_frames = ret > tx_ntc ? ret - tx_ntc : 246 ret + tx_cnt - tx_ntc; 247 goto next; 248 } 249 250 switch (ret) { 251 case -ENODATA: 252 goto out; 253 case -EINVAL: 254 break; 255 } 256 257 next: 258 if (unlikely(++ntc == cnt)) { 259 ntc = 0; 260 gen = !gen; 261 idpf_queue_change(GEN_CHK, cq); 262 } 263 } 264 265 out: 266 cq->next_to_clean = ntc; 267 268 return done_frames; 269 } 270 271 static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget) 272 { 273 struct libeth_xdpsq_napi_stats ss = { }; 274 struct idpf_tx_queue *xdpsq = _xdpsq; 275 u32 tx_ntc = xdpsq->next_to_clean; 276 u32 tx_cnt = xdpsq->desc_count; 277 struct xdp_frame_bulk bq; 278 struct libeth_cq_pp cp = { 279 .dev = xdpsq->dev, 280 .bq = &bq, 281 .xss = &ss, 282 .napi = true, 283 }; 284 u32 done_frames; 285 286 done_frames = idpf_xdpsq_poll(xdpsq, budget); 287 if (unlikely(!done_frames)) 288 return 0; 289 290 xdp_frame_bulk_init(&bq); 291 292 for (u32 i = 0; likely(i < done_frames); i++) { 293 libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp); 294 295 if (unlikely(++tx_ntc == tx_cnt)) 296 tx_ntc = 0; 297 } 298 299 xdp_flush_frame_bulk(&bq); 300 301 xdpsq->next_to_clean = tx_ntc; 302 xdpsq->pending -= done_frames; 303 xdpsq->xdp_tx -= cp.xdp_tx; 304 305 return done_frames; 306 } 307 308 static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 309 { 310 struct idpf_tx_queue *xdpsq = _xdpsq; 311 u32 free; 312 313 libeth_xdpsq_lock(&xdpsq->xdp_lock); 314 315 free = xdpsq->desc_count - xdpsq->pending; 316 if (free < xdpsq->thresh) 317 free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh); 318 319 *sq = (struct libeth_xdpsq){ 320 .sqes = xdpsq->tx_buf, 321 .descs = xdpsq->desc_ring, 322 .count = xdpsq->desc_count, 323 .lock = &xdpsq->xdp_lock, 324 .ntu = &xdpsq->next_to_use, 325 .pending = &xdpsq->pending, 326 .xdp_tx = &xdpsq->xdp_tx, 327 }; 328 329 return free; 330 } 331 332 LIBETH_XDP_DEFINE_START(); 333 LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete); 334 LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, 335 idpf_xdp_tx_xmit); 336 LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep, 337 idpf_xdp_tx_xmit); 338 LIBETH_XDP_DEFINE_END(); 339 340 int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 341 u32 flags) 342 { 343 const struct idpf_netdev_priv *np = netdev_priv(dev); 344 const struct idpf_vport *vport = np->vport; 345 346 if (unlikely(!netif_carrier_ok(dev) || !vport->link_up)) 347 return -ENETDOWN; 348 349 return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, 350 &vport->txqs[vport->xdp_txq_offset], 351 vport->num_xdp_txq, 352 idpf_xdp_xmit_flush_bulk, 353 idpf_xdp_tx_finalize); 354 } 355 356 static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash, 357 enum xdp_rss_hash_type *rss_type) 358 { 359 const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; 360 struct idpf_xdp_rx_desc desc __uninitialized; 361 const struct idpf_rx_queue *rxq; 362 struct libeth_rx_pt pt; 363 364 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); 365 366 idpf_xdp_get_qw0(&desc, xdp->desc); 367 368 pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)]; 369 if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt)) 370 return -ENODATA; 371 372 idpf_xdp_get_qw2(&desc, xdp->desc); 373 374 return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc), 375 pt); 376 } 377 378 static const struct xdp_metadata_ops idpf_xdpmo = { 379 .xmo_rx_hash = idpf_xdpmo_rx_hash, 380 }; 381 382 void idpf_xdp_set_features(const struct idpf_vport *vport) 383 { 384 if (!idpf_is_queue_model_split(vport->rxq_model)) 385 return; 386 387 libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo); 388 } 389 390 static int idpf_xdp_setup_prog(struct idpf_vport *vport, 391 const struct netdev_bpf *xdp) 392 { 393 const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 394 struct bpf_prog *old, *prog = xdp->prog; 395 struct idpf_vport_config *cfg; 396 int ret; 397 398 cfg = vport->adapter->vport_config[vport->idx]; 399 400 if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || 401 !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || 402 !!vport->xdp_prog == !!prog) { 403 if (np->state == __IDPF_VPORT_UP) 404 idpf_xdp_copy_prog_to_rqs(vport, prog); 405 406 old = xchg(&vport->xdp_prog, prog); 407 if (old) 408 bpf_prog_put(old); 409 410 cfg->user_config.xdp_prog = prog; 411 412 return 0; 413 } 414 415 if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) { 416 NL_SET_ERR_MSG_MOD(xdp->extack, 417 "No Tx queues available for XDP, please decrease the number of regular SQs"); 418 return -ENOSPC; 419 } 420 421 old = cfg->user_config.xdp_prog; 422 cfg->user_config.xdp_prog = prog; 423 424 ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); 425 if (ret) { 426 NL_SET_ERR_MSG_MOD(xdp->extack, 427 "Could not reopen the vport after XDP setup"); 428 429 cfg->user_config.xdp_prog = old; 430 old = prog; 431 } 432 433 if (old) 434 bpf_prog_put(old); 435 436 libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog); 437 438 return ret; 439 } 440 441 int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 442 { 443 struct idpf_vport *vport; 444 int ret; 445 446 idpf_vport_ctrl_lock(dev); 447 vport = idpf_netdev_to_vport(dev); 448 449 if (!idpf_is_queue_model_split(vport->txq_model)) 450 goto notsupp; 451 452 switch (xdp->command) { 453 case XDP_SETUP_PROG: 454 ret = idpf_xdp_setup_prog(vport, xdp); 455 break; 456 case XDP_SETUP_XSK_POOL: 457 ret = idpf_xsk_pool_setup(vport, xdp); 458 break; 459 default: 460 notsupp: 461 ret = -EOPNOTSUPP; 462 break; 463 } 464 465 idpf_vport_ctrl_unlock(dev); 466 467 return ret; 468 } 469