1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2025 Intel Corporation */ 3 4 #include "idpf.h" 5 #include "idpf_virtchnl.h" 6 #include "xdp.h" 7 8 static int idpf_rxq_for_each(const struct idpf_vport *vport, 9 int (*fn)(struct idpf_rx_queue *rxq, void *arg), 10 void *arg) 11 { 12 bool splitq = idpf_is_queue_model_split(vport->rxq_model); 13 14 if (!vport->rxq_grps) 15 return -ENETDOWN; 16 17 for (u32 i = 0; i < vport->num_rxq_grp; i++) { 18 const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 19 u32 num_rxq; 20 21 if (splitq) 22 num_rxq = rx_qgrp->splitq.num_rxq_sets; 23 else 24 num_rxq = rx_qgrp->singleq.num_rxq; 25 26 for (u32 j = 0; j < num_rxq; j++) { 27 struct idpf_rx_queue *q; 28 int err; 29 30 if (splitq) 31 q = &rx_qgrp->splitq.rxq_sets[j]->rxq; 32 else 33 q = rx_qgrp->singleq.rxqs[j]; 34 35 err = fn(q, arg); 36 if (err) 37 return err; 38 } 39 } 40 41 return 0; 42 } 43 44 static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) 45 { 46 const struct idpf_vport *vport = rxq->q_vector->vport; 47 bool split = idpf_is_queue_model_split(vport->rxq_model); 48 const struct page_pool *pp; 49 int err; 50 51 err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, 52 rxq->q_vector->napi.napi_id, 53 rxq->rx_buf_size); 54 if (err) 55 return err; 56 57 pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 58 xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 59 60 if (!split) 61 return 0; 62 63 rxq->xdpsqs = &vport->txqs[vport->xdp_txq_offset]; 64 rxq->num_xdp_txq = vport->num_xdp_txq; 65 66 return 0; 67 } 68 69 int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) 70 { 71 return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL); 72 } 73 74 static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) 75 { 76 if (idpf_is_queue_model_split((size_t)arg)) { 77 rxq->xdpsqs = NULL; 78 rxq->num_xdp_txq = 0; 79 } 80 81 xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 82 xdp_rxq_info_unreg(&rxq->xdp_rxq); 83 84 return 0; 85 } 86 87 void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) 88 { 89 idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit, 90 (void *)(size_t)vport->rxq_model); 91 } 92 93 static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) 94 { 95 struct bpf_prog *prog = arg; 96 struct bpf_prog *old; 97 98 if (prog) 99 bpf_prog_inc(prog); 100 101 old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held()); 102 if (old) 103 bpf_prog_put(old); 104 105 return 0; 106 } 107 108 void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, 109 struct bpf_prog *xdp_prog) 110 { 111 idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); 112 } 113 114 static void idpf_xdp_tx_timer(struct work_struct *work); 115 116 int idpf_xdpsqs_get(const struct idpf_vport *vport) 117 { 118 struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; 119 struct net_device *dev; 120 u32 sqs; 121 122 if (!idpf_xdp_enabled(vport)) 123 return 0; 124 125 timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL); 126 if (!timers) 127 return -ENOMEM; 128 129 for (u32 i = 0; i < vport->num_xdp_txq; i++) { 130 timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, 131 cpu_to_mem(i)); 132 if (!timers[i]) { 133 for (int j = i - 1; j >= 0; j--) 134 kfree(timers[j]); 135 136 return -ENOMEM; 137 } 138 } 139 140 dev = vport->netdev; 141 sqs = vport->xdp_txq_offset; 142 143 for (u32 i = sqs; i < vport->num_txq; i++) { 144 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 145 146 xdpsq->complq = xdpsq->txq_grp->complq; 147 kfree(xdpsq->refillq); 148 xdpsq->refillq = NULL; 149 150 idpf_queue_clear(FLOW_SCH_EN, xdpsq); 151 idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq); 152 idpf_queue_set(NOIRQ, xdpsq); 153 idpf_queue_set(XDP, xdpsq); 154 idpf_queue_set(XDP, xdpsq->complq); 155 156 xdpsq->timer = timers[i - sqs]; 157 libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share); 158 libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock, 159 idpf_xdp_tx_timer); 160 161 xdpsq->pending = 0; 162 xdpsq->xdp_tx = 0; 163 xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count); 164 } 165 166 return 0; 167 } 168 169 void idpf_xdpsqs_put(const struct idpf_vport *vport) 170 { 171 struct net_device *dev; 172 u32 sqs; 173 174 if (!idpf_xdp_enabled(vport)) 175 return; 176 177 dev = vport->netdev; 178 sqs = vport->xdp_txq_offset; 179 180 for (u32 i = sqs; i < vport->num_txq; i++) { 181 struct idpf_tx_queue *xdpsq = vport->txqs[i]; 182 183 if (!idpf_queue_has_clear(XDP, xdpsq)) 184 continue; 185 186 libeth_xdpsq_deinit_timer(xdpsq->timer); 187 libeth_xdpsq_put(&xdpsq->xdp_lock, dev); 188 189 kfree(xdpsq->timer); 190 xdpsq->refillq = NULL; 191 idpf_queue_clear(NOIRQ, xdpsq); 192 } 193 } 194 195 static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc, 196 bool gen) 197 { 198 u32 val; 199 200 #ifdef __LIBETH_WORD_ACCESS 201 val = *(const u32 *)desc; 202 #else 203 val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | 204 le16_to_cpu(desc->qid_comptype_gen); 205 #endif 206 if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) 207 return -ENODATA; 208 209 if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != 210 FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, 211 IDPF_TXD_COMPLT_RS))) 212 return -EINVAL; 213 214 return upper_16_bits(val); 215 } 216 217 static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 218 { 219 struct idpf_compl_queue *cq = xdpsq->complq; 220 u32 tx_ntc = xdpsq->next_to_clean; 221 u32 tx_cnt = xdpsq->desc_count; 222 u32 ntc = cq->next_to_clean; 223 u32 cnt = cq->desc_count; 224 u32 done_frames; 225 bool gen; 226 227 gen = idpf_queue_has(GEN_CHK, cq); 228 229 for (done_frames = 0; done_frames < budget; ) { 230 int ret; 231 232 ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen); 233 if (ret >= 0) { 234 done_frames = ret > tx_ntc ? ret - tx_ntc : 235 ret + tx_cnt - tx_ntc; 236 goto next; 237 } 238 239 switch (ret) { 240 case -ENODATA: 241 goto out; 242 case -EINVAL: 243 break; 244 } 245 246 next: 247 if (unlikely(++ntc == cnt)) { 248 ntc = 0; 249 gen = !gen; 250 idpf_queue_change(GEN_CHK, cq); 251 } 252 } 253 254 out: 255 cq->next_to_clean = ntc; 256 257 return done_frames; 258 } 259 260 static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget) 261 { 262 struct libeth_xdpsq_napi_stats ss = { }; 263 struct idpf_tx_queue *xdpsq = _xdpsq; 264 u32 tx_ntc = xdpsq->next_to_clean; 265 u32 tx_cnt = xdpsq->desc_count; 266 struct xdp_frame_bulk bq; 267 struct libeth_cq_pp cp = { 268 .dev = xdpsq->dev, 269 .bq = &bq, 270 .xss = &ss, 271 .napi = true, 272 }; 273 u32 done_frames; 274 275 done_frames = idpf_xdpsq_poll(xdpsq, budget); 276 if (unlikely(!done_frames)) 277 return 0; 278 279 xdp_frame_bulk_init(&bq); 280 281 for (u32 i = 0; likely(i < done_frames); i++) { 282 libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp); 283 284 if (unlikely(++tx_ntc == tx_cnt)) 285 tx_ntc = 0; 286 } 287 288 xdp_flush_frame_bulk(&bq); 289 290 xdpsq->next_to_clean = tx_ntc; 291 xdpsq->pending -= done_frames; 292 xdpsq->xdp_tx -= cp.xdp_tx; 293 294 return done_frames; 295 } 296 297 static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 298 { 299 struct idpf_tx_queue *xdpsq = _xdpsq; 300 u32 free; 301 302 libeth_xdpsq_lock(&xdpsq->xdp_lock); 303 304 free = xdpsq->desc_count - xdpsq->pending; 305 if (free < xdpsq->thresh) 306 free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh); 307 308 *sq = (struct libeth_xdpsq){ 309 .sqes = xdpsq->tx_buf, 310 .descs = xdpsq->desc_ring, 311 .count = xdpsq->desc_count, 312 .lock = &xdpsq->xdp_lock, 313 .ntu = &xdpsq->next_to_use, 314 .pending = &xdpsq->pending, 315 .xdp_tx = &xdpsq->xdp_tx, 316 }; 317 318 return free; 319 } 320 321 LIBETH_XDP_DEFINE_START(); 322 LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete); 323 LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, 324 idpf_xdp_tx_xmit); 325 LIBETH_XDP_DEFINE_END(); 326 327 void idpf_xdp_set_features(const struct idpf_vport *vport) 328 { 329 if (!idpf_is_queue_model_split(vport->rxq_model)) 330 return; 331 332 libeth_xdp_set_features_noredir(vport->netdev); 333 } 334 335 static int idpf_xdp_setup_prog(struct idpf_vport *vport, 336 const struct netdev_bpf *xdp) 337 { 338 const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); 339 struct bpf_prog *old, *prog = xdp->prog; 340 struct idpf_vport_config *cfg; 341 int ret; 342 343 cfg = vport->adapter->vport_config[vport->idx]; 344 345 if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || 346 !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || 347 !!vport->xdp_prog == !!prog) { 348 if (np->state == __IDPF_VPORT_UP) 349 idpf_xdp_copy_prog_to_rqs(vport, prog); 350 351 old = xchg(&vport->xdp_prog, prog); 352 if (old) 353 bpf_prog_put(old); 354 355 cfg->user_config.xdp_prog = prog; 356 357 return 0; 358 } 359 360 if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) { 361 NL_SET_ERR_MSG_MOD(xdp->extack, 362 "No Tx queues available for XDP, please decrease the number of regular SQs"); 363 return -ENOSPC; 364 } 365 366 old = cfg->user_config.xdp_prog; 367 cfg->user_config.xdp_prog = prog; 368 369 ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); 370 if (ret) { 371 NL_SET_ERR_MSG_MOD(xdp->extack, 372 "Could not reopen the vport after XDP setup"); 373 374 cfg->user_config.xdp_prog = old; 375 old = prog; 376 } 377 378 if (old) 379 bpf_prog_put(old); 380 381 return ret; 382 } 383 384 int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) 385 { 386 struct idpf_vport *vport; 387 int ret; 388 389 idpf_vport_ctrl_lock(dev); 390 vport = idpf_netdev_to_vport(dev); 391 392 if (!idpf_is_queue_model_split(vport->txq_model)) 393 goto notsupp; 394 395 switch (xdp->command) { 396 case XDP_SETUP_PROG: 397 ret = idpf_xdp_setup_prog(vport, xdp); 398 break; 399 default: 400 notsupp: 401 ret = -EOPNOTSUPP; 402 break; 403 } 404 405 idpf_vport_ctrl_unlock(dev); 406 407 return ret; 408 } 409