1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include "ena_xdp.h" 7 8 static int validate_xdp_req_id(struct ena_ring *tx_ring, u16 req_id) 9 { 10 struct ena_tx_buffer *tx_info; 11 12 tx_info = &tx_ring->tx_buffer_info[req_id]; 13 if (likely(tx_info->xdpf)) 14 return 0; 15 16 return handle_invalid_req_id(tx_ring, req_id, tx_info, true); 17 } 18 19 static int ena_xdp_tx_map_frame(struct ena_ring *tx_ring, 20 struct ena_tx_buffer *tx_info, 21 struct xdp_frame *xdpf, 22 struct ena_com_tx_ctx *ena_tx_ctx) 23 { 24 struct ena_adapter *adapter = tx_ring->adapter; 25 struct ena_com_buf *ena_buf; 26 int push_len = 0; 27 dma_addr_t dma; 28 void *data; 29 u32 size; 30 31 tx_info->xdpf = xdpf; 32 data = tx_info->xdpf->data; 33 size = tx_info->xdpf->len; 34 35 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 36 /* Designate part of the packet for LLQ */ 37 push_len = min_t(u32, size, tx_ring->tx_max_header_size); 38 39 ena_tx_ctx->push_header = data; 40 41 size -= push_len; 42 data += push_len; 43 } 44 45 ena_tx_ctx->header_len = push_len; 46 47 if (size > 0) { 48 dma = dma_map_single(tx_ring->dev, 49 data, 50 size, 51 DMA_TO_DEVICE); 52 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) 53 goto error_report_dma_error; 54 55 tx_info->map_linear_data = 0; 56 57 ena_buf = tx_info->bufs; 58 ena_buf->paddr = dma; 59 ena_buf->len = size; 60 61 ena_tx_ctx->ena_bufs = ena_buf; 62 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1; 63 } 64 65 return 0; 66 67 error_report_dma_error: 68 ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1, 69 &tx_ring->syncp); 70 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n"); 71 72 return -EINVAL; 73 } 74 75 int ena_xdp_xmit_frame(struct ena_ring *tx_ring, 76 struct ena_adapter *adapter, 77 struct xdp_frame *xdpf, 78 int flags) 79 { 80 struct ena_com_tx_ctx ena_tx_ctx = {}; 81 struct ena_tx_buffer *tx_info; 82 u16 next_to_use, req_id; 83 int rc; 84 85 next_to_use = tx_ring->next_to_use; 86 req_id = tx_ring->free_ids[next_to_use]; 87 tx_info = &tx_ring->tx_buffer_info[req_id]; 88 tx_info->num_of_bufs = 0; 89 90 rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx); 91 if (unlikely(rc)) 92 return rc; 93 94 ena_tx_ctx.req_id = req_id; 95 96 rc = ena_xmit_common(adapter, 97 tx_ring, 98 tx_info, 99 &ena_tx_ctx, 100 next_to_use, 101 xdpf->len); 102 if (rc) 103 goto error_unmap_dma; 104 105 /* trigger the dma engine. ena_ring_tx_doorbell() 106 * calls a memory barrier inside it. 107 */ 108 if (flags & XDP_XMIT_FLUSH) 109 ena_ring_tx_doorbell(tx_ring); 110 111 return rc; 112 113 error_unmap_dma: 114 ena_unmap_tx_buff(tx_ring, tx_info); 115 tx_info->xdpf = NULL; 116 return rc; 117 } 118 119 int ena_xdp_xmit(struct net_device *dev, int n, 120 struct xdp_frame **frames, u32 flags) 121 { 122 struct ena_adapter *adapter = netdev_priv(dev); 123 struct ena_ring *tx_ring; 124 int qid, i, nxmit = 0; 125 126 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 127 return -EINVAL; 128 129 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) 130 return -ENETDOWN; 131 132 /* We assume that all rings have the same XDP program */ 133 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog)) 134 return -ENXIO; 135 136 qid = smp_processor_id() % adapter->xdp_num_queues; 137 qid += adapter->xdp_first_ring; 138 tx_ring = &adapter->tx_ring[qid]; 139 140 /* Other CPU ids might try to send thorugh this queue */ 141 spin_lock(&tx_ring->xdp_tx_lock); 142 143 for (i = 0; i < n; i++) { 144 if (ena_xdp_xmit_frame(tx_ring, adapter, frames[i], 0)) 145 break; 146 nxmit++; 147 } 148 149 /* Ring doorbell to make device aware of the packets */ 150 if (flags & XDP_XMIT_FLUSH) 151 ena_ring_tx_doorbell(tx_ring); 152 153 spin_unlock(&tx_ring->xdp_tx_lock); 154 155 /* Return number of packets sent */ 156 return nxmit; 157 } 158 159 static void ena_init_all_xdp_queues(struct ena_adapter *adapter) 160 { 161 adapter->xdp_first_ring = adapter->num_io_queues; 162 adapter->xdp_num_queues = adapter->num_io_queues; 163 164 ena_init_io_rings(adapter, 165 adapter->xdp_first_ring, 166 adapter->xdp_num_queues); 167 } 168 169 int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter) 170 { 171 u32 xdp_first_ring = adapter->xdp_first_ring; 172 u32 xdp_num_queues = adapter->xdp_num_queues; 173 int rc = 0; 174 175 rc = ena_setup_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues); 176 if (rc) 177 goto setup_err; 178 179 rc = ena_create_io_tx_queues_in_range(adapter, xdp_first_ring, xdp_num_queues); 180 if (rc) 181 goto create_err; 182 183 return 0; 184 185 create_err: 186 ena_free_all_io_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues); 187 setup_err: 188 return rc; 189 } 190 191 /* Provides a way for both kernel and bpf-prog to know 192 * more about the RX-queue a given XDP frame arrived on. 193 */ 194 int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) 195 { 196 int rc; 197 198 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); 199 200 netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d", 201 rx_ring->qid); 202 if (rc) { 203 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, 204 "Failed to register xdp rx queue info. RX queue num %d rc: %d\n", 205 rx_ring->qid, rc); 206 goto err; 207 } 208 209 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); 210 211 if (rc) { 212 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, 213 "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n", 214 rx_ring->qid, rc); 215 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 216 } 217 218 err: 219 return rc; 220 } 221 222 void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring) 223 { 224 netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev, 225 "Unregistering RX info for queue %d", 226 rx_ring->qid); 227 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq); 228 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 229 } 230 231 void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter, 232 struct bpf_prog *prog, 233 int first, int count) 234 { 235 struct bpf_prog *old_bpf_prog; 236 struct ena_ring *rx_ring; 237 int i = 0; 238 239 for (i = first; i < count; i++) { 240 rx_ring = &adapter->rx_ring[i]; 241 old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog); 242 243 if (!old_bpf_prog && prog) { 244 rx_ring->rx_headroom = XDP_PACKET_HEADROOM; 245 } else if (old_bpf_prog && !prog) { 246 rx_ring->rx_headroom = NET_SKB_PAD; 247 } 248 } 249 } 250 251 static void ena_xdp_exchange_program(struct ena_adapter *adapter, 252 struct bpf_prog *prog) 253 { 254 struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog); 255 256 ena_xdp_exchange_program_rx_in_range(adapter, 257 prog, 258 0, 259 adapter->num_io_queues); 260 261 if (old_bpf_prog) 262 bpf_prog_put(old_bpf_prog); 263 } 264 265 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter) 266 { 267 bool was_up; 268 int rc; 269 270 was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 271 272 if (was_up) 273 ena_down(adapter); 274 275 adapter->xdp_first_ring = 0; 276 adapter->xdp_num_queues = 0; 277 ena_xdp_exchange_program(adapter, NULL); 278 if (was_up) { 279 rc = ena_up(adapter); 280 if (rc) 281 return rc; 282 } 283 return 0; 284 } 285 286 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf) 287 { 288 struct ena_adapter *adapter = netdev_priv(netdev); 289 struct bpf_prog *prog = bpf->prog; 290 struct bpf_prog *old_bpf_prog; 291 int rc, prev_mtu; 292 bool is_up; 293 294 is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); 295 rc = ena_xdp_allowed(adapter); 296 if (rc == ENA_XDP_ALLOWED) { 297 old_bpf_prog = adapter->xdp_bpf_prog; 298 if (prog) { 299 if (!is_up) { 300 ena_init_all_xdp_queues(adapter); 301 } else if (!old_bpf_prog) { 302 ena_down(adapter); 303 ena_init_all_xdp_queues(adapter); 304 } 305 ena_xdp_exchange_program(adapter, prog); 306 307 netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n"); 308 309 if (is_up && !old_bpf_prog) { 310 rc = ena_up(adapter); 311 if (rc) 312 return rc; 313 } 314 xdp_features_set_redirect_target(netdev, false); 315 } else if (old_bpf_prog) { 316 xdp_features_clear_redirect_target(netdev); 317 netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n"); 318 319 rc = ena_destroy_and_free_all_xdp_queues(adapter); 320 if (rc) 321 return rc; 322 } 323 324 prev_mtu = netdev->max_mtu; 325 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu; 326 327 if (!old_bpf_prog) 328 netif_info(adapter, drv, adapter->netdev, 329 "XDP program is set, changing the max_mtu from %d to %d", 330 prev_mtu, netdev->max_mtu); 331 332 } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) { 333 netif_err(adapter, drv, adapter->netdev, 334 "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on", 335 netdev->mtu, ENA_XDP_MAX_MTU); 336 NL_SET_ERR_MSG_MOD(bpf->extack, 337 "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info"); 338 return -EINVAL; 339 } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) { 340 netif_err(adapter, drv, adapter->netdev, 341 "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n", 342 adapter->num_io_queues, adapter->max_num_io_queues); 343 NL_SET_ERR_MSG_MOD(bpf->extack, 344 "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info"); 345 return -EINVAL; 346 } 347 348 return 0; 349 } 350 351 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp 352 * program as well as to query the current xdp program id. 353 */ 354 int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf) 355 { 356 switch (bpf->command) { 357 case XDP_SETUP_PROG: 358 return ena_xdp_set(netdev, bpf); 359 default: 360 return -EINVAL; 361 } 362 return 0; 363 } 364 365 static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget) 366 { 367 u32 total_done = 0; 368 u16 next_to_clean; 369 int tx_pkts = 0; 370 u16 req_id; 371 int rc; 372 373 if (unlikely(!tx_ring)) 374 return 0; 375 next_to_clean = tx_ring->next_to_clean; 376 377 while (tx_pkts < budget) { 378 struct ena_tx_buffer *tx_info; 379 struct xdp_frame *xdpf; 380 381 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, 382 &req_id); 383 if (rc) { 384 if (unlikely(rc == -EINVAL)) 385 handle_invalid_req_id(tx_ring, req_id, NULL, true); 386 break; 387 } 388 389 /* validate that the request id points to a valid xdp_frame */ 390 rc = validate_xdp_req_id(tx_ring, req_id); 391 if (rc) 392 break; 393 394 tx_info = &tx_ring->tx_buffer_info[req_id]; 395 396 tx_info->last_jiffies = 0; 397 398 xdpf = tx_info->xdpf; 399 tx_info->xdpf = NULL; 400 ena_unmap_tx_buff(tx_ring, tx_info); 401 xdp_return_frame(xdpf); 402 403 tx_pkts++; 404 total_done += tx_info->tx_descs; 405 tx_ring->free_ids[next_to_clean] = req_id; 406 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 407 tx_ring->ring_size); 408 409 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, 410 "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id); 411 } 412 413 tx_ring->next_to_clean = next_to_clean; 414 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done); 415 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 416 417 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, 418 "tx_poll: q %d done. total pkts: %d\n", 419 tx_ring->qid, tx_pkts); 420 421 return tx_pkts; 422 } 423 424 /* This is the XDP napi callback. XDP queues use a separate napi callback 425 * than Rx/Tx queues. 426 */ 427 int ena_xdp_io_poll(struct napi_struct *napi, int budget) 428 { 429 struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi); 430 struct ena_ring *tx_ring; 431 u32 work_done; 432 int ret; 433 434 tx_ring = ena_napi->tx_ring; 435 436 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || 437 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) { 438 napi_complete_done(napi, 0); 439 return 0; 440 } 441 442 work_done = ena_clean_xdp_irq(tx_ring, budget); 443 444 /* If the device is about to reset or down, avoid unmask 445 * the interrupt and return 0 so NAPI won't reschedule 446 */ 447 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) { 448 napi_complete_done(napi, 0); 449 ret = 0; 450 } else if (budget > work_done) { 451 ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1, 452 &tx_ring->syncp); 453 if (napi_complete_done(napi, work_done)) 454 ena_unmask_interrupt(tx_ring, NULL); 455 456 ena_update_ring_numa_node(tx_ring, NULL); 457 ret = work_done; 458 } else { 459 ret = budget; 460 } 461 462 u64_stats_update_begin(&tx_ring->syncp); 463 tx_ring->tx_stats.tx_poll++; 464 u64_stats_update_end(&tx_ring->syncp); 465 tx_ring->tx_stats.last_napi_jiffies = jiffies; 466 467 return ret; 468 } 469