1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
4 */
5
6 #include "ena_xdp.h"
7
validate_xdp_req_id(struct ena_ring * tx_ring,u16 req_id)8 static int validate_xdp_req_id(struct ena_ring *tx_ring, u16 req_id)
9 {
10 struct ena_tx_buffer *tx_info;
11
12 tx_info = &tx_ring->tx_buffer_info[req_id];
13 if (likely(tx_info->xdpf))
14 return 0;
15
16 return handle_invalid_req_id(tx_ring, req_id, tx_info, true);
17 }
18
ena_xdp_tx_map_frame(struct ena_ring * tx_ring,struct ena_tx_buffer * tx_info,struct xdp_frame * xdpf,struct ena_com_tx_ctx * ena_tx_ctx)19 static int ena_xdp_tx_map_frame(struct ena_ring *tx_ring,
20 struct ena_tx_buffer *tx_info,
21 struct xdp_frame *xdpf,
22 struct ena_com_tx_ctx *ena_tx_ctx)
23 {
24 struct ena_adapter *adapter = tx_ring->adapter;
25 struct ena_com_buf *ena_buf;
26 int push_len = 0;
27 dma_addr_t dma;
28 void *data;
29 u32 size;
30
31 tx_info->xdpf = xdpf;
32 data = tx_info->xdpf->data;
33 size = tx_info->xdpf->len;
34
35 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
36 /* Designate part of the packet for LLQ */
37 push_len = min_t(u32, size, tx_ring->tx_max_header_size);
38
39 ena_tx_ctx->push_header = data;
40
41 size -= push_len;
42 data += push_len;
43 }
44
45 ena_tx_ctx->header_len = push_len;
46
47 if (size > 0) {
48 dma = dma_map_single(tx_ring->dev,
49 data,
50 size,
51 DMA_TO_DEVICE);
52 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
53 goto error_report_dma_error;
54
55 tx_info->map_linear_data = 0;
56
57 ena_buf = tx_info->bufs;
58 ena_buf->paddr = dma;
59 ena_buf->len = size;
60
61 ena_tx_ctx->ena_bufs = ena_buf;
62 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;
63 }
64
65 return 0;
66
67 error_report_dma_error:
68 ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
69 &tx_ring->syncp);
70 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
71
72 return -EINVAL;
73 }
74
ena_xdp_xmit_frame(struct ena_ring * tx_ring,struct ena_adapter * adapter,struct xdp_frame * xdpf,int flags)75 int ena_xdp_xmit_frame(struct ena_ring *tx_ring,
76 struct ena_adapter *adapter,
77 struct xdp_frame *xdpf,
78 int flags)
79 {
80 struct ena_com_tx_ctx ena_tx_ctx = {};
81 struct ena_tx_buffer *tx_info;
82 u16 next_to_use, req_id;
83 int rc;
84
85 next_to_use = tx_ring->next_to_use;
86 req_id = tx_ring->free_ids[next_to_use];
87 tx_info = &tx_ring->tx_buffer_info[req_id];
88 tx_info->num_of_bufs = 0;
89
90 rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx);
91 if (unlikely(rc))
92 goto err;
93
94 ena_tx_ctx.req_id = req_id;
95
96 rc = ena_xmit_common(adapter,
97 tx_ring,
98 tx_info,
99 &ena_tx_ctx,
100 next_to_use,
101 xdpf->len);
102 if (rc)
103 goto error_unmap_dma;
104
105 /* trigger the dma engine. ena_ring_tx_doorbell()
106 * calls a memory barrier inside it.
107 */
108 if (flags & XDP_XMIT_FLUSH)
109 ena_ring_tx_doorbell(tx_ring);
110
111 return rc;
112
113 error_unmap_dma:
114 ena_unmap_tx_buff(tx_ring, tx_info);
115 err:
116 tx_info->xdpf = NULL;
117
118 return rc;
119 }
120
ena_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)121 int ena_xdp_xmit(struct net_device *dev, int n,
122 struct xdp_frame **frames, u32 flags)
123 {
124 struct ena_adapter *adapter = netdev_priv(dev);
125 struct ena_ring *tx_ring;
126 int qid, i, nxmit = 0;
127
128 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
129 return -EINVAL;
130
131 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
132 return -ENETDOWN;
133
134 /* We assume that all rings have the same XDP program */
135 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))
136 return -ENXIO;
137
138 qid = smp_processor_id() % adapter->xdp_num_queues;
139 qid += adapter->xdp_first_ring;
140 tx_ring = &adapter->tx_ring[qid];
141
142 /* Other CPU ids might try to send thorugh this queue */
143 spin_lock(&tx_ring->xdp_tx_lock);
144
145 for (i = 0; i < n; i++) {
146 if (ena_xdp_xmit_frame(tx_ring, adapter, frames[i], 0))
147 break;
148 nxmit++;
149 }
150
151 /* Ring doorbell to make device aware of the packets */
152 if (flags & XDP_XMIT_FLUSH)
153 ena_ring_tx_doorbell(tx_ring);
154
155 spin_unlock(&tx_ring->xdp_tx_lock);
156
157 /* Return number of packets sent */
158 return nxmit;
159 }
160
ena_init_all_xdp_queues(struct ena_adapter * adapter)161 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
162 {
163 adapter->xdp_first_ring = adapter->num_io_queues;
164 adapter->xdp_num_queues = adapter->num_io_queues;
165
166 ena_init_io_rings(adapter,
167 adapter->xdp_first_ring,
168 adapter->xdp_num_queues);
169 }
170
ena_setup_and_create_all_xdp_queues(struct ena_adapter * adapter)171 int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
172 {
173 u32 xdp_first_ring = adapter->xdp_first_ring;
174 u32 xdp_num_queues = adapter->xdp_num_queues;
175 int rc = 0;
176
177 rc = ena_setup_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
178 if (rc)
179 goto setup_err;
180
181 rc = ena_create_io_tx_queues_in_range(adapter, xdp_first_ring, xdp_num_queues);
182 if (rc)
183 goto create_err;
184
185 return 0;
186
187 create_err:
188 ena_free_all_io_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
189 setup_err:
190 return rc;
191 }
192
193 /* Provides a way for both kernel and bpf-prog to know
194 * more about the RX-queue a given XDP frame arrived on.
195 */
ena_xdp_register_rxq_info(struct ena_ring * rx_ring)196 int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
197 {
198 int rc;
199
200 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
201
202 netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d",
203 rx_ring->qid);
204 if (rc) {
205 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
206 "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
207 rx_ring->qid, rc);
208 goto err;
209 }
210
211 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL);
212
213 if (rc) {
214 netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
215 "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
216 rx_ring->qid, rc);
217 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
218 }
219
220 err:
221 return rc;
222 }
223
ena_xdp_unregister_rxq_info(struct ena_ring * rx_ring)224 void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
225 {
226 netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev,
227 "Unregistering RX info for queue %d",
228 rx_ring->qid);
229 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
230 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
231 }
232
ena_xdp_exchange_program_rx_in_range(struct ena_adapter * adapter,struct bpf_prog * prog,int first,int count)233 void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
234 struct bpf_prog *prog,
235 int first, int count)
236 {
237 struct bpf_prog *old_bpf_prog;
238 struct ena_ring *rx_ring;
239 int i = 0;
240
241 for (i = first; i < count; i++) {
242 rx_ring = &adapter->rx_ring[i];
243 old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog);
244
245 if (!old_bpf_prog && prog) {
246 rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
247 } else if (old_bpf_prog && !prog) {
248 rx_ring->rx_headroom = NET_SKB_PAD;
249 }
250 }
251 }
252
ena_xdp_exchange_program(struct ena_adapter * adapter,struct bpf_prog * prog)253 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
254 struct bpf_prog *prog)
255 {
256 struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
257
258 ena_xdp_exchange_program_rx_in_range(adapter,
259 prog,
260 0,
261 adapter->num_io_queues);
262
263 if (old_bpf_prog)
264 bpf_prog_put(old_bpf_prog);
265 }
266
ena_destroy_and_free_all_xdp_queues(struct ena_adapter * adapter)267 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
268 {
269 bool was_up;
270 int rc;
271
272 was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
273
274 if (was_up)
275 ena_down(adapter);
276
277 adapter->xdp_first_ring = 0;
278 adapter->xdp_num_queues = 0;
279 ena_xdp_exchange_program(adapter, NULL);
280 if (was_up) {
281 rc = ena_up(adapter);
282 if (rc)
283 return rc;
284 }
285 return 0;
286 }
287
ena_xdp_set(struct net_device * netdev,struct netdev_bpf * bpf)288 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
289 {
290 struct ena_adapter *adapter = netdev_priv(netdev);
291 struct bpf_prog *prog = bpf->prog;
292 struct bpf_prog *old_bpf_prog;
293 int rc, prev_mtu;
294 bool is_up;
295
296 is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
297 rc = ena_xdp_allowed(adapter);
298 if (rc == ENA_XDP_ALLOWED) {
299 old_bpf_prog = adapter->xdp_bpf_prog;
300 if (prog) {
301 if (!is_up) {
302 ena_init_all_xdp_queues(adapter);
303 } else if (!old_bpf_prog) {
304 ena_down(adapter);
305 ena_init_all_xdp_queues(adapter);
306 }
307 ena_xdp_exchange_program(adapter, prog);
308
309 netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n");
310
311 if (is_up && !old_bpf_prog) {
312 rc = ena_up(adapter);
313 if (rc)
314 return rc;
315 }
316 xdp_features_set_redirect_target(netdev, false);
317 } else if (old_bpf_prog) {
318 xdp_features_clear_redirect_target(netdev);
319 netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n");
320
321 rc = ena_destroy_and_free_all_xdp_queues(adapter);
322 if (rc)
323 return rc;
324 }
325
326 prev_mtu = netdev->max_mtu;
327 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
328
329 if (!old_bpf_prog)
330 netif_info(adapter, drv, adapter->netdev,
331 "XDP program is set, changing the max_mtu from %d to %d",
332 prev_mtu, netdev->max_mtu);
333
334 } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
335 netif_err(adapter, drv, adapter->netdev,
336 "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
337 netdev->mtu, ENA_XDP_MAX_MTU);
338 NL_SET_ERR_MSG_MOD(bpf->extack,
339 "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
340 return -EINVAL;
341 } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
342 netif_err(adapter, drv, adapter->netdev,
343 "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
344 adapter->num_io_queues, adapter->max_num_io_queues);
345 NL_SET_ERR_MSG_MOD(bpf->extack,
346 "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
347 return -EINVAL;
348 }
349
350 return 0;
351 }
352
353 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
354 * program as well as to query the current xdp program id.
355 */
ena_xdp(struct net_device * netdev,struct netdev_bpf * bpf)356 int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
357 {
358 switch (bpf->command) {
359 case XDP_SETUP_PROG:
360 return ena_xdp_set(netdev, bpf);
361 default:
362 return -EINVAL;
363 }
364 return 0;
365 }
366
ena_clean_xdp_irq(struct ena_ring * tx_ring,u32 budget)367 static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget)
368 {
369 u32 total_done = 0;
370 u16 next_to_clean;
371 int tx_pkts = 0;
372 u16 req_id;
373 int rc;
374
375 if (unlikely(!tx_ring))
376 return 0;
377 next_to_clean = tx_ring->next_to_clean;
378
379 while (tx_pkts < budget) {
380 struct ena_tx_buffer *tx_info;
381 struct xdp_frame *xdpf;
382
383 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
384 &req_id);
385 if (rc) {
386 if (unlikely(rc == -EINVAL))
387 handle_invalid_req_id(tx_ring, req_id, NULL, true);
388 break;
389 }
390
391 /* validate that the request id points to a valid xdp_frame */
392 rc = validate_xdp_req_id(tx_ring, req_id);
393 if (rc)
394 break;
395
396 tx_info = &tx_ring->tx_buffer_info[req_id];
397
398 tx_info->last_jiffies = 0;
399
400 xdpf = tx_info->xdpf;
401 tx_info->xdpf = NULL;
402 ena_unmap_tx_buff(tx_ring, tx_info);
403 xdp_return_frame(xdpf);
404
405 tx_pkts++;
406 total_done += tx_info->tx_descs;
407 tx_ring->free_ids[next_to_clean] = req_id;
408 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
409 tx_ring->ring_size);
410
411 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
412 "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id);
413 }
414
415 tx_ring->next_to_clean = next_to_clean;
416 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
417
418 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
419 "tx_poll: q %d done. total pkts: %d\n",
420 tx_ring->qid, tx_pkts);
421
422 return tx_pkts;
423 }
424
425 /* This is the XDP napi callback. XDP queues use a separate napi callback
426 * than Rx/Tx queues.
427 */
ena_xdp_io_poll(struct napi_struct * napi,int budget)428 int ena_xdp_io_poll(struct napi_struct *napi, int budget)
429 {
430 struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
431 struct ena_ring *tx_ring;
432 u32 work_done;
433 int ret;
434
435 tx_ring = ena_napi->tx_ring;
436
437 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
438 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
439 napi_complete_done(napi, 0);
440 return 0;
441 }
442
443 work_done = ena_clean_xdp_irq(tx_ring, budget);
444
445 /* If the device is about to reset or down, avoid unmask
446 * the interrupt and return 0 so NAPI won't reschedule
447 */
448 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) {
449 napi_complete_done(napi, 0);
450 ret = 0;
451 } else if (budget > work_done) {
452 ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1,
453 &tx_ring->syncp);
454 if (napi_complete_done(napi, work_done))
455 ena_unmask_interrupt(tx_ring, NULL);
456
457 ena_update_ring_numa_node(tx_ring, NULL);
458 ret = work_done;
459 } else {
460 ret = budget;
461 }
462
463 u64_stats_update_begin(&tx_ring->syncp);
464 tx_ring->tx_stats.tx_poll++;
465 u64_stats_update_end(&tx_ring->syncp);
466 tx_ring->tx_stats.last_napi_jiffies = jiffies;
467
468 return ret;
469 }
470