xref: /linux/drivers/net/ethernet/amazon/ena/ena_xdp.c (revision f6e0a4984c2e7244689ea87b62b433bed9d07e94)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #include "ena_xdp.h"
7 
8 static int validate_xdp_req_id(struct ena_ring *tx_ring, u16 req_id)
9 {
10 	struct ena_tx_buffer *tx_info;
11 
12 	tx_info = &tx_ring->tx_buffer_info[req_id];
13 	if (likely(tx_info->xdpf))
14 		return 0;
15 
16 	return handle_invalid_req_id(tx_ring, req_id, tx_info, true);
17 }
18 
19 static int ena_xdp_tx_map_frame(struct ena_ring *tx_ring,
20 				struct ena_tx_buffer *tx_info,
21 				struct xdp_frame *xdpf,
22 				struct ena_com_tx_ctx *ena_tx_ctx)
23 {
24 	struct ena_adapter *adapter = tx_ring->adapter;
25 	struct ena_com_buf *ena_buf;
26 	int push_len = 0;
27 	dma_addr_t dma;
28 	void *data;
29 	u32 size;
30 
31 	tx_info->xdpf = xdpf;
32 	data = tx_info->xdpf->data;
33 	size = tx_info->xdpf->len;
34 
35 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
36 		/* Designate part of the packet for LLQ */
37 		push_len = min_t(u32, size, tx_ring->tx_max_header_size);
38 
39 		ena_tx_ctx->push_header = data;
40 
41 		size -= push_len;
42 		data += push_len;
43 	}
44 
45 	ena_tx_ctx->header_len = push_len;
46 
47 	if (size > 0) {
48 		dma = dma_map_single(tx_ring->dev,
49 				     data,
50 				     size,
51 				     DMA_TO_DEVICE);
52 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
53 			goto error_report_dma_error;
54 
55 		tx_info->map_linear_data = 0;
56 
57 		ena_buf = tx_info->bufs;
58 		ena_buf->paddr = dma;
59 		ena_buf->len = size;
60 
61 		ena_tx_ctx->ena_bufs = ena_buf;
62 		ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;
63 	}
64 
65 	return 0;
66 
67 error_report_dma_error:
68 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
69 			  &tx_ring->syncp);
70 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
71 
72 	return -EINVAL;
73 }
74 
75 int ena_xdp_xmit_frame(struct ena_ring *tx_ring,
76 		       struct ena_adapter *adapter,
77 		       struct xdp_frame *xdpf,
78 		       int flags)
79 {
80 	struct ena_com_tx_ctx ena_tx_ctx = {};
81 	struct ena_tx_buffer *tx_info;
82 	u16 next_to_use, req_id;
83 	int rc;
84 
85 	next_to_use = tx_ring->next_to_use;
86 	req_id = tx_ring->free_ids[next_to_use];
87 	tx_info = &tx_ring->tx_buffer_info[req_id];
88 	tx_info->num_of_bufs = 0;
89 
90 	rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx);
91 	if (unlikely(rc))
92 		return rc;
93 
94 	ena_tx_ctx.req_id = req_id;
95 
96 	rc = ena_xmit_common(adapter,
97 			     tx_ring,
98 			     tx_info,
99 			     &ena_tx_ctx,
100 			     next_to_use,
101 			     xdpf->len);
102 	if (rc)
103 		goto error_unmap_dma;
104 
105 	/* trigger the dma engine. ena_ring_tx_doorbell()
106 	 * calls a memory barrier inside it.
107 	 */
108 	if (flags & XDP_XMIT_FLUSH)
109 		ena_ring_tx_doorbell(tx_ring);
110 
111 	return rc;
112 
113 error_unmap_dma:
114 	ena_unmap_tx_buff(tx_ring, tx_info);
115 	tx_info->xdpf = NULL;
116 	return rc;
117 }
118 
119 int ena_xdp_xmit(struct net_device *dev, int n,
120 		 struct xdp_frame **frames, u32 flags)
121 {
122 	struct ena_adapter *adapter = netdev_priv(dev);
123 	struct ena_ring *tx_ring;
124 	int qid, i, nxmit = 0;
125 
126 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
127 		return -EINVAL;
128 
129 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
130 		return -ENETDOWN;
131 
132 	/* We assume that all rings have the same XDP program */
133 	if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))
134 		return -ENXIO;
135 
136 	qid = smp_processor_id() % adapter->xdp_num_queues;
137 	qid += adapter->xdp_first_ring;
138 	tx_ring = &adapter->tx_ring[qid];
139 
140 	/* Other CPU ids might try to send thorugh this queue */
141 	spin_lock(&tx_ring->xdp_tx_lock);
142 
143 	for (i = 0; i < n; i++) {
144 		if (ena_xdp_xmit_frame(tx_ring, adapter, frames[i], 0))
145 			break;
146 		nxmit++;
147 	}
148 
149 	/* Ring doorbell to make device aware of the packets */
150 	if (flags & XDP_XMIT_FLUSH)
151 		ena_ring_tx_doorbell(tx_ring);
152 
153 	spin_unlock(&tx_ring->xdp_tx_lock);
154 
155 	/* Return number of packets sent */
156 	return nxmit;
157 }
158 
159 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
160 {
161 	adapter->xdp_first_ring = adapter->num_io_queues;
162 	adapter->xdp_num_queues = adapter->num_io_queues;
163 
164 	ena_init_io_rings(adapter,
165 			  adapter->xdp_first_ring,
166 			  adapter->xdp_num_queues);
167 }
168 
169 int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
170 {
171 	u32 xdp_first_ring = adapter->xdp_first_ring;
172 	u32 xdp_num_queues = adapter->xdp_num_queues;
173 	int rc = 0;
174 
175 	rc = ena_setup_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
176 	if (rc)
177 		goto setup_err;
178 
179 	rc = ena_create_io_tx_queues_in_range(adapter, xdp_first_ring, xdp_num_queues);
180 	if (rc)
181 		goto create_err;
182 
183 	return 0;
184 
185 create_err:
186 	ena_free_all_io_tx_resources_in_range(adapter, xdp_first_ring, xdp_num_queues);
187 setup_err:
188 	return rc;
189 }
190 
191 /* Provides a way for both kernel and bpf-prog to know
192  * more about the RX-queue a given XDP frame arrived on.
193  */
194 int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
195 {
196 	int rc;
197 
198 	rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
199 
200 	netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d",
201 		  rx_ring->qid);
202 	if (rc) {
203 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
204 			  "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
205 			  rx_ring->qid, rc);
206 		goto err;
207 	}
208 
209 	rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL);
210 
211 	if (rc) {
212 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
213 			  "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
214 			  rx_ring->qid, rc);
215 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
216 	}
217 
218 err:
219 	return rc;
220 }
221 
222 void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
223 {
224 	netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev,
225 		  "Unregistering RX info for queue %d",
226 		  rx_ring->qid);
227 	xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
228 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
229 }
230 
231 void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
232 					  struct bpf_prog *prog,
233 					  int first, int count)
234 {
235 	struct bpf_prog *old_bpf_prog;
236 	struct ena_ring *rx_ring;
237 	int i = 0;
238 
239 	for (i = first; i < count; i++) {
240 		rx_ring = &adapter->rx_ring[i];
241 		old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog);
242 
243 		if (!old_bpf_prog && prog) {
244 			rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
245 		} else if (old_bpf_prog && !prog) {
246 			rx_ring->rx_headroom = NET_SKB_PAD;
247 		}
248 	}
249 }
250 
251 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
252 				     struct bpf_prog *prog)
253 {
254 	struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
255 
256 	ena_xdp_exchange_program_rx_in_range(adapter,
257 					     prog,
258 					     0,
259 					     adapter->num_io_queues);
260 
261 	if (old_bpf_prog)
262 		bpf_prog_put(old_bpf_prog);
263 }
264 
265 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
266 {
267 	bool was_up;
268 	int rc;
269 
270 	was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
271 
272 	if (was_up)
273 		ena_down(adapter);
274 
275 	adapter->xdp_first_ring = 0;
276 	adapter->xdp_num_queues = 0;
277 	ena_xdp_exchange_program(adapter, NULL);
278 	if (was_up) {
279 		rc = ena_up(adapter);
280 		if (rc)
281 			return rc;
282 	}
283 	return 0;
284 }
285 
286 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
287 {
288 	struct ena_adapter *adapter = netdev_priv(netdev);
289 	struct bpf_prog *prog = bpf->prog;
290 	struct bpf_prog *old_bpf_prog;
291 	int rc, prev_mtu;
292 	bool is_up;
293 
294 	is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
295 	rc = ena_xdp_allowed(adapter);
296 	if (rc == ENA_XDP_ALLOWED) {
297 		old_bpf_prog = adapter->xdp_bpf_prog;
298 		if (prog) {
299 			if (!is_up) {
300 				ena_init_all_xdp_queues(adapter);
301 			} else if (!old_bpf_prog) {
302 				ena_down(adapter);
303 				ena_init_all_xdp_queues(adapter);
304 			}
305 			ena_xdp_exchange_program(adapter, prog);
306 
307 			netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n");
308 
309 			if (is_up && !old_bpf_prog) {
310 				rc = ena_up(adapter);
311 				if (rc)
312 					return rc;
313 			}
314 			xdp_features_set_redirect_target(netdev, false);
315 		} else if (old_bpf_prog) {
316 			xdp_features_clear_redirect_target(netdev);
317 			netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n");
318 
319 			rc = ena_destroy_and_free_all_xdp_queues(adapter);
320 			if (rc)
321 				return rc;
322 		}
323 
324 		prev_mtu = netdev->max_mtu;
325 		netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
326 
327 		if (!old_bpf_prog)
328 			netif_info(adapter, drv, adapter->netdev,
329 				   "XDP program is set, changing the max_mtu from %d to %d",
330 				   prev_mtu, netdev->max_mtu);
331 
332 	} else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
333 		netif_err(adapter, drv, adapter->netdev,
334 			  "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
335 			  netdev->mtu, ENA_XDP_MAX_MTU);
336 		NL_SET_ERR_MSG_MOD(bpf->extack,
337 				   "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
338 		return -EINVAL;
339 	} else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
340 		netif_err(adapter, drv, adapter->netdev,
341 			  "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
342 			  adapter->num_io_queues, adapter->max_num_io_queues);
343 		NL_SET_ERR_MSG_MOD(bpf->extack,
344 				   "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
345 		return -EINVAL;
346 	}
347 
348 	return 0;
349 }
350 
351 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
352  * program as well as to query the current xdp program id.
353  */
354 int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
355 {
356 	switch (bpf->command) {
357 	case XDP_SETUP_PROG:
358 		return ena_xdp_set(netdev, bpf);
359 	default:
360 		return -EINVAL;
361 	}
362 	return 0;
363 }
364 
365 static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget)
366 {
367 	u32 total_done = 0;
368 	u16 next_to_clean;
369 	int tx_pkts = 0;
370 	u16 req_id;
371 	int rc;
372 
373 	if (unlikely(!tx_ring))
374 		return 0;
375 	next_to_clean = tx_ring->next_to_clean;
376 
377 	while (tx_pkts < budget) {
378 		struct ena_tx_buffer *tx_info;
379 		struct xdp_frame *xdpf;
380 
381 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
382 						&req_id);
383 		if (rc) {
384 			if (unlikely(rc == -EINVAL))
385 				handle_invalid_req_id(tx_ring, req_id, NULL, true);
386 			break;
387 		}
388 
389 		/* validate that the request id points to a valid xdp_frame */
390 		rc = validate_xdp_req_id(tx_ring, req_id);
391 		if (rc)
392 			break;
393 
394 		tx_info = &tx_ring->tx_buffer_info[req_id];
395 
396 		tx_info->last_jiffies = 0;
397 
398 		xdpf = tx_info->xdpf;
399 		tx_info->xdpf = NULL;
400 		ena_unmap_tx_buff(tx_ring, tx_info);
401 		xdp_return_frame(xdpf);
402 
403 		tx_pkts++;
404 		total_done += tx_info->tx_descs;
405 		tx_ring->free_ids[next_to_clean] = req_id;
406 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
407 						     tx_ring->ring_size);
408 
409 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
410 			  "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id);
411 	}
412 
413 	tx_ring->next_to_clean = next_to_clean;
414 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
415 
416 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
417 		  "tx_poll: q %d done. total pkts: %d\n",
418 		  tx_ring->qid, tx_pkts);
419 
420 	return tx_pkts;
421 }
422 
423 /* This is the XDP napi callback. XDP queues use a separate napi callback
424  * than Rx/Tx queues.
425  */
426 int ena_xdp_io_poll(struct napi_struct *napi, int budget)
427 {
428 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
429 	struct ena_ring *tx_ring;
430 	u32 work_done;
431 	int ret;
432 
433 	tx_ring = ena_napi->tx_ring;
434 
435 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
436 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
437 		napi_complete_done(napi, 0);
438 		return 0;
439 	}
440 
441 	work_done = ena_clean_xdp_irq(tx_ring, budget);
442 
443 	/* If the device is about to reset or down, avoid unmask
444 	 * the interrupt and return 0 so NAPI won't reschedule
445 	 */
446 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) {
447 		napi_complete_done(napi, 0);
448 		ret = 0;
449 	} else if (budget > work_done) {
450 		ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1,
451 				  &tx_ring->syncp);
452 		if (napi_complete_done(napi, work_done))
453 			ena_unmask_interrupt(tx_ring, NULL);
454 
455 		ena_update_ring_numa_node(tx_ring, NULL);
456 		ret = work_done;
457 	} else {
458 		ret = budget;
459 	}
460 
461 	u64_stats_update_begin(&tx_ring->syncp);
462 	tx_ring->tx_stats.tx_poll++;
463 	u64_stats_update_end(&tx_ring->syncp);
464 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
465 
466 	return ret;
467 }
468