xref: /linux/net/core/xdp.c (revision 7505576d1c1ac0cfe85fdf90999433dd8b673012) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* net/core/xdp.c
3  *
4  * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
5  */
6 #include <linux/bpf.h>
7 #include <linux/filter.h>
8 #include <linux/types.h>
9 #include <linux/mm.h>
10 #include <linux/netdevice.h>
11 #include <linux/slab.h>
12 #include <linux/idr.h>
13 #include <linux/rhashtable.h>
14 #include <net/page_pool.h>
15 
16 #include <net/xdp.h>
17 #include <net/xdp_priv.h> /* struct xdp_mem_allocator */
18 #include <trace/events/xdp.h>
19 
20 #define REG_STATE_NEW		0x0
21 #define REG_STATE_REGISTERED	0x1
22 #define REG_STATE_UNREGISTERED	0x2
23 #define REG_STATE_UNUSED	0x3
24 
25 static DEFINE_IDA(mem_id_pool);
26 static DEFINE_MUTEX(mem_id_lock);
27 #define MEM_ID_MAX 0xFFFE
28 #define MEM_ID_MIN 1
29 static int mem_id_next = MEM_ID_MIN;
30 
31 static bool mem_id_init; /* false */
32 static struct rhashtable *mem_id_ht;
33 
34 static u32 xdp_mem_id_hashfn(const void *data, u32 len, u32 seed)
35 {
36 	const u32 *k = data;
37 	const u32 key = *k;
38 
39 	BUILD_BUG_ON(FIELD_SIZEOF(struct xdp_mem_allocator, mem.id)
40 		     != sizeof(u32));
41 
42 	/* Use cyclic increasing ID as direct hash key */
43 	return key;
44 }
45 
46 static int xdp_mem_id_cmp(struct rhashtable_compare_arg *arg,
47 			  const void *ptr)
48 {
49 	const struct xdp_mem_allocator *xa = ptr;
50 	u32 mem_id = *(u32 *)arg->key;
51 
52 	return xa->mem.id != mem_id;
53 }
54 
55 static const struct rhashtable_params mem_id_rht_params = {
56 	.nelem_hint = 64,
57 	.head_offset = offsetof(struct xdp_mem_allocator, node),
58 	.key_offset  = offsetof(struct xdp_mem_allocator, mem.id),
59 	.key_len = FIELD_SIZEOF(struct xdp_mem_allocator, mem.id),
60 	.max_size = MEM_ID_MAX,
61 	.min_size = 8,
62 	.automatic_shrinking = true,
63 	.hashfn    = xdp_mem_id_hashfn,
64 	.obj_cmpfn = xdp_mem_id_cmp,
65 };
66 
67 static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu)
68 {
69 	struct xdp_mem_allocator *xa;
70 
71 	xa = container_of(rcu, struct xdp_mem_allocator, rcu);
72 
73 	/* Allocator have indicated safe to remove before this is called */
74 	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
75 		page_pool_free(xa->page_pool);
76 
77 	/* Allow this ID to be reused */
78 	ida_simple_remove(&mem_id_pool, xa->mem.id);
79 
80 	/* Poison memory */
81 	xa->mem.id = 0xFFFF;
82 	xa->mem.type = 0xF0F0;
83 	xa->allocator = (void *)0xDEAD9001;
84 
85 	kfree(xa);
86 }
87 
88 static bool __mem_id_disconnect(int id, bool force)
89 {
90 	struct xdp_mem_allocator *xa;
91 	bool safe_to_remove = true;
92 
93 	mutex_lock(&mem_id_lock);
94 
95 	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
96 	if (!xa) {
97 		mutex_unlock(&mem_id_lock);
98 		WARN(1, "Request remove non-existing id(%d), driver bug?", id);
99 		return true;
100 	}
101 	xa->disconnect_cnt++;
102 
103 	/* Detects in-flight packet-pages for page_pool */
104 	if (xa->mem.type == MEM_TYPE_PAGE_POOL)
105 		safe_to_remove = page_pool_request_shutdown(xa->page_pool);
106 
107 	trace_mem_disconnect(xa, safe_to_remove, force);
108 
109 	if ((safe_to_remove || force) &&
110 	    !rhashtable_remove_fast(mem_id_ht, &xa->node, mem_id_rht_params))
111 		call_rcu(&xa->rcu, __xdp_mem_allocator_rcu_free);
112 
113 	mutex_unlock(&mem_id_lock);
114 	return (safe_to_remove|force);
115 }
116 
117 #define DEFER_TIME (msecs_to_jiffies(1000))
118 #define DEFER_WARN_INTERVAL (30 * HZ)
119 #define DEFER_MAX_RETRIES 120
120 
121 static void mem_id_disconnect_defer_retry(struct work_struct *wq)
122 {
123 	struct delayed_work *dwq = to_delayed_work(wq);
124 	struct xdp_mem_allocator *xa = container_of(dwq, typeof(*xa), defer_wq);
125 	bool force = false;
126 
127 	if (xa->disconnect_cnt > DEFER_MAX_RETRIES)
128 		force = true;
129 
130 	if (__mem_id_disconnect(xa->mem.id, force))
131 		return;
132 
133 	/* Periodic warning */
134 	if (time_after_eq(jiffies, xa->defer_warn)) {
135 		int sec = (s32)((u32)jiffies - (u32)xa->defer_start) / HZ;
136 
137 		pr_warn("%s() stalled mem.id=%u shutdown %d attempts %d sec\n",
138 			__func__, xa->mem.id, xa->disconnect_cnt, sec);
139 		xa->defer_warn = jiffies + DEFER_WARN_INTERVAL;
140 	}
141 
142 	/* Still not ready to be disconnected, retry later */
143 	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
144 }
145 
146 void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
147 {
148 	struct xdp_mem_allocator *xa;
149 	int id = xdp_rxq->mem.id;
150 
151 	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
152 		WARN(1, "Missing register, driver bug");
153 		return;
154 	}
155 
156 	if (xdp_rxq->mem.type != MEM_TYPE_PAGE_POOL &&
157 	    xdp_rxq->mem.type != MEM_TYPE_ZERO_COPY) {
158 		return;
159 	}
160 
161 	if (id == 0)
162 		return;
163 
164 	if (__mem_id_disconnect(id, false))
165 		return;
166 
167 	/* Could not disconnect, defer new disconnect attempt to later */
168 	mutex_lock(&mem_id_lock);
169 
170 	xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params);
171 	if (!xa) {
172 		mutex_unlock(&mem_id_lock);
173 		return;
174 	}
175 	xa->defer_start = jiffies;
176 	xa->defer_warn  = jiffies + DEFER_WARN_INTERVAL;
177 
178 	INIT_DELAYED_WORK(&xa->defer_wq, mem_id_disconnect_defer_retry);
179 	mutex_unlock(&mem_id_lock);
180 	schedule_delayed_work(&xa->defer_wq, DEFER_TIME);
181 }
182 EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg_mem_model);
183 
184 /* This unregister operation will also cleanup and destroy the
185  * allocator. The page_pool_free() operation is first called when it's
186  * safe to remove, possibly deferred to a workqueue.
187  */
188 void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq)
189 {
190 	/* Simplify driver cleanup code paths, allow unreg "unused" */
191 	if (xdp_rxq->reg_state == REG_STATE_UNUSED)
192 		return;
193 
194 	WARN(!(xdp_rxq->reg_state == REG_STATE_REGISTERED), "Driver BUG");
195 
196 	xdp_rxq_info_unreg_mem_model(xdp_rxq);
197 
198 	xdp_rxq->reg_state = REG_STATE_UNREGISTERED;
199 	xdp_rxq->dev = NULL;
200 
201 	/* Reset mem info to defaults */
202 	xdp_rxq->mem.id = 0;
203 	xdp_rxq->mem.type = 0;
204 }
205 EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg);
206 
207 static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq)
208 {
209 	memset(xdp_rxq, 0, sizeof(*xdp_rxq));
210 }
211 
212 /* Returns 0 on success, negative on failure */
213 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
214 		     struct net_device *dev, u32 queue_index)
215 {
216 	if (xdp_rxq->reg_state == REG_STATE_UNUSED) {
217 		WARN(1, "Driver promised not to register this");
218 		return -EINVAL;
219 	}
220 
221 	if (xdp_rxq->reg_state == REG_STATE_REGISTERED) {
222 		WARN(1, "Missing unregister, handled but fix driver");
223 		xdp_rxq_info_unreg(xdp_rxq);
224 	}
225 
226 	if (!dev) {
227 		WARN(1, "Missing net_device from driver");
228 		return -ENODEV;
229 	}
230 
231 	/* State either UNREGISTERED or NEW */
232 	xdp_rxq_info_init(xdp_rxq);
233 	xdp_rxq->dev = dev;
234 	xdp_rxq->queue_index = queue_index;
235 
236 	xdp_rxq->reg_state = REG_STATE_REGISTERED;
237 	return 0;
238 }
239 EXPORT_SYMBOL_GPL(xdp_rxq_info_reg);
240 
241 void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq)
242 {
243 	xdp_rxq->reg_state = REG_STATE_UNUSED;
244 }
245 EXPORT_SYMBOL_GPL(xdp_rxq_info_unused);
246 
247 bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq)
248 {
249 	return (xdp_rxq->reg_state == REG_STATE_REGISTERED);
250 }
251 EXPORT_SYMBOL_GPL(xdp_rxq_info_is_reg);
252 
253 static int __mem_id_init_hash_table(void)
254 {
255 	struct rhashtable *rht;
256 	int ret;
257 
258 	if (unlikely(mem_id_init))
259 		return 0;
260 
261 	rht = kzalloc(sizeof(*rht), GFP_KERNEL);
262 	if (!rht)
263 		return -ENOMEM;
264 
265 	ret = rhashtable_init(rht, &mem_id_rht_params);
266 	if (ret < 0) {
267 		kfree(rht);
268 		return ret;
269 	}
270 	mem_id_ht = rht;
271 	smp_mb(); /* mutex lock should provide enough pairing */
272 	mem_id_init = true;
273 
274 	return 0;
275 }
276 
277 /* Allocate a cyclic ID that maps to allocator pointer.
278  * See: https://www.kernel.org/doc/html/latest/core-api/idr.html
279  *
280  * Caller must lock mem_id_lock.
281  */
282 static int __mem_id_cyclic_get(gfp_t gfp)
283 {
284 	int retries = 1;
285 	int id;
286 
287 again:
288 	id = ida_simple_get(&mem_id_pool, mem_id_next, MEM_ID_MAX, gfp);
289 	if (id < 0) {
290 		if (id == -ENOSPC) {
291 			/* Cyclic allocator, reset next id */
292 			if (retries--) {
293 				mem_id_next = MEM_ID_MIN;
294 				goto again;
295 			}
296 		}
297 		return id; /* errno */
298 	}
299 	mem_id_next = id + 1;
300 
301 	return id;
302 }
303 
304 static bool __is_supported_mem_type(enum xdp_mem_type type)
305 {
306 	if (type == MEM_TYPE_PAGE_POOL)
307 		return is_page_pool_compiled_in();
308 
309 	if (type >= MEM_TYPE_MAX)
310 		return false;
311 
312 	return true;
313 }
314 
315 int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
316 			       enum xdp_mem_type type, void *allocator)
317 {
318 	struct xdp_mem_allocator *xdp_alloc;
319 	gfp_t gfp = GFP_KERNEL;
320 	int id, errno, ret;
321 	void *ptr;
322 
323 	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
324 		WARN(1, "Missing register, driver bug");
325 		return -EFAULT;
326 	}
327 
328 	if (!__is_supported_mem_type(type))
329 		return -EOPNOTSUPP;
330 
331 	xdp_rxq->mem.type = type;
332 
333 	if (!allocator) {
334 		if (type == MEM_TYPE_PAGE_POOL || type == MEM_TYPE_ZERO_COPY)
335 			return -EINVAL; /* Setup time check page_pool req */
336 		return 0;
337 	}
338 
339 	/* Delay init of rhashtable to save memory if feature isn't used */
340 	if (!mem_id_init) {
341 		mutex_lock(&mem_id_lock);
342 		ret = __mem_id_init_hash_table();
343 		mutex_unlock(&mem_id_lock);
344 		if (ret < 0) {
345 			WARN_ON(1);
346 			return ret;
347 		}
348 	}
349 
350 	xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp);
351 	if (!xdp_alloc)
352 		return -ENOMEM;
353 
354 	mutex_lock(&mem_id_lock);
355 	id = __mem_id_cyclic_get(gfp);
356 	if (id < 0) {
357 		errno = id;
358 		goto err;
359 	}
360 	xdp_rxq->mem.id = id;
361 	xdp_alloc->mem  = xdp_rxq->mem;
362 	xdp_alloc->allocator = allocator;
363 
364 	/* Insert allocator into ID lookup table */
365 	ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node);
366 	if (IS_ERR(ptr)) {
367 		ida_simple_remove(&mem_id_pool, xdp_rxq->mem.id);
368 		xdp_rxq->mem.id = 0;
369 		errno = PTR_ERR(ptr);
370 		goto err;
371 	}
372 
373 	if (type == MEM_TYPE_PAGE_POOL)
374 		page_pool_get(xdp_alloc->page_pool);
375 
376 	mutex_unlock(&mem_id_lock);
377 
378 	trace_mem_connect(xdp_alloc, xdp_rxq);
379 	return 0;
380 err:
381 	mutex_unlock(&mem_id_lock);
382 	kfree(xdp_alloc);
383 	return errno;
384 }
385 EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
386 
387 /* XDP RX runs under NAPI protection, and in different delivery error
388  * scenarios (e.g. queue full), it is possible to return the xdp_frame
389  * while still leveraging this protection.  The @napi_direct boolian
390  * is used for those calls sites.  Thus, allowing for faster recycling
391  * of xdp_frames/pages in those cases.
392  */
393 static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
394 			 unsigned long handle)
395 {
396 	struct xdp_mem_allocator *xa;
397 	struct page *page;
398 
399 	switch (mem->type) {
400 	case MEM_TYPE_PAGE_POOL:
401 		rcu_read_lock();
402 		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
403 		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
404 		page = virt_to_head_page(data);
405 		if (likely(xa)) {
406 			napi_direct &= !xdp_return_frame_no_direct();
407 			page_pool_put_page(xa->page_pool, page, napi_direct);
408 		} else {
409 			/* Hopefully stack show who to blame for late return */
410 			WARN_ONCE(1, "page_pool gone mem.id=%d", mem->id);
411 			trace_mem_return_failed(mem, page);
412 			put_page(page);
413 		}
414 		rcu_read_unlock();
415 		break;
416 	case MEM_TYPE_PAGE_SHARED:
417 		page_frag_free(data);
418 		break;
419 	case MEM_TYPE_PAGE_ORDER0:
420 		page = virt_to_page(data); /* Assumes order0 page*/
421 		put_page(page);
422 		break;
423 	case MEM_TYPE_ZERO_COPY:
424 		/* NB! Only valid from an xdp_buff! */
425 		rcu_read_lock();
426 		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
427 		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
428 		xa->zc_alloc->free(xa->zc_alloc, handle);
429 		rcu_read_unlock();
430 	default:
431 		/* Not possible, checked in xdp_rxq_info_reg_mem_model() */
432 		break;
433 	}
434 }
435 
436 void xdp_return_frame(struct xdp_frame *xdpf)
437 {
438 	__xdp_return(xdpf->data, &xdpf->mem, false, 0);
439 }
440 EXPORT_SYMBOL_GPL(xdp_return_frame);
441 
442 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
443 {
444 	__xdp_return(xdpf->data, &xdpf->mem, true, 0);
445 }
446 EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
447 
448 void xdp_return_buff(struct xdp_buff *xdp)
449 {
450 	__xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle);
451 }
452 EXPORT_SYMBOL_GPL(xdp_return_buff);
453 
454 /* Only called for MEM_TYPE_PAGE_POOL see xdp.h */
455 void __xdp_release_frame(void *data, struct xdp_mem_info *mem)
456 {
457 	struct xdp_mem_allocator *xa;
458 	struct page *page;
459 
460 	rcu_read_lock();
461 	xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
462 	page = virt_to_head_page(data);
463 	if (xa)
464 		page_pool_release_page(xa->page_pool, page);
465 	rcu_read_unlock();
466 }
467 EXPORT_SYMBOL_GPL(__xdp_release_frame);
468 
469 int xdp_attachment_query(struct xdp_attachment_info *info,
470 			 struct netdev_bpf *bpf)
471 {
472 	bpf->prog_id = info->prog ? info->prog->aux->id : 0;
473 	bpf->prog_flags = info->prog ? info->flags : 0;
474 	return 0;
475 }
476 EXPORT_SYMBOL_GPL(xdp_attachment_query);
477 
478 bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
479 			     struct netdev_bpf *bpf)
480 {
481 	if (info->prog && (bpf->flags ^ info->flags) & XDP_FLAGS_MODES) {
482 		NL_SET_ERR_MSG(bpf->extack,
483 			       "program loaded with different flags");
484 		return false;
485 	}
486 	return true;
487 }
488 EXPORT_SYMBOL_GPL(xdp_attachment_flags_ok);
489 
490 void xdp_attachment_setup(struct xdp_attachment_info *info,
491 			  struct netdev_bpf *bpf)
492 {
493 	if (info->prog)
494 		bpf_prog_put(info->prog);
495 	info->prog = bpf->prog;
496 	info->flags = bpf->flags;
497 }
498 EXPORT_SYMBOL_GPL(xdp_attachment_setup);
499 
500 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
501 {
502 	unsigned int metasize, totsize;
503 	void *addr, *data_to_copy;
504 	struct xdp_frame *xdpf;
505 	struct page *page;
506 
507 	/* Clone into a MEM_TYPE_PAGE_ORDER0 xdp_frame. */
508 	metasize = xdp_data_meta_unsupported(xdp) ? 0 :
509 		   xdp->data - xdp->data_meta;
510 	totsize = xdp->data_end - xdp->data + metasize;
511 
512 	if (sizeof(*xdpf) + totsize > PAGE_SIZE)
513 		return NULL;
514 
515 	page = dev_alloc_page();
516 	if (!page)
517 		return NULL;
518 
519 	addr = page_to_virt(page);
520 	xdpf = addr;
521 	memset(xdpf, 0, sizeof(*xdpf));
522 
523 	addr += sizeof(*xdpf);
524 	data_to_copy = metasize ? xdp->data_meta : xdp->data;
525 	memcpy(addr, data_to_copy, totsize);
526 
527 	xdpf->data = addr + metasize;
528 	xdpf->len = totsize - metasize;
529 	xdpf->headroom = 0;
530 	xdpf->metasize = metasize;
531 	xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
532 
533 	xdp_return_buff(xdp);
534 	return xdpf;
535 }
536 EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame);
537