xref: /linux/kernel/bpf/bpf_iter.c (revision 55fbedcec622a943cb09a969aef1ddcba87ee40a)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020 Facebook */
3 
4 #include <linux/fs.h>
5 #include <linux/anon_inodes.h>
6 #include <linux/filter.h>
7 #include <linux/bpf.h>
8 #include <linux/rcupdate_trace.h>
9 
10 struct bpf_iter_target_info {
11 	struct list_head list;
12 	const struct bpf_iter_reg *reg_info;
13 	u32 btf_id;	/* cached value */
14 };
15 
16 struct bpf_iter_link {
17 	struct bpf_link link;
18 	struct bpf_iter_aux_info aux;
19 	struct bpf_iter_target_info *tinfo;
20 };
21 
22 struct bpf_iter_priv_data {
23 	struct bpf_iter_target_info *tinfo;
24 	const struct bpf_iter_seq_info *seq_info;
25 	struct bpf_prog *prog;
26 	u64 session_id;
27 	u64 seq_num;
28 	bool done_stop;
29 	u8 target_private[] __aligned(8);
30 };
31 
32 static struct list_head targets = LIST_HEAD_INIT(targets);
33 static DEFINE_MUTEX(targets_mutex);
34 
35 /* protect bpf_iter_link changes */
36 static DEFINE_MUTEX(link_mutex);
37 
38 /* incremented on every opened seq_file */
39 static atomic64_t session_id;
40 
41 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link);
42 
43 static void bpf_iter_inc_seq_num(struct seq_file *seq)
44 {
45 	struct bpf_iter_priv_data *iter_priv;
46 
47 	iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
48 				 target_private);
49 	iter_priv->seq_num++;
50 }
51 
52 static void bpf_iter_dec_seq_num(struct seq_file *seq)
53 {
54 	struct bpf_iter_priv_data *iter_priv;
55 
56 	iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
57 				 target_private);
58 	iter_priv->seq_num--;
59 }
60 
61 static void bpf_iter_done_stop(struct seq_file *seq)
62 {
63 	struct bpf_iter_priv_data *iter_priv;
64 
65 	iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
66 				 target_private);
67 	iter_priv->done_stop = true;
68 }
69 
70 static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
71 {
72 	return tinfo->reg_info->feature & BPF_ITER_RESCHED;
73 }
74 
75 /* Only allow sleepable program for resched-able iterator that can run one */
76 static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo,
77 					 const struct bpf_prog *prog)
78 {
79 	return !prog->sleepable ||
80 	       (bpf_iter_target_support_resched(tinfo) &&
81 		!(tinfo->reg_info->feature & BPF_ITER_NOSLEEP));
82 }
83 
84 static bool bpf_iter_support_resched(struct seq_file *seq)
85 {
86 	struct bpf_iter_priv_data *iter_priv;
87 
88 	iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
89 				 target_private);
90 	return bpf_iter_target_support_resched(iter_priv->tinfo);
91 }
92 
93 /* maximum visited objects before bailing out */
94 #define MAX_ITER_OBJECTS	1000000
95 
96 /* bpf_seq_read, a customized and simpler version for bpf iterator.
97  * The following are differences from seq_read():
98  *  . fixed buffer size (PAGE_SIZE << 3)
99  *  . assuming NULL ->llseek()
100  *  . stop() may call bpf program, handling potential overflow there
101  */
102 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
103 			    loff_t *ppos)
104 {
105 	struct seq_file *seq = file->private_data;
106 	size_t n, offs, copied = 0;
107 	int err = 0, num_objs = 0;
108 	bool can_resched;
109 	void *p;
110 
111 	mutex_lock(&seq->lock);
112 
113 	if (!seq->buf) {
114 		seq->size = PAGE_SIZE << 3;
115 		seq->buf = kvmalloc(seq->size, GFP_KERNEL);
116 		if (!seq->buf) {
117 			err = -ENOMEM;
118 			goto done;
119 		}
120 	}
121 
122 	if (seq->count) {
123 		n = min(seq->count, size);
124 		err = copy_to_user(buf, seq->buf + seq->from, n);
125 		if (err) {
126 			err = -EFAULT;
127 			goto done;
128 		}
129 		seq->count -= n;
130 		seq->from += n;
131 		copied = n;
132 		goto done;
133 	}
134 
135 	seq->from = 0;
136 	p = seq->op->start(seq, &seq->index);
137 	if (!p)
138 		goto stop;
139 	if (IS_ERR(p)) {
140 		err = PTR_ERR(p);
141 		seq->op->stop(seq, p);
142 		seq->count = 0;
143 		goto done;
144 	}
145 
146 	err = seq->op->show(seq, p);
147 	if (err > 0) {
148 		/* object is skipped, decrease seq_num, so next
149 		 * valid object can reuse the same seq_num.
150 		 */
151 		bpf_iter_dec_seq_num(seq);
152 		seq->count = 0;
153 	} else if (err < 0 || seq_has_overflowed(seq)) {
154 		if (!err)
155 			err = -E2BIG;
156 		seq->op->stop(seq, p);
157 		seq->count = 0;
158 		goto done;
159 	}
160 
161 	can_resched = bpf_iter_support_resched(seq);
162 	while (1) {
163 		loff_t pos = seq->index;
164 
165 		num_objs++;
166 		offs = seq->count;
167 		p = seq->op->next(seq, p, &seq->index);
168 		if (pos == seq->index) {
169 			pr_info_ratelimited("buggy seq_file .next function %ps "
170 				"did not updated position index\n",
171 				seq->op->next);
172 			seq->index++;
173 		}
174 
175 		if (IS_ERR_OR_NULL(p))
176 			break;
177 
178 		/* got a valid next object, increase seq_num */
179 		bpf_iter_inc_seq_num(seq);
180 
181 		if (seq->count >= size)
182 			break;
183 
184 		if (num_objs >= MAX_ITER_OBJECTS) {
185 			if (offs == 0) {
186 				err = -EAGAIN;
187 				seq->op->stop(seq, p);
188 				goto done;
189 			}
190 			break;
191 		}
192 
193 		err = seq->op->show(seq, p);
194 		if (err > 0) {
195 			bpf_iter_dec_seq_num(seq);
196 			seq->count = offs;
197 		} else if (err < 0 || seq_has_overflowed(seq)) {
198 			seq->count = offs;
199 			if (offs == 0) {
200 				if (!err)
201 					err = -E2BIG;
202 				seq->op->stop(seq, p);
203 				goto done;
204 			}
205 			break;
206 		}
207 
208 		if (can_resched)
209 			cond_resched();
210 	}
211 stop:
212 	offs = seq->count;
213 	if (IS_ERR(p)) {
214 		seq->op->stop(seq, NULL);
215 		err = PTR_ERR(p);
216 		goto done;
217 	}
218 	/* bpf program called if !p */
219 	seq->op->stop(seq, p);
220 	if (!p) {
221 		if (!seq_has_overflowed(seq)) {
222 			bpf_iter_done_stop(seq);
223 		} else {
224 			seq->count = offs;
225 			if (offs == 0) {
226 				err = -E2BIG;
227 				goto done;
228 			}
229 		}
230 	}
231 
232 	n = min(seq->count, size);
233 	err = copy_to_user(buf, seq->buf, n);
234 	if (err) {
235 		err = -EFAULT;
236 		goto done;
237 	}
238 	copied = n;
239 	seq->count -= n;
240 	seq->from = n;
241 done:
242 	if (!copied)
243 		copied = err;
244 	else
245 		*ppos += copied;
246 	mutex_unlock(&seq->lock);
247 	return copied;
248 }
249 
250 static const struct bpf_iter_seq_info *
251 __get_seq_info(struct bpf_iter_link *link)
252 {
253 	const struct bpf_iter_seq_info *seq_info;
254 
255 	if (link->aux.map) {
256 		seq_info = link->aux.map->ops->iter_seq_info;
257 		if (seq_info)
258 			return seq_info;
259 	}
260 
261 	return link->tinfo->reg_info->seq_info;
262 }
263 
264 static int iter_open(struct inode *inode, struct file *file)
265 {
266 	struct bpf_iter_link *link = inode->i_private;
267 
268 	return prepare_seq_file(file, link);
269 }
270 
271 static int iter_release(struct inode *inode, struct file *file)
272 {
273 	struct bpf_iter_priv_data *iter_priv;
274 	struct seq_file *seq;
275 
276 	seq = file->private_data;
277 	if (!seq)
278 		return 0;
279 
280 	iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
281 				 target_private);
282 
283 	if (iter_priv->seq_info->fini_seq_private)
284 		iter_priv->seq_info->fini_seq_private(seq->private);
285 
286 	bpf_prog_put(iter_priv->prog);
287 	seq->private = iter_priv;
288 
289 	return seq_release_private(inode, file);
290 }
291 
292 const struct file_operations bpf_iter_fops = {
293 	.open		= iter_open,
294 	.read		= bpf_seq_read,
295 	.release	= iter_release,
296 };
297 
298 /* The argument reg_info will be cached in bpf_iter_target_info.
299  * The common practice is to declare target reg_info as
300  * a const static variable and passed as an argument to
301  * bpf_iter_reg_target().
302  */
303 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
304 {
305 	struct bpf_iter_target_info *tinfo;
306 
307 	tinfo = kzalloc_obj(*tinfo);
308 	if (!tinfo)
309 		return -ENOMEM;
310 
311 	tinfo->reg_info = reg_info;
312 	INIT_LIST_HEAD(&tinfo->list);
313 
314 	mutex_lock(&targets_mutex);
315 	list_add(&tinfo->list, &targets);
316 	mutex_unlock(&targets_mutex);
317 
318 	return 0;
319 }
320 
321 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
322 {
323 	struct bpf_iter_target_info *tinfo;
324 	bool found = false;
325 
326 	mutex_lock(&targets_mutex);
327 	list_for_each_entry(tinfo, &targets, list) {
328 		if (reg_info == tinfo->reg_info) {
329 			list_del(&tinfo->list);
330 			kfree(tinfo);
331 			found = true;
332 			break;
333 		}
334 	}
335 	mutex_unlock(&targets_mutex);
336 
337 	WARN_ON(found == false);
338 }
339 
340 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
341 			 struct bpf_prog *prog)
342 {
343 	tinfo->btf_id = prog->aux->attach_btf_id;
344 }
345 
346 int bpf_iter_prog_supported(struct bpf_prog *prog)
347 {
348 	const char *attach_fname = prog->aux->attach_func_name;
349 	struct bpf_iter_target_info *tinfo = NULL, *iter;
350 	u32 prog_btf_id = prog->aux->attach_btf_id;
351 	const char *prefix = BPF_ITER_FUNC_PREFIX;
352 	int prefix_len = strlen(prefix);
353 
354 	/*
355 	 * The attach function of a program that has a target program is a
356 	 * function of that program, whatever its name is.
357 	 */
358 	if (prog->aux->dst_prog)
359 		return -EINVAL;
360 
361 	if (strncmp(attach_fname, prefix, prefix_len))
362 		return -EINVAL;
363 
364 	mutex_lock(&targets_mutex);
365 	list_for_each_entry(iter, &targets, list) {
366 		if (iter->btf_id && iter->btf_id == prog_btf_id) {
367 			tinfo = iter;
368 			break;
369 		}
370 		if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) {
371 			cache_btf_id(iter, prog);
372 			tinfo = iter;
373 			break;
374 		}
375 	}
376 	mutex_unlock(&targets_mutex);
377 
378 	if (!tinfo)
379 		return -EINVAL;
380 
381 	return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info,
382 					  tinfo->reg_info->ctx_arg_info_size);
383 }
384 
385 const struct bpf_func_proto *
386 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
387 {
388 	const struct bpf_iter_target_info *tinfo;
389 	const struct bpf_func_proto *fn = NULL;
390 
391 	mutex_lock(&targets_mutex);
392 	list_for_each_entry(tinfo, &targets, list) {
393 		if (tinfo->btf_id == prog->aux->attach_btf_id) {
394 			const struct bpf_iter_reg *reg_info;
395 
396 			reg_info = tinfo->reg_info;
397 			if (reg_info->get_func_proto)
398 				fn = reg_info->get_func_proto(func_id, prog);
399 			break;
400 		}
401 	}
402 	mutex_unlock(&targets_mutex);
403 
404 	return fn;
405 }
406 
407 static void bpf_iter_link_release(struct bpf_link *link)
408 {
409 	struct bpf_iter_link *iter_link =
410 		container_of(link, struct bpf_iter_link, link);
411 
412 	if (iter_link->tinfo->reg_info->detach_target)
413 		iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
414 }
415 
416 static void bpf_iter_link_dealloc(struct bpf_link *link)
417 {
418 	struct bpf_iter_link *iter_link =
419 		container_of(link, struct bpf_iter_link, link);
420 
421 	kfree(iter_link);
422 }
423 
424 static int bpf_iter_link_replace(struct bpf_link *link,
425 				 struct bpf_prog *new_prog,
426 				 struct bpf_prog *old_prog)
427 {
428 	struct bpf_iter_link *iter_link =
429 		container_of(link, struct bpf_iter_link, link);
430 	int ret = 0;
431 
432 	mutex_lock(&link_mutex);
433 	if (old_prog && link->prog != old_prog) {
434 		ret = -EPERM;
435 		goto out_unlock;
436 	}
437 
438 	if (link->prog->type != new_prog->type ||
439 	    link->prog->expected_attach_type != new_prog->expected_attach_type ||
440 	    link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id ||
441 	    !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) {
442 		ret = -EINVAL;
443 		goto out_unlock;
444 	}
445 
446 	if (iter_link->tinfo->reg_info->check_prog) {
447 		ret = iter_link->tinfo->reg_info->check_prog(new_prog,
448 							     &iter_link->aux);
449 		if (ret)
450 			goto out_unlock;
451 	}
452 
453 	old_prog = xchg(&link->prog, new_prog);
454 	bpf_prog_put(old_prog);
455 
456 out_unlock:
457 	mutex_unlock(&link_mutex);
458 	return ret;
459 }
460 
461 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
462 				      struct seq_file *seq)
463 {
464 	struct bpf_iter_link *iter_link =
465 		container_of(link, struct bpf_iter_link, link);
466 	bpf_iter_show_fdinfo_t show_fdinfo;
467 
468 	seq_printf(seq,
469 		   "target_name:\t%s\n",
470 		   iter_link->tinfo->reg_info->target);
471 
472 	show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
473 	if (show_fdinfo)
474 		show_fdinfo(&iter_link->aux, seq);
475 }
476 
477 static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
478 					struct bpf_link_info *info)
479 {
480 	struct bpf_iter_link *iter_link =
481 		container_of(link, struct bpf_iter_link, link);
482 	char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
483 	bpf_iter_fill_link_info_t fill_link_info;
484 	u32 ulen = info->iter.target_name_len;
485 	const char *target_name;
486 	u32 target_len;
487 
488 	if (!ulen ^ !ubuf)
489 		return -EINVAL;
490 
491 	target_name = iter_link->tinfo->reg_info->target;
492 	target_len =  strlen(target_name);
493 	info->iter.target_name_len = target_len + 1;
494 
495 	if (ubuf) {
496 		if (ulen >= target_len + 1) {
497 			if (copy_to_user(ubuf, target_name, target_len + 1))
498 				return -EFAULT;
499 		} else {
500 			char zero = '\0';
501 
502 			if (copy_to_user(ubuf, target_name, ulen - 1))
503 				return -EFAULT;
504 			if (put_user(zero, ubuf + ulen - 1))
505 				return -EFAULT;
506 			return -ENOSPC;
507 		}
508 	}
509 
510 	fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
511 	if (fill_link_info)
512 		return fill_link_info(&iter_link->aux, info);
513 
514 	return 0;
515 }
516 
517 static const struct bpf_link_ops bpf_iter_link_lops = {
518 	.release = bpf_iter_link_release,
519 	.dealloc = bpf_iter_link_dealloc,
520 	.update_prog = bpf_iter_link_replace,
521 	.show_fdinfo = bpf_iter_link_show_fdinfo,
522 	.fill_link_info = bpf_iter_link_fill_link_info,
523 };
524 
525 bool bpf_link_is_iter(struct bpf_link *link)
526 {
527 	return link->ops == &bpf_iter_link_lops;
528 }
529 
530 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
531 			 struct bpf_prog *prog)
532 {
533 	struct bpf_iter_target_info *tinfo = NULL, *iter;
534 	struct bpf_link_primer link_primer;
535 	union bpf_iter_link_info linfo;
536 	struct bpf_iter_link *link;
537 	u32 prog_btf_id, linfo_len;
538 	bpfptr_t ulinfo;
539 	int err;
540 
541 	if (attr->link_create.target_fd || attr->link_create.flags)
542 		return -EINVAL;
543 
544 	memset(&linfo, 0, sizeof(union bpf_iter_link_info));
545 
546 	ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
547 	linfo_len = attr->link_create.iter_info_len;
548 	if (bpfptr_is_null(ulinfo) ^ !linfo_len)
549 		return -EINVAL;
550 
551 	if (!bpfptr_is_null(ulinfo)) {
552 		err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
553 					       linfo_len);
554 		if (err)
555 			return err;
556 		linfo_len = min_t(u32, linfo_len, sizeof(linfo));
557 		if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
558 			return -EFAULT;
559 	}
560 
561 	prog_btf_id = prog->aux->attach_btf_id;
562 	mutex_lock(&targets_mutex);
563 	list_for_each_entry(iter, &targets, list) {
564 		if (iter->btf_id == prog_btf_id) {
565 			tinfo = iter;
566 			break;
567 		}
568 	}
569 	mutex_unlock(&targets_mutex);
570 	if (!tinfo)
571 		return -ENOENT;
572 
573 	if (!bpf_iter_target_support_prog(tinfo, prog))
574 		return -EINVAL;
575 
576 	link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN);
577 	if (!link)
578 		return -ENOMEM;
579 
580 	bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog,
581 		      attr->link_create.attach_type);
582 	link->tinfo = tinfo;
583 
584 	err = bpf_link_prime(&link->link, &link_primer);
585 	if (err) {
586 		kfree(link);
587 		return err;
588 	}
589 
590 	if (tinfo->reg_info->attach_target) {
591 		err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
592 		if (err) {
593 			bpf_link_cleanup(&link_primer);
594 			return err;
595 		}
596 	}
597 
598 	return bpf_link_settle(&link_primer);
599 }
600 
601 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
602 			  struct bpf_iter_target_info *tinfo,
603 			  const struct bpf_iter_seq_info *seq_info,
604 			  struct bpf_prog *prog)
605 {
606 	priv_data->tinfo = tinfo;
607 	priv_data->seq_info = seq_info;
608 	priv_data->prog = prog;
609 	priv_data->session_id = atomic64_inc_return(&session_id);
610 	priv_data->seq_num = 0;
611 	priv_data->done_stop = false;
612 }
613 
614 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link)
615 {
616 	const struct bpf_iter_seq_info *seq_info = __get_seq_info(link);
617 	struct bpf_iter_priv_data *priv_data;
618 	struct bpf_iter_target_info *tinfo;
619 	struct bpf_prog *prog;
620 	u32 total_priv_dsize;
621 	struct seq_file *seq;
622 	int err = 0;
623 
624 	mutex_lock(&link_mutex);
625 	prog = link->link.prog;
626 	bpf_prog_inc(prog);
627 	mutex_unlock(&link_mutex);
628 
629 	tinfo = link->tinfo;
630 	total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
631 			   seq_info->seq_priv_size;
632 	priv_data = __seq_open_private(file, seq_info->seq_ops,
633 				       total_priv_dsize);
634 	if (!priv_data) {
635 		err = -ENOMEM;
636 		goto release_prog;
637 	}
638 
639 	if (seq_info->init_seq_private) {
640 		err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
641 		if (err)
642 			goto release_seq_file;
643 	}
644 
645 	init_seq_meta(priv_data, tinfo, seq_info, prog);
646 	seq = file->private_data;
647 	seq->private = priv_data->target_private;
648 
649 	return 0;
650 
651 release_seq_file:
652 	seq_release_private(file->f_inode, file);
653 	file->private_data = NULL;
654 release_prog:
655 	bpf_prog_put(prog);
656 	return err;
657 }
658 
659 int bpf_iter_new_fd(struct bpf_link *link)
660 {
661 	struct bpf_iter_link *iter_link;
662 	unsigned int flags;
663 	int err;
664 
665 	if (link->ops != &bpf_iter_link_lops)
666 		return -EINVAL;
667 
668 	flags = O_RDONLY | O_CLOEXEC;
669 
670 	FD_PREPARE(fdf, flags, anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags));
671 	if (fdf->fd < 0)
672 		return fdf->fd;
673 
674 	iter_link = container_of(link, struct bpf_iter_link, link);
675 	err = prepare_seq_file(fdf->file, iter_link);
676 	if (err)
677 		return err; /* Automatic cleanup handles fput */
678 
679 	return fd_publish(fdf);
680 }
681 
682 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
683 {
684 	struct bpf_iter_priv_data *iter_priv;
685 	struct seq_file *seq;
686 	void *seq_priv;
687 
688 	seq = meta->seq;
689 	if (seq->file->f_op != &bpf_iter_fops)
690 		return NULL;
691 
692 	seq_priv = seq->private;
693 	iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
694 				 target_private);
695 
696 	if (in_stop && iter_priv->done_stop)
697 		return NULL;
698 
699 	meta->session_id = iter_priv->session_id;
700 	meta->seq_num = iter_priv->seq_num;
701 
702 	return iter_priv->prog;
703 }
704 
705 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
706 {
707 	struct bpf_run_ctx run_ctx, *old_run_ctx;
708 	int ret;
709 
710 	if (prog->sleepable) {
711 		rcu_read_lock_trace();
712 		migrate_disable();
713 		might_fault();
714 		old_run_ctx = bpf_set_run_ctx(&run_ctx);
715 		ret = bpf_prog_run(prog, ctx);
716 		bpf_reset_run_ctx(old_run_ctx);
717 		migrate_enable();
718 		rcu_read_unlock_trace();
719 	} else {
720 		rcu_read_lock_dont_migrate();
721 		old_run_ctx = bpf_set_run_ctx(&run_ctx);
722 		ret = bpf_prog_run(prog, ctx);
723 		bpf_reset_run_ctx(old_run_ctx);
724 		rcu_read_unlock_migrate();
725 	}
726 
727 	/* bpf program can only return 0 or 1:
728 	 *  0 : okay
729 	 *  1 : retry the same object
730 	 * The bpf_iter_run_prog() return value
731 	 * will be seq_ops->show() return value.
732 	 */
733 	return ret == 0 ? 0 : -EAGAIN;
734 }
735 
736 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
737 	   void *, callback_ctx, u64, flags)
738 {
739 	return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
740 }
741 
742 const struct bpf_func_proto bpf_for_each_map_elem_proto = {
743 	.func		= bpf_for_each_map_elem,
744 	.gpl_only	= false,
745 	.ret_type	= RET_INTEGER,
746 	.arg1_type	= ARG_CONST_MAP_PTR,
747 	.arg2_type	= ARG_PTR_TO_FUNC,
748 	.arg3_type	= ARG_PTR_TO_STACK_OR_NULL,
749 	.arg4_type	= ARG_ANYTHING,
750 };
751 
752 BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
753 	   u64, flags)
754 {
755 	bpf_callback_t callback = (bpf_callback_t)callback_fn;
756 	u64 ret;
757 	u32 i;
758 
759 	/* Note: these safety checks are also verified when bpf_loop
760 	 * is inlined, be careful to modify this code in sync. See
761 	 * function verifier.c:inline_bpf_loop.
762 	 */
763 	if (flags)
764 		return -EINVAL;
765 	if (nr_loops > BPF_MAX_LOOPS)
766 		return -E2BIG;
767 
768 	for (i = 0; i < nr_loops; i++) {
769 		ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0);
770 		/* return value: 0 - continue, 1 - stop and return */
771 		if (ret)
772 			return i + 1;
773 	}
774 
775 	return i;
776 }
777 
778 const struct bpf_func_proto bpf_loop_proto = {
779 	.func		= bpf_loop,
780 	.gpl_only	= false,
781 	.ret_type	= RET_INTEGER,
782 	.arg1_type	= ARG_SCALAR,
783 	.arg2_type	= ARG_PTR_TO_FUNC,
784 	.arg3_type	= ARG_PTR_TO_STACK_OR_NULL,
785 	.arg4_type	= ARG_ANYTHING,
786 };
787 
788 struct bpf_iter_num_kern {
789 	int cur; /* current value, inclusive */
790 	int end; /* final value, exclusive */
791 } __aligned(8);
792 
793 __bpf_kfunc_start_defs();
794 
795 __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end)
796 {
797 	struct bpf_iter_num_kern *s = (void *)it;
798 
799 	BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num));
800 	BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num));
801 
802 	/* start == end is legit, it's an empty range and we'll just get NULL
803 	 * on first (and any subsequent) bpf_iter_num_next() call
804 	 */
805 	if (start > end) {
806 		s->cur = s->end = 0;
807 		return -EINVAL;
808 	}
809 
810 	/* start <= end here, so end - start fits in a u32 without overflow */
811 	if ((u32)(end - start) > BPF_MAX_LOOPS) {
812 		s->cur = s->end = 0;
813 		return -E2BIG;
814 	}
815 
816 	/* user will call bpf_iter_num_next() first,
817 	 * which will set s->cur to exactly start value;
818 	 * underflow shouldn't matter
819 	 */
820 	s->cur = start - 1;
821 	s->end = end;
822 
823 	return 0;
824 }
825 
826 __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it)
827 {
828 	struct bpf_iter_num_kern *s = (void *)it;
829 
830 	/*
831 	 * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed
832 	 * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN).
833 	 */
834 	if (s->cur + 1 >= s->end) {
835 		s->cur = s->end = 0;
836 		return NULL;
837 	}
838 
839 	s->cur++;
840 
841 	return &s->cur;
842 }
843 
844 __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
845 {
846 	/* no-op */
847 }
848 
849 __bpf_kfunc_end_defs();
850