1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 Facebook */ 3 4 #include <linux/fs.h> 5 #include <linux/anon_inodes.h> 6 #include <linux/filter.h> 7 #include <linux/bpf.h> 8 #include <linux/rcupdate_trace.h> 9 10 struct bpf_iter_target_info { 11 struct list_head list; 12 const struct bpf_iter_reg *reg_info; 13 u32 btf_id; /* cached value */ 14 }; 15 16 struct bpf_iter_link { 17 struct bpf_link link; 18 struct bpf_iter_aux_info aux; 19 struct bpf_iter_target_info *tinfo; 20 }; 21 22 struct bpf_iter_priv_data { 23 struct bpf_iter_target_info *tinfo; 24 const struct bpf_iter_seq_info *seq_info; 25 struct bpf_prog *prog; 26 u64 session_id; 27 u64 seq_num; 28 bool done_stop; 29 u8 target_private[] __aligned(8); 30 }; 31 32 static struct list_head targets = LIST_HEAD_INIT(targets); 33 static DEFINE_MUTEX(targets_mutex); 34 35 /* protect bpf_iter_link changes */ 36 static DEFINE_MUTEX(link_mutex); 37 38 /* incremented on every opened seq_file */ 39 static atomic64_t session_id; 40 41 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link); 42 43 static void bpf_iter_inc_seq_num(struct seq_file *seq) 44 { 45 struct bpf_iter_priv_data *iter_priv; 46 47 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 48 target_private); 49 iter_priv->seq_num++; 50 } 51 52 static void bpf_iter_dec_seq_num(struct seq_file *seq) 53 { 54 struct bpf_iter_priv_data *iter_priv; 55 56 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 57 target_private); 58 iter_priv->seq_num--; 59 } 60 61 static void bpf_iter_done_stop(struct seq_file *seq) 62 { 63 struct bpf_iter_priv_data *iter_priv; 64 65 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 66 target_private); 67 iter_priv->done_stop = true; 68 } 69 70 static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo) 71 { 72 return tinfo->reg_info->feature & BPF_ITER_RESCHED; 73 } 74 75 /* Only allow sleepable program for resched-able iterator that can run one */ 76 static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, 77 const struct bpf_prog *prog) 78 { 79 return !prog->sleepable || 80 (bpf_iter_target_support_resched(tinfo) && 81 !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); 82 } 83 84 static bool bpf_iter_support_resched(struct seq_file *seq) 85 { 86 struct bpf_iter_priv_data *iter_priv; 87 88 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 89 target_private); 90 return bpf_iter_target_support_resched(iter_priv->tinfo); 91 } 92 93 /* maximum visited objects before bailing out */ 94 #define MAX_ITER_OBJECTS 1000000 95 96 /* bpf_seq_read, a customized and simpler version for bpf iterator. 97 * The following are differences from seq_read(): 98 * . fixed buffer size (PAGE_SIZE << 3) 99 * . assuming NULL ->llseek() 100 * . stop() may call bpf program, handling potential overflow there 101 */ 102 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, 103 loff_t *ppos) 104 { 105 struct seq_file *seq = file->private_data; 106 size_t n, offs, copied = 0; 107 int err = 0, num_objs = 0; 108 bool can_resched; 109 void *p; 110 111 mutex_lock(&seq->lock); 112 113 if (!seq->buf) { 114 seq->size = PAGE_SIZE << 3; 115 seq->buf = kvmalloc(seq->size, GFP_KERNEL); 116 if (!seq->buf) { 117 err = -ENOMEM; 118 goto done; 119 } 120 } 121 122 if (seq->count) { 123 n = min(seq->count, size); 124 err = copy_to_user(buf, seq->buf + seq->from, n); 125 if (err) { 126 err = -EFAULT; 127 goto done; 128 } 129 seq->count -= n; 130 seq->from += n; 131 copied = n; 132 goto done; 133 } 134 135 seq->from = 0; 136 p = seq->op->start(seq, &seq->index); 137 if (!p) 138 goto stop; 139 if (IS_ERR(p)) { 140 err = PTR_ERR(p); 141 seq->op->stop(seq, p); 142 seq->count = 0; 143 goto done; 144 } 145 146 err = seq->op->show(seq, p); 147 if (err > 0) { 148 /* object is skipped, decrease seq_num, so next 149 * valid object can reuse the same seq_num. 150 */ 151 bpf_iter_dec_seq_num(seq); 152 seq->count = 0; 153 } else if (err < 0 || seq_has_overflowed(seq)) { 154 if (!err) 155 err = -E2BIG; 156 seq->op->stop(seq, p); 157 seq->count = 0; 158 goto done; 159 } 160 161 can_resched = bpf_iter_support_resched(seq); 162 while (1) { 163 loff_t pos = seq->index; 164 165 num_objs++; 166 offs = seq->count; 167 p = seq->op->next(seq, p, &seq->index); 168 if (pos == seq->index) { 169 pr_info_ratelimited("buggy seq_file .next function %ps " 170 "did not updated position index\n", 171 seq->op->next); 172 seq->index++; 173 } 174 175 if (IS_ERR_OR_NULL(p)) 176 break; 177 178 /* got a valid next object, increase seq_num */ 179 bpf_iter_inc_seq_num(seq); 180 181 if (seq->count >= size) 182 break; 183 184 if (num_objs >= MAX_ITER_OBJECTS) { 185 if (offs == 0) { 186 err = -EAGAIN; 187 seq->op->stop(seq, p); 188 goto done; 189 } 190 break; 191 } 192 193 err = seq->op->show(seq, p); 194 if (err > 0) { 195 bpf_iter_dec_seq_num(seq); 196 seq->count = offs; 197 } else if (err < 0 || seq_has_overflowed(seq)) { 198 seq->count = offs; 199 if (offs == 0) { 200 if (!err) 201 err = -E2BIG; 202 seq->op->stop(seq, p); 203 goto done; 204 } 205 break; 206 } 207 208 if (can_resched) 209 cond_resched(); 210 } 211 stop: 212 offs = seq->count; 213 if (IS_ERR(p)) { 214 seq->op->stop(seq, NULL); 215 err = PTR_ERR(p); 216 goto done; 217 } 218 /* bpf program called if !p */ 219 seq->op->stop(seq, p); 220 if (!p) { 221 if (!seq_has_overflowed(seq)) { 222 bpf_iter_done_stop(seq); 223 } else { 224 seq->count = offs; 225 if (offs == 0) { 226 err = -E2BIG; 227 goto done; 228 } 229 } 230 } 231 232 n = min(seq->count, size); 233 err = copy_to_user(buf, seq->buf, n); 234 if (err) { 235 err = -EFAULT; 236 goto done; 237 } 238 copied = n; 239 seq->count -= n; 240 seq->from = n; 241 done: 242 if (!copied) 243 copied = err; 244 else 245 *ppos += copied; 246 mutex_unlock(&seq->lock); 247 return copied; 248 } 249 250 static const struct bpf_iter_seq_info * 251 __get_seq_info(struct bpf_iter_link *link) 252 { 253 const struct bpf_iter_seq_info *seq_info; 254 255 if (link->aux.map) { 256 seq_info = link->aux.map->ops->iter_seq_info; 257 if (seq_info) 258 return seq_info; 259 } 260 261 return link->tinfo->reg_info->seq_info; 262 } 263 264 static int iter_open(struct inode *inode, struct file *file) 265 { 266 struct bpf_iter_link *link = inode->i_private; 267 268 return prepare_seq_file(file, link); 269 } 270 271 static int iter_release(struct inode *inode, struct file *file) 272 { 273 struct bpf_iter_priv_data *iter_priv; 274 struct seq_file *seq; 275 276 seq = file->private_data; 277 if (!seq) 278 return 0; 279 280 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 281 target_private); 282 283 if (iter_priv->seq_info->fini_seq_private) 284 iter_priv->seq_info->fini_seq_private(seq->private); 285 286 bpf_prog_put(iter_priv->prog); 287 seq->private = iter_priv; 288 289 return seq_release_private(inode, file); 290 } 291 292 const struct file_operations bpf_iter_fops = { 293 .open = iter_open, 294 .read = bpf_seq_read, 295 .release = iter_release, 296 }; 297 298 /* The argument reg_info will be cached in bpf_iter_target_info. 299 * The common practice is to declare target reg_info as 300 * a const static variable and passed as an argument to 301 * bpf_iter_reg_target(). 302 */ 303 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) 304 { 305 struct bpf_iter_target_info *tinfo; 306 307 tinfo = kzalloc_obj(*tinfo); 308 if (!tinfo) 309 return -ENOMEM; 310 311 tinfo->reg_info = reg_info; 312 INIT_LIST_HEAD(&tinfo->list); 313 314 mutex_lock(&targets_mutex); 315 list_add(&tinfo->list, &targets); 316 mutex_unlock(&targets_mutex); 317 318 return 0; 319 } 320 321 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info) 322 { 323 struct bpf_iter_target_info *tinfo; 324 bool found = false; 325 326 mutex_lock(&targets_mutex); 327 list_for_each_entry(tinfo, &targets, list) { 328 if (reg_info == tinfo->reg_info) { 329 list_del(&tinfo->list); 330 kfree(tinfo); 331 found = true; 332 break; 333 } 334 } 335 mutex_unlock(&targets_mutex); 336 337 WARN_ON(found == false); 338 } 339 340 static void cache_btf_id(struct bpf_iter_target_info *tinfo, 341 struct bpf_prog *prog) 342 { 343 tinfo->btf_id = prog->aux->attach_btf_id; 344 } 345 346 int bpf_iter_prog_supported(struct bpf_prog *prog) 347 { 348 const char *attach_fname = prog->aux->attach_func_name; 349 struct bpf_iter_target_info *tinfo = NULL, *iter; 350 u32 prog_btf_id = prog->aux->attach_btf_id; 351 const char *prefix = BPF_ITER_FUNC_PREFIX; 352 int prefix_len = strlen(prefix); 353 354 /* 355 * The attach function of a program that has a target program is a 356 * function of that program, whatever its name is. 357 */ 358 if (prog->aux->dst_prog) 359 return -EINVAL; 360 361 if (strncmp(attach_fname, prefix, prefix_len)) 362 return -EINVAL; 363 364 mutex_lock(&targets_mutex); 365 list_for_each_entry(iter, &targets, list) { 366 if (iter->btf_id && iter->btf_id == prog_btf_id) { 367 tinfo = iter; 368 break; 369 } 370 if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) { 371 cache_btf_id(iter, prog); 372 tinfo = iter; 373 break; 374 } 375 } 376 mutex_unlock(&targets_mutex); 377 378 if (!tinfo) 379 return -EINVAL; 380 381 return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info, 382 tinfo->reg_info->ctx_arg_info_size); 383 } 384 385 const struct bpf_func_proto * 386 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 387 { 388 const struct bpf_iter_target_info *tinfo; 389 const struct bpf_func_proto *fn = NULL; 390 391 mutex_lock(&targets_mutex); 392 list_for_each_entry(tinfo, &targets, list) { 393 if (tinfo->btf_id == prog->aux->attach_btf_id) { 394 const struct bpf_iter_reg *reg_info; 395 396 reg_info = tinfo->reg_info; 397 if (reg_info->get_func_proto) 398 fn = reg_info->get_func_proto(func_id, prog); 399 break; 400 } 401 } 402 mutex_unlock(&targets_mutex); 403 404 return fn; 405 } 406 407 static void bpf_iter_link_release(struct bpf_link *link) 408 { 409 struct bpf_iter_link *iter_link = 410 container_of(link, struct bpf_iter_link, link); 411 412 if (iter_link->tinfo->reg_info->detach_target) 413 iter_link->tinfo->reg_info->detach_target(&iter_link->aux); 414 } 415 416 static void bpf_iter_link_dealloc(struct bpf_link *link) 417 { 418 struct bpf_iter_link *iter_link = 419 container_of(link, struct bpf_iter_link, link); 420 421 kfree(iter_link); 422 } 423 424 static int bpf_iter_link_replace(struct bpf_link *link, 425 struct bpf_prog *new_prog, 426 struct bpf_prog *old_prog) 427 { 428 struct bpf_iter_link *iter_link = 429 container_of(link, struct bpf_iter_link, link); 430 int ret = 0; 431 432 mutex_lock(&link_mutex); 433 if (old_prog && link->prog != old_prog) { 434 ret = -EPERM; 435 goto out_unlock; 436 } 437 438 if (link->prog->type != new_prog->type || 439 link->prog->expected_attach_type != new_prog->expected_attach_type || 440 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || 441 !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { 442 ret = -EINVAL; 443 goto out_unlock; 444 } 445 446 if (iter_link->tinfo->reg_info->check_prog) { 447 ret = iter_link->tinfo->reg_info->check_prog(new_prog, 448 &iter_link->aux); 449 if (ret) 450 goto out_unlock; 451 } 452 453 old_prog = xchg(&link->prog, new_prog); 454 bpf_prog_put(old_prog); 455 456 out_unlock: 457 mutex_unlock(&link_mutex); 458 return ret; 459 } 460 461 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link, 462 struct seq_file *seq) 463 { 464 struct bpf_iter_link *iter_link = 465 container_of(link, struct bpf_iter_link, link); 466 bpf_iter_show_fdinfo_t show_fdinfo; 467 468 seq_printf(seq, 469 "target_name:\t%s\n", 470 iter_link->tinfo->reg_info->target); 471 472 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo; 473 if (show_fdinfo) 474 show_fdinfo(&iter_link->aux, seq); 475 } 476 477 static int bpf_iter_link_fill_link_info(const struct bpf_link *link, 478 struct bpf_link_info *info) 479 { 480 struct bpf_iter_link *iter_link = 481 container_of(link, struct bpf_iter_link, link); 482 char __user *ubuf = u64_to_user_ptr(info->iter.target_name); 483 bpf_iter_fill_link_info_t fill_link_info; 484 u32 ulen = info->iter.target_name_len; 485 const char *target_name; 486 u32 target_len; 487 488 if (!ulen ^ !ubuf) 489 return -EINVAL; 490 491 target_name = iter_link->tinfo->reg_info->target; 492 target_len = strlen(target_name); 493 info->iter.target_name_len = target_len + 1; 494 495 if (ubuf) { 496 if (ulen >= target_len + 1) { 497 if (copy_to_user(ubuf, target_name, target_len + 1)) 498 return -EFAULT; 499 } else { 500 char zero = '\0'; 501 502 if (copy_to_user(ubuf, target_name, ulen - 1)) 503 return -EFAULT; 504 if (put_user(zero, ubuf + ulen - 1)) 505 return -EFAULT; 506 return -ENOSPC; 507 } 508 } 509 510 fill_link_info = iter_link->tinfo->reg_info->fill_link_info; 511 if (fill_link_info) 512 return fill_link_info(&iter_link->aux, info); 513 514 return 0; 515 } 516 517 static const struct bpf_link_ops bpf_iter_link_lops = { 518 .release = bpf_iter_link_release, 519 .dealloc = bpf_iter_link_dealloc, 520 .update_prog = bpf_iter_link_replace, 521 .show_fdinfo = bpf_iter_link_show_fdinfo, 522 .fill_link_info = bpf_iter_link_fill_link_info, 523 }; 524 525 bool bpf_link_is_iter(struct bpf_link *link) 526 { 527 return link->ops == &bpf_iter_link_lops; 528 } 529 530 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, 531 struct bpf_prog *prog) 532 { 533 struct bpf_iter_target_info *tinfo = NULL, *iter; 534 struct bpf_link_primer link_primer; 535 union bpf_iter_link_info linfo; 536 struct bpf_iter_link *link; 537 u32 prog_btf_id, linfo_len; 538 bpfptr_t ulinfo; 539 int err; 540 541 if (attr->link_create.target_fd || attr->link_create.flags) 542 return -EINVAL; 543 544 memset(&linfo, 0, sizeof(union bpf_iter_link_info)); 545 546 ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel); 547 linfo_len = attr->link_create.iter_info_len; 548 if (bpfptr_is_null(ulinfo) ^ !linfo_len) 549 return -EINVAL; 550 551 if (!bpfptr_is_null(ulinfo)) { 552 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo), 553 linfo_len); 554 if (err) 555 return err; 556 linfo_len = min_t(u32, linfo_len, sizeof(linfo)); 557 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len)) 558 return -EFAULT; 559 } 560 561 prog_btf_id = prog->aux->attach_btf_id; 562 mutex_lock(&targets_mutex); 563 list_for_each_entry(iter, &targets, list) { 564 if (iter->btf_id == prog_btf_id) { 565 tinfo = iter; 566 break; 567 } 568 } 569 mutex_unlock(&targets_mutex); 570 if (!tinfo) 571 return -ENOENT; 572 573 if (!bpf_iter_target_support_prog(tinfo, prog)) 574 return -EINVAL; 575 576 link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN); 577 if (!link) 578 return -ENOMEM; 579 580 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog, 581 attr->link_create.attach_type); 582 link->tinfo = tinfo; 583 584 err = bpf_link_prime(&link->link, &link_primer); 585 if (err) { 586 kfree(link); 587 return err; 588 } 589 590 if (tinfo->reg_info->attach_target) { 591 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux); 592 if (err) { 593 bpf_link_cleanup(&link_primer); 594 return err; 595 } 596 } 597 598 return bpf_link_settle(&link_primer); 599 } 600 601 static void init_seq_meta(struct bpf_iter_priv_data *priv_data, 602 struct bpf_iter_target_info *tinfo, 603 const struct bpf_iter_seq_info *seq_info, 604 struct bpf_prog *prog) 605 { 606 priv_data->tinfo = tinfo; 607 priv_data->seq_info = seq_info; 608 priv_data->prog = prog; 609 priv_data->session_id = atomic64_inc_return(&session_id); 610 priv_data->seq_num = 0; 611 priv_data->done_stop = false; 612 } 613 614 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link) 615 { 616 const struct bpf_iter_seq_info *seq_info = __get_seq_info(link); 617 struct bpf_iter_priv_data *priv_data; 618 struct bpf_iter_target_info *tinfo; 619 struct bpf_prog *prog; 620 u32 total_priv_dsize; 621 struct seq_file *seq; 622 int err = 0; 623 624 mutex_lock(&link_mutex); 625 prog = link->link.prog; 626 bpf_prog_inc(prog); 627 mutex_unlock(&link_mutex); 628 629 tinfo = link->tinfo; 630 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) + 631 seq_info->seq_priv_size; 632 priv_data = __seq_open_private(file, seq_info->seq_ops, 633 total_priv_dsize); 634 if (!priv_data) { 635 err = -ENOMEM; 636 goto release_prog; 637 } 638 639 if (seq_info->init_seq_private) { 640 err = seq_info->init_seq_private(priv_data->target_private, &link->aux); 641 if (err) 642 goto release_seq_file; 643 } 644 645 init_seq_meta(priv_data, tinfo, seq_info, prog); 646 seq = file->private_data; 647 seq->private = priv_data->target_private; 648 649 return 0; 650 651 release_seq_file: 652 seq_release_private(file->f_inode, file); 653 file->private_data = NULL; 654 release_prog: 655 bpf_prog_put(prog); 656 return err; 657 } 658 659 int bpf_iter_new_fd(struct bpf_link *link) 660 { 661 struct bpf_iter_link *iter_link; 662 unsigned int flags; 663 int err; 664 665 if (link->ops != &bpf_iter_link_lops) 666 return -EINVAL; 667 668 flags = O_RDONLY | O_CLOEXEC; 669 670 FD_PREPARE(fdf, flags, anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags)); 671 if (fdf->fd < 0) 672 return fdf->fd; 673 674 iter_link = container_of(link, struct bpf_iter_link, link); 675 err = prepare_seq_file(fdf->file, iter_link); 676 if (err) 677 return err; /* Automatic cleanup handles fput */ 678 679 return fd_publish(fdf); 680 } 681 682 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) 683 { 684 struct bpf_iter_priv_data *iter_priv; 685 struct seq_file *seq; 686 void *seq_priv; 687 688 seq = meta->seq; 689 if (seq->file->f_op != &bpf_iter_fops) 690 return NULL; 691 692 seq_priv = seq->private; 693 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data, 694 target_private); 695 696 if (in_stop && iter_priv->done_stop) 697 return NULL; 698 699 meta->session_id = iter_priv->session_id; 700 meta->seq_num = iter_priv->seq_num; 701 702 return iter_priv->prog; 703 } 704 705 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) 706 { 707 struct bpf_run_ctx run_ctx, *old_run_ctx; 708 int ret; 709 710 if (prog->sleepable) { 711 rcu_read_lock_trace(); 712 migrate_disable(); 713 might_fault(); 714 old_run_ctx = bpf_set_run_ctx(&run_ctx); 715 ret = bpf_prog_run(prog, ctx); 716 bpf_reset_run_ctx(old_run_ctx); 717 migrate_enable(); 718 rcu_read_unlock_trace(); 719 } else { 720 rcu_read_lock_dont_migrate(); 721 old_run_ctx = bpf_set_run_ctx(&run_ctx); 722 ret = bpf_prog_run(prog, ctx); 723 bpf_reset_run_ctx(old_run_ctx); 724 rcu_read_unlock_migrate(); 725 } 726 727 /* bpf program can only return 0 or 1: 728 * 0 : okay 729 * 1 : retry the same object 730 * The bpf_iter_run_prog() return value 731 * will be seq_ops->show() return value. 732 */ 733 return ret == 0 ? 0 : -EAGAIN; 734 } 735 736 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn, 737 void *, callback_ctx, u64, flags) 738 { 739 return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags); 740 } 741 742 const struct bpf_func_proto bpf_for_each_map_elem_proto = { 743 .func = bpf_for_each_map_elem, 744 .gpl_only = false, 745 .ret_type = RET_INTEGER, 746 .arg1_type = ARG_CONST_MAP_PTR, 747 .arg2_type = ARG_PTR_TO_FUNC, 748 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 749 .arg4_type = ARG_ANYTHING, 750 }; 751 752 BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, 753 u64, flags) 754 { 755 bpf_callback_t callback = (bpf_callback_t)callback_fn; 756 u64 ret; 757 u32 i; 758 759 /* Note: these safety checks are also verified when bpf_loop 760 * is inlined, be careful to modify this code in sync. See 761 * function verifier.c:inline_bpf_loop. 762 */ 763 if (flags) 764 return -EINVAL; 765 if (nr_loops > BPF_MAX_LOOPS) 766 return -E2BIG; 767 768 for (i = 0; i < nr_loops; i++) { 769 ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); 770 /* return value: 0 - continue, 1 - stop and return */ 771 if (ret) 772 return i + 1; 773 } 774 775 return i; 776 } 777 778 const struct bpf_func_proto bpf_loop_proto = { 779 .func = bpf_loop, 780 .gpl_only = false, 781 .ret_type = RET_INTEGER, 782 .arg1_type = ARG_SCALAR, 783 .arg2_type = ARG_PTR_TO_FUNC, 784 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 785 .arg4_type = ARG_ANYTHING, 786 }; 787 788 struct bpf_iter_num_kern { 789 int cur; /* current value, inclusive */ 790 int end; /* final value, exclusive */ 791 } __aligned(8); 792 793 __bpf_kfunc_start_defs(); 794 795 __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) 796 { 797 struct bpf_iter_num_kern *s = (void *)it; 798 799 BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num)); 800 BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num)); 801 802 /* start == end is legit, it's an empty range and we'll just get NULL 803 * on first (and any subsequent) bpf_iter_num_next() call 804 */ 805 if (start > end) { 806 s->cur = s->end = 0; 807 return -EINVAL; 808 } 809 810 /* start <= end here, so end - start fits in a u32 without overflow */ 811 if ((u32)(end - start) > BPF_MAX_LOOPS) { 812 s->cur = s->end = 0; 813 return -E2BIG; 814 } 815 816 /* user will call bpf_iter_num_next() first, 817 * which will set s->cur to exactly start value; 818 * underflow shouldn't matter 819 */ 820 s->cur = start - 1; 821 s->end = end; 822 823 return 0; 824 } 825 826 __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) 827 { 828 struct bpf_iter_num_kern *s = (void *)it; 829 830 /* 831 * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed 832 * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN). 833 */ 834 if (s->cur + 1 >= s->end) { 835 s->cur = s->end = 0; 836 return NULL; 837 } 838 839 s->cur++; 840 841 return &s->cur; 842 } 843 844 __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) 845 { 846 /* no-op */ 847 } 848 849 __bpf_kfunc_end_defs(); 850