1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 Facebook */ 3 4 #include <linux/fs.h> 5 #include <linux/anon_inodes.h> 6 #include <linux/filter.h> 7 #include <linux/bpf.h> 8 #include <linux/rcupdate_trace.h> 9 10 struct bpf_iter_target_info { 11 struct list_head list; 12 const struct bpf_iter_reg *reg_info; 13 u32 btf_id; /* cached value */ 14 }; 15 16 struct bpf_iter_link { 17 struct bpf_link link; 18 struct bpf_iter_aux_info aux; 19 struct bpf_iter_target_info *tinfo; 20 }; 21 22 struct bpf_iter_priv_data { 23 struct bpf_iter_target_info *tinfo; 24 const struct bpf_iter_seq_info *seq_info; 25 struct bpf_prog *prog; 26 u64 session_id; 27 u64 seq_num; 28 bool done_stop; 29 u8 target_private[] __aligned(8); 30 }; 31 32 static struct list_head targets = LIST_HEAD_INIT(targets); 33 static DEFINE_MUTEX(targets_mutex); 34 35 /* protect bpf_iter_link changes */ 36 static DEFINE_MUTEX(link_mutex); 37 38 /* incremented on every opened seq_file */ 39 static atomic64_t session_id; 40 41 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 42 const struct bpf_iter_seq_info *seq_info); 43 44 static void bpf_iter_inc_seq_num(struct seq_file *seq) 45 { 46 struct bpf_iter_priv_data *iter_priv; 47 48 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 49 target_private); 50 iter_priv->seq_num++; 51 } 52 53 static void bpf_iter_dec_seq_num(struct seq_file *seq) 54 { 55 struct bpf_iter_priv_data *iter_priv; 56 57 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 58 target_private); 59 iter_priv->seq_num--; 60 } 61 62 static void bpf_iter_done_stop(struct seq_file *seq) 63 { 64 struct bpf_iter_priv_data *iter_priv; 65 66 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 67 target_private); 68 iter_priv->done_stop = true; 69 } 70 71 static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo) 72 { 73 return tinfo->reg_info->feature & BPF_ITER_RESCHED; 74 } 75 76 static bool bpf_iter_support_resched(struct seq_file *seq) 77 { 78 struct bpf_iter_priv_data *iter_priv; 79 80 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 81 target_private); 82 return bpf_iter_target_support_resched(iter_priv->tinfo); 83 } 84 85 /* maximum visited objects before bailing out */ 86 #define MAX_ITER_OBJECTS 1000000 87 88 /* bpf_seq_read, a customized and simpler version for bpf iterator. 89 * The following are differences from seq_read(): 90 * . fixed buffer size (PAGE_SIZE) 91 * . assuming NULL ->llseek() 92 * . stop() may call bpf program, handling potential overflow there 93 */ 94 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, 95 loff_t *ppos) 96 { 97 struct seq_file *seq = file->private_data; 98 size_t n, offs, copied = 0; 99 int err = 0, num_objs = 0; 100 bool can_resched; 101 void *p; 102 103 mutex_lock(&seq->lock); 104 105 if (!seq->buf) { 106 seq->size = PAGE_SIZE << 3; 107 seq->buf = kvmalloc(seq->size, GFP_KERNEL); 108 if (!seq->buf) { 109 err = -ENOMEM; 110 goto done; 111 } 112 } 113 114 if (seq->count) { 115 n = min(seq->count, size); 116 err = copy_to_user(buf, seq->buf + seq->from, n); 117 if (err) { 118 err = -EFAULT; 119 goto done; 120 } 121 seq->count -= n; 122 seq->from += n; 123 copied = n; 124 goto done; 125 } 126 127 seq->from = 0; 128 p = seq->op->start(seq, &seq->index); 129 if (!p) 130 goto stop; 131 if (IS_ERR(p)) { 132 err = PTR_ERR(p); 133 seq->op->stop(seq, p); 134 seq->count = 0; 135 goto done; 136 } 137 138 err = seq->op->show(seq, p); 139 if (err > 0) { 140 /* object is skipped, decrease seq_num, so next 141 * valid object can reuse the same seq_num. 142 */ 143 bpf_iter_dec_seq_num(seq); 144 seq->count = 0; 145 } else if (err < 0 || seq_has_overflowed(seq)) { 146 if (!err) 147 err = -E2BIG; 148 seq->op->stop(seq, p); 149 seq->count = 0; 150 goto done; 151 } 152 153 can_resched = bpf_iter_support_resched(seq); 154 while (1) { 155 loff_t pos = seq->index; 156 157 num_objs++; 158 offs = seq->count; 159 p = seq->op->next(seq, p, &seq->index); 160 if (pos == seq->index) { 161 pr_info_ratelimited("buggy seq_file .next function %ps " 162 "did not updated position index\n", 163 seq->op->next); 164 seq->index++; 165 } 166 167 if (IS_ERR_OR_NULL(p)) 168 break; 169 170 /* got a valid next object, increase seq_num */ 171 bpf_iter_inc_seq_num(seq); 172 173 if (seq->count >= size) 174 break; 175 176 if (num_objs >= MAX_ITER_OBJECTS) { 177 if (offs == 0) { 178 err = -EAGAIN; 179 seq->op->stop(seq, p); 180 goto done; 181 } 182 break; 183 } 184 185 err = seq->op->show(seq, p); 186 if (err > 0) { 187 bpf_iter_dec_seq_num(seq); 188 seq->count = offs; 189 } else if (err < 0 || seq_has_overflowed(seq)) { 190 seq->count = offs; 191 if (offs == 0) { 192 if (!err) 193 err = -E2BIG; 194 seq->op->stop(seq, p); 195 goto done; 196 } 197 break; 198 } 199 200 if (can_resched) 201 cond_resched(); 202 } 203 stop: 204 offs = seq->count; 205 if (IS_ERR(p)) { 206 seq->op->stop(seq, NULL); 207 err = PTR_ERR(p); 208 goto done; 209 } 210 /* bpf program called if !p */ 211 seq->op->stop(seq, p); 212 if (!p) { 213 if (!seq_has_overflowed(seq)) { 214 bpf_iter_done_stop(seq); 215 } else { 216 seq->count = offs; 217 if (offs == 0) { 218 err = -E2BIG; 219 goto done; 220 } 221 } 222 } 223 224 n = min(seq->count, size); 225 err = copy_to_user(buf, seq->buf, n); 226 if (err) { 227 err = -EFAULT; 228 goto done; 229 } 230 copied = n; 231 seq->count -= n; 232 seq->from = n; 233 done: 234 if (!copied) 235 copied = err; 236 else 237 *ppos += copied; 238 mutex_unlock(&seq->lock); 239 return copied; 240 } 241 242 static const struct bpf_iter_seq_info * 243 __get_seq_info(struct bpf_iter_link *link) 244 { 245 const struct bpf_iter_seq_info *seq_info; 246 247 if (link->aux.map) { 248 seq_info = link->aux.map->ops->iter_seq_info; 249 if (seq_info) 250 return seq_info; 251 } 252 253 return link->tinfo->reg_info->seq_info; 254 } 255 256 static int iter_open(struct inode *inode, struct file *file) 257 { 258 struct bpf_iter_link *link = inode->i_private; 259 260 return prepare_seq_file(file, link, __get_seq_info(link)); 261 } 262 263 static int iter_release(struct inode *inode, struct file *file) 264 { 265 struct bpf_iter_priv_data *iter_priv; 266 struct seq_file *seq; 267 268 seq = file->private_data; 269 if (!seq) 270 return 0; 271 272 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 273 target_private); 274 275 if (iter_priv->seq_info->fini_seq_private) 276 iter_priv->seq_info->fini_seq_private(seq->private); 277 278 bpf_prog_put(iter_priv->prog); 279 seq->private = iter_priv; 280 281 return seq_release_private(inode, file); 282 } 283 284 const struct file_operations bpf_iter_fops = { 285 .open = iter_open, 286 .read = bpf_seq_read, 287 .release = iter_release, 288 }; 289 290 /* The argument reg_info will be cached in bpf_iter_target_info. 291 * The common practice is to declare target reg_info as 292 * a const static variable and passed as an argument to 293 * bpf_iter_reg_target(). 294 */ 295 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) 296 { 297 struct bpf_iter_target_info *tinfo; 298 299 tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL); 300 if (!tinfo) 301 return -ENOMEM; 302 303 tinfo->reg_info = reg_info; 304 INIT_LIST_HEAD(&tinfo->list); 305 306 mutex_lock(&targets_mutex); 307 list_add(&tinfo->list, &targets); 308 mutex_unlock(&targets_mutex); 309 310 return 0; 311 } 312 313 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info) 314 { 315 struct bpf_iter_target_info *tinfo; 316 bool found = false; 317 318 mutex_lock(&targets_mutex); 319 list_for_each_entry(tinfo, &targets, list) { 320 if (reg_info == tinfo->reg_info) { 321 list_del(&tinfo->list); 322 kfree(tinfo); 323 found = true; 324 break; 325 } 326 } 327 mutex_unlock(&targets_mutex); 328 329 WARN_ON(found == false); 330 } 331 332 static void cache_btf_id(struct bpf_iter_target_info *tinfo, 333 struct bpf_prog *prog) 334 { 335 tinfo->btf_id = prog->aux->attach_btf_id; 336 } 337 338 bool bpf_iter_prog_supported(struct bpf_prog *prog) 339 { 340 const char *attach_fname = prog->aux->attach_func_name; 341 struct bpf_iter_target_info *tinfo = NULL, *iter; 342 u32 prog_btf_id = prog->aux->attach_btf_id; 343 const char *prefix = BPF_ITER_FUNC_PREFIX; 344 int prefix_len = strlen(prefix); 345 346 if (strncmp(attach_fname, prefix, prefix_len)) 347 return false; 348 349 mutex_lock(&targets_mutex); 350 list_for_each_entry(iter, &targets, list) { 351 if (iter->btf_id && iter->btf_id == prog_btf_id) { 352 tinfo = iter; 353 break; 354 } 355 if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) { 356 cache_btf_id(iter, prog); 357 tinfo = iter; 358 break; 359 } 360 } 361 mutex_unlock(&targets_mutex); 362 363 if (tinfo) { 364 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size; 365 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info; 366 } 367 368 return tinfo != NULL; 369 } 370 371 const struct bpf_func_proto * 372 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 373 { 374 const struct bpf_iter_target_info *tinfo; 375 const struct bpf_func_proto *fn = NULL; 376 377 mutex_lock(&targets_mutex); 378 list_for_each_entry(tinfo, &targets, list) { 379 if (tinfo->btf_id == prog->aux->attach_btf_id) { 380 const struct bpf_iter_reg *reg_info; 381 382 reg_info = tinfo->reg_info; 383 if (reg_info->get_func_proto) 384 fn = reg_info->get_func_proto(func_id, prog); 385 break; 386 } 387 } 388 mutex_unlock(&targets_mutex); 389 390 return fn; 391 } 392 393 static void bpf_iter_link_release(struct bpf_link *link) 394 { 395 struct bpf_iter_link *iter_link = 396 container_of(link, struct bpf_iter_link, link); 397 398 if (iter_link->tinfo->reg_info->detach_target) 399 iter_link->tinfo->reg_info->detach_target(&iter_link->aux); 400 } 401 402 static void bpf_iter_link_dealloc(struct bpf_link *link) 403 { 404 struct bpf_iter_link *iter_link = 405 container_of(link, struct bpf_iter_link, link); 406 407 kfree(iter_link); 408 } 409 410 static int bpf_iter_link_replace(struct bpf_link *link, 411 struct bpf_prog *new_prog, 412 struct bpf_prog *old_prog) 413 { 414 int ret = 0; 415 416 mutex_lock(&link_mutex); 417 if (old_prog && link->prog != old_prog) { 418 ret = -EPERM; 419 goto out_unlock; 420 } 421 422 if (link->prog->type != new_prog->type || 423 link->prog->expected_attach_type != new_prog->expected_attach_type || 424 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { 425 ret = -EINVAL; 426 goto out_unlock; 427 } 428 429 old_prog = xchg(&link->prog, new_prog); 430 bpf_prog_put(old_prog); 431 432 out_unlock: 433 mutex_unlock(&link_mutex); 434 return ret; 435 } 436 437 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link, 438 struct seq_file *seq) 439 { 440 struct bpf_iter_link *iter_link = 441 container_of(link, struct bpf_iter_link, link); 442 bpf_iter_show_fdinfo_t show_fdinfo; 443 444 seq_printf(seq, 445 "target_name:\t%s\n", 446 iter_link->tinfo->reg_info->target); 447 448 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo; 449 if (show_fdinfo) 450 show_fdinfo(&iter_link->aux, seq); 451 } 452 453 static int bpf_iter_link_fill_link_info(const struct bpf_link *link, 454 struct bpf_link_info *info) 455 { 456 struct bpf_iter_link *iter_link = 457 container_of(link, struct bpf_iter_link, link); 458 char __user *ubuf = u64_to_user_ptr(info->iter.target_name); 459 bpf_iter_fill_link_info_t fill_link_info; 460 u32 ulen = info->iter.target_name_len; 461 const char *target_name; 462 u32 target_len; 463 464 if (!ulen ^ !ubuf) 465 return -EINVAL; 466 467 target_name = iter_link->tinfo->reg_info->target; 468 target_len = strlen(target_name); 469 info->iter.target_name_len = target_len + 1; 470 471 if (ubuf) { 472 if (ulen >= target_len + 1) { 473 if (copy_to_user(ubuf, target_name, target_len + 1)) 474 return -EFAULT; 475 } else { 476 char zero = '\0'; 477 478 if (copy_to_user(ubuf, target_name, ulen - 1)) 479 return -EFAULT; 480 if (put_user(zero, ubuf + ulen - 1)) 481 return -EFAULT; 482 return -ENOSPC; 483 } 484 } 485 486 fill_link_info = iter_link->tinfo->reg_info->fill_link_info; 487 if (fill_link_info) 488 return fill_link_info(&iter_link->aux, info); 489 490 return 0; 491 } 492 493 static const struct bpf_link_ops bpf_iter_link_lops = { 494 .release = bpf_iter_link_release, 495 .dealloc = bpf_iter_link_dealloc, 496 .update_prog = bpf_iter_link_replace, 497 .show_fdinfo = bpf_iter_link_show_fdinfo, 498 .fill_link_info = bpf_iter_link_fill_link_info, 499 }; 500 501 bool bpf_link_is_iter(struct bpf_link *link) 502 { 503 return link->ops == &bpf_iter_link_lops; 504 } 505 506 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, 507 struct bpf_prog *prog) 508 { 509 struct bpf_iter_target_info *tinfo = NULL, *iter; 510 struct bpf_link_primer link_primer; 511 union bpf_iter_link_info linfo; 512 struct bpf_iter_link *link; 513 u32 prog_btf_id, linfo_len; 514 bpfptr_t ulinfo; 515 int err; 516 517 if (attr->link_create.target_fd || attr->link_create.flags) 518 return -EINVAL; 519 520 memset(&linfo, 0, sizeof(union bpf_iter_link_info)); 521 522 ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel); 523 linfo_len = attr->link_create.iter_info_len; 524 if (bpfptr_is_null(ulinfo) ^ !linfo_len) 525 return -EINVAL; 526 527 if (!bpfptr_is_null(ulinfo)) { 528 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo), 529 linfo_len); 530 if (err) 531 return err; 532 linfo_len = min_t(u32, linfo_len, sizeof(linfo)); 533 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len)) 534 return -EFAULT; 535 } 536 537 prog_btf_id = prog->aux->attach_btf_id; 538 mutex_lock(&targets_mutex); 539 list_for_each_entry(iter, &targets, list) { 540 if (iter->btf_id == prog_btf_id) { 541 tinfo = iter; 542 break; 543 } 544 } 545 mutex_unlock(&targets_mutex); 546 if (!tinfo) 547 return -ENOENT; 548 549 /* Only allow sleepable program for resched-able iterator */ 550 if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) 551 return -EINVAL; 552 553 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN); 554 if (!link) 555 return -ENOMEM; 556 557 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog); 558 link->tinfo = tinfo; 559 560 err = bpf_link_prime(&link->link, &link_primer); 561 if (err) { 562 kfree(link); 563 return err; 564 } 565 566 if (tinfo->reg_info->attach_target) { 567 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux); 568 if (err) { 569 bpf_link_cleanup(&link_primer); 570 return err; 571 } 572 } 573 574 return bpf_link_settle(&link_primer); 575 } 576 577 static void init_seq_meta(struct bpf_iter_priv_data *priv_data, 578 struct bpf_iter_target_info *tinfo, 579 const struct bpf_iter_seq_info *seq_info, 580 struct bpf_prog *prog) 581 { 582 priv_data->tinfo = tinfo; 583 priv_data->seq_info = seq_info; 584 priv_data->prog = prog; 585 priv_data->session_id = atomic64_inc_return(&session_id); 586 priv_data->seq_num = 0; 587 priv_data->done_stop = false; 588 } 589 590 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 591 const struct bpf_iter_seq_info *seq_info) 592 { 593 struct bpf_iter_priv_data *priv_data; 594 struct bpf_iter_target_info *tinfo; 595 struct bpf_prog *prog; 596 u32 total_priv_dsize; 597 struct seq_file *seq; 598 int err = 0; 599 600 mutex_lock(&link_mutex); 601 prog = link->link.prog; 602 bpf_prog_inc(prog); 603 mutex_unlock(&link_mutex); 604 605 tinfo = link->tinfo; 606 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) + 607 seq_info->seq_priv_size; 608 priv_data = __seq_open_private(file, seq_info->seq_ops, 609 total_priv_dsize); 610 if (!priv_data) { 611 err = -ENOMEM; 612 goto release_prog; 613 } 614 615 if (seq_info->init_seq_private) { 616 err = seq_info->init_seq_private(priv_data->target_private, &link->aux); 617 if (err) 618 goto release_seq_file; 619 } 620 621 init_seq_meta(priv_data, tinfo, seq_info, prog); 622 seq = file->private_data; 623 seq->private = priv_data->target_private; 624 625 return 0; 626 627 release_seq_file: 628 seq_release_private(file->f_inode, file); 629 file->private_data = NULL; 630 release_prog: 631 bpf_prog_put(prog); 632 return err; 633 } 634 635 int bpf_iter_new_fd(struct bpf_link *link) 636 { 637 struct bpf_iter_link *iter_link; 638 struct file *file; 639 unsigned int flags; 640 int err, fd; 641 642 if (link->ops != &bpf_iter_link_lops) 643 return -EINVAL; 644 645 flags = O_RDONLY | O_CLOEXEC; 646 fd = get_unused_fd_flags(flags); 647 if (fd < 0) 648 return fd; 649 650 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags); 651 if (IS_ERR(file)) { 652 err = PTR_ERR(file); 653 goto free_fd; 654 } 655 656 iter_link = container_of(link, struct bpf_iter_link, link); 657 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link)); 658 if (err) 659 goto free_file; 660 661 fd_install(fd, file); 662 return fd; 663 664 free_file: 665 fput(file); 666 free_fd: 667 put_unused_fd(fd); 668 return err; 669 } 670 671 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) 672 { 673 struct bpf_iter_priv_data *iter_priv; 674 struct seq_file *seq; 675 void *seq_priv; 676 677 seq = meta->seq; 678 if (seq->file->f_op != &bpf_iter_fops) 679 return NULL; 680 681 seq_priv = seq->private; 682 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data, 683 target_private); 684 685 if (in_stop && iter_priv->done_stop) 686 return NULL; 687 688 meta->session_id = iter_priv->session_id; 689 meta->seq_num = iter_priv->seq_num; 690 691 return iter_priv->prog; 692 } 693 694 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) 695 { 696 struct bpf_run_ctx run_ctx, *old_run_ctx; 697 int ret; 698 699 if (prog->sleepable) { 700 rcu_read_lock_trace(); 701 migrate_disable(); 702 might_fault(); 703 old_run_ctx = bpf_set_run_ctx(&run_ctx); 704 ret = bpf_prog_run(prog, ctx); 705 bpf_reset_run_ctx(old_run_ctx); 706 migrate_enable(); 707 rcu_read_unlock_trace(); 708 } else { 709 rcu_read_lock(); 710 migrate_disable(); 711 old_run_ctx = bpf_set_run_ctx(&run_ctx); 712 ret = bpf_prog_run(prog, ctx); 713 bpf_reset_run_ctx(old_run_ctx); 714 migrate_enable(); 715 rcu_read_unlock(); 716 } 717 718 /* bpf program can only return 0 or 1: 719 * 0 : okay 720 * 1 : retry the same object 721 * The bpf_iter_run_prog() return value 722 * will be seq_ops->show() return value. 723 */ 724 return ret == 0 ? 0 : -EAGAIN; 725 } 726 727 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn, 728 void *, callback_ctx, u64, flags) 729 { 730 return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags); 731 } 732 733 const struct bpf_func_proto bpf_for_each_map_elem_proto = { 734 .func = bpf_for_each_map_elem, 735 .gpl_only = false, 736 .ret_type = RET_INTEGER, 737 .arg1_type = ARG_CONST_MAP_PTR, 738 .arg2_type = ARG_PTR_TO_FUNC, 739 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 740 .arg4_type = ARG_ANYTHING, 741 }; 742 743 BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, 744 u64, flags) 745 { 746 bpf_callback_t callback = (bpf_callback_t)callback_fn; 747 u64 ret; 748 u32 i; 749 750 /* Note: these safety checks are also verified when bpf_loop 751 * is inlined, be careful to modify this code in sync. See 752 * function verifier.c:inline_bpf_loop. 753 */ 754 if (flags) 755 return -EINVAL; 756 if (nr_loops > BPF_MAX_LOOPS) 757 return -E2BIG; 758 759 for (i = 0; i < nr_loops; i++) { 760 ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); 761 /* return value: 0 - continue, 1 - stop and return */ 762 if (ret) 763 return i + 1; 764 } 765 766 return i; 767 } 768 769 const struct bpf_func_proto bpf_loop_proto = { 770 .func = bpf_loop, 771 .gpl_only = false, 772 .ret_type = RET_INTEGER, 773 .arg1_type = ARG_ANYTHING, 774 .arg2_type = ARG_PTR_TO_FUNC, 775 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 776 .arg4_type = ARG_ANYTHING, 777 }; 778 779 struct bpf_iter_num_kern { 780 int cur; /* current value, inclusive */ 781 int end; /* final value, exclusive */ 782 } __aligned(8); 783 784 __bpf_kfunc_start_defs(); 785 786 __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) 787 { 788 struct bpf_iter_num_kern *s = (void *)it; 789 790 BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num)); 791 BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num)); 792 793 /* start == end is legit, it's an empty range and we'll just get NULL 794 * on first (and any subsequent) bpf_iter_num_next() call 795 */ 796 if (start > end) { 797 s->cur = s->end = 0; 798 return -EINVAL; 799 } 800 801 /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ 802 if ((s64)end - (s64)start > BPF_MAX_LOOPS) { 803 s->cur = s->end = 0; 804 return -E2BIG; 805 } 806 807 /* user will call bpf_iter_num_next() first, 808 * which will set s->cur to exactly start value; 809 * underflow shouldn't matter 810 */ 811 s->cur = start - 1; 812 s->end = end; 813 814 return 0; 815 } 816 817 __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) 818 { 819 struct bpf_iter_num_kern *s = (void *)it; 820 821 /* check failed initialization or if we are done (same behavior); 822 * need to be careful about overflow, so convert to s64 for checks, 823 * e.g., if s->cur == s->end == INT_MAX, we can't just do 824 * s->cur + 1 >= s->end 825 */ 826 if ((s64)(s->cur + 1) >= s->end) { 827 s->cur = s->end = 0; 828 return NULL; 829 } 830 831 s->cur++; 832 833 return &s->cur; 834 } 835 836 __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) 837 { 838 struct bpf_iter_num_kern *s = (void *)it; 839 840 s->cur = s->end = 0; 841 } 842 843 __bpf_kfunc_end_defs(); 844