1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 Facebook */ 3 4 #include <linux/fs.h> 5 #include <linux/anon_inodes.h> 6 #include <linux/filter.h> 7 #include <linux/bpf.h> 8 #include <linux/rcupdate_trace.h> 9 10 struct bpf_iter_target_info { 11 struct list_head list; 12 const struct bpf_iter_reg *reg_info; 13 u32 btf_id; /* cached value */ 14 }; 15 16 struct bpf_iter_link { 17 struct bpf_link link; 18 struct bpf_iter_aux_info aux; 19 struct bpf_iter_target_info *tinfo; 20 }; 21 22 struct bpf_iter_priv_data { 23 struct bpf_iter_target_info *tinfo; 24 const struct bpf_iter_seq_info *seq_info; 25 struct bpf_prog *prog; 26 u64 session_id; 27 u64 seq_num; 28 bool done_stop; 29 u8 target_private[] __aligned(8); 30 }; 31 32 static struct list_head targets = LIST_HEAD_INIT(targets); 33 static DEFINE_MUTEX(targets_mutex); 34 35 /* protect bpf_iter_link changes */ 36 static DEFINE_MUTEX(link_mutex); 37 38 /* incremented on every opened seq_file */ 39 static atomic64_t session_id; 40 41 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 42 const struct bpf_iter_seq_info *seq_info); 43 44 static void bpf_iter_inc_seq_num(struct seq_file *seq) 45 { 46 struct bpf_iter_priv_data *iter_priv; 47 48 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 49 target_private); 50 iter_priv->seq_num++; 51 } 52 53 static void bpf_iter_dec_seq_num(struct seq_file *seq) 54 { 55 struct bpf_iter_priv_data *iter_priv; 56 57 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 58 target_private); 59 iter_priv->seq_num--; 60 } 61 62 static void bpf_iter_done_stop(struct seq_file *seq) 63 { 64 struct bpf_iter_priv_data *iter_priv; 65 66 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 67 target_private); 68 iter_priv->done_stop = true; 69 } 70 71 static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo) 72 { 73 return tinfo->reg_info->feature & BPF_ITER_RESCHED; 74 } 75 76 static bool bpf_iter_support_resched(struct seq_file *seq) 77 { 78 struct bpf_iter_priv_data *iter_priv; 79 80 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 81 target_private); 82 return bpf_iter_target_support_resched(iter_priv->tinfo); 83 } 84 85 /* maximum visited objects before bailing out */ 86 #define MAX_ITER_OBJECTS 1000000 87 88 /* bpf_seq_read, a customized and simpler version for bpf iterator. 89 * The following are differences from seq_read(): 90 * . fixed buffer size (PAGE_SIZE) 91 * . assuming NULL ->llseek() 92 * . stop() may call bpf program, handling potential overflow there 93 */ 94 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, 95 loff_t *ppos) 96 { 97 struct seq_file *seq = file->private_data; 98 size_t n, offs, copied = 0; 99 int err = 0, num_objs = 0; 100 bool can_resched; 101 void *p; 102 103 mutex_lock(&seq->lock); 104 105 if (!seq->buf) { 106 seq->size = PAGE_SIZE << 3; 107 seq->buf = kvmalloc(seq->size, GFP_KERNEL); 108 if (!seq->buf) { 109 err = -ENOMEM; 110 goto done; 111 } 112 } 113 114 if (seq->count) { 115 n = min(seq->count, size); 116 err = copy_to_user(buf, seq->buf + seq->from, n); 117 if (err) { 118 err = -EFAULT; 119 goto done; 120 } 121 seq->count -= n; 122 seq->from += n; 123 copied = n; 124 goto done; 125 } 126 127 seq->from = 0; 128 p = seq->op->start(seq, &seq->index); 129 if (!p) 130 goto stop; 131 if (IS_ERR(p)) { 132 err = PTR_ERR(p); 133 seq->op->stop(seq, p); 134 seq->count = 0; 135 goto done; 136 } 137 138 err = seq->op->show(seq, p); 139 if (err > 0) { 140 /* object is skipped, decrease seq_num, so next 141 * valid object can reuse the same seq_num. 142 */ 143 bpf_iter_dec_seq_num(seq); 144 seq->count = 0; 145 } else if (err < 0 || seq_has_overflowed(seq)) { 146 if (!err) 147 err = -E2BIG; 148 seq->op->stop(seq, p); 149 seq->count = 0; 150 goto done; 151 } 152 153 can_resched = bpf_iter_support_resched(seq); 154 while (1) { 155 loff_t pos = seq->index; 156 157 num_objs++; 158 offs = seq->count; 159 p = seq->op->next(seq, p, &seq->index); 160 if (pos == seq->index) { 161 pr_info_ratelimited("buggy seq_file .next function %ps " 162 "did not updated position index\n", 163 seq->op->next); 164 seq->index++; 165 } 166 167 if (IS_ERR_OR_NULL(p)) 168 break; 169 170 /* got a valid next object, increase seq_num */ 171 bpf_iter_inc_seq_num(seq); 172 173 if (seq->count >= size) 174 break; 175 176 if (num_objs >= MAX_ITER_OBJECTS) { 177 if (offs == 0) { 178 err = -EAGAIN; 179 seq->op->stop(seq, p); 180 goto done; 181 } 182 break; 183 } 184 185 err = seq->op->show(seq, p); 186 if (err > 0) { 187 bpf_iter_dec_seq_num(seq); 188 seq->count = offs; 189 } else if (err < 0 || seq_has_overflowed(seq)) { 190 seq->count = offs; 191 if (offs == 0) { 192 if (!err) 193 err = -E2BIG; 194 seq->op->stop(seq, p); 195 goto done; 196 } 197 break; 198 } 199 200 if (can_resched) 201 cond_resched(); 202 } 203 stop: 204 offs = seq->count; 205 if (IS_ERR(p)) { 206 seq->op->stop(seq, NULL); 207 err = PTR_ERR(p); 208 goto done; 209 } 210 /* bpf program called if !p */ 211 seq->op->stop(seq, p); 212 if (!p) { 213 if (!seq_has_overflowed(seq)) { 214 bpf_iter_done_stop(seq); 215 } else { 216 seq->count = offs; 217 if (offs == 0) { 218 err = -E2BIG; 219 goto done; 220 } 221 } 222 } 223 224 n = min(seq->count, size); 225 err = copy_to_user(buf, seq->buf, n); 226 if (err) { 227 err = -EFAULT; 228 goto done; 229 } 230 copied = n; 231 seq->count -= n; 232 seq->from = n; 233 done: 234 if (!copied) 235 copied = err; 236 else 237 *ppos += copied; 238 mutex_unlock(&seq->lock); 239 return copied; 240 } 241 242 static const struct bpf_iter_seq_info * 243 __get_seq_info(struct bpf_iter_link *link) 244 { 245 const struct bpf_iter_seq_info *seq_info; 246 247 if (link->aux.map) { 248 seq_info = link->aux.map->ops->iter_seq_info; 249 if (seq_info) 250 return seq_info; 251 } 252 253 return link->tinfo->reg_info->seq_info; 254 } 255 256 static int iter_open(struct inode *inode, struct file *file) 257 { 258 struct bpf_iter_link *link = inode->i_private; 259 260 return prepare_seq_file(file, link, __get_seq_info(link)); 261 } 262 263 static int iter_release(struct inode *inode, struct file *file) 264 { 265 struct bpf_iter_priv_data *iter_priv; 266 struct seq_file *seq; 267 268 seq = file->private_data; 269 if (!seq) 270 return 0; 271 272 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 273 target_private); 274 275 if (iter_priv->seq_info->fini_seq_private) 276 iter_priv->seq_info->fini_seq_private(seq->private); 277 278 bpf_prog_put(iter_priv->prog); 279 seq->private = iter_priv; 280 281 return seq_release_private(inode, file); 282 } 283 284 const struct file_operations bpf_iter_fops = { 285 .open = iter_open, 286 .read = bpf_seq_read, 287 .release = iter_release, 288 }; 289 290 /* The argument reg_info will be cached in bpf_iter_target_info. 291 * The common practice is to declare target reg_info as 292 * a const static variable and passed as an argument to 293 * bpf_iter_reg_target(). 294 */ 295 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) 296 { 297 struct bpf_iter_target_info *tinfo; 298 299 tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL); 300 if (!tinfo) 301 return -ENOMEM; 302 303 tinfo->reg_info = reg_info; 304 INIT_LIST_HEAD(&tinfo->list); 305 306 mutex_lock(&targets_mutex); 307 list_add(&tinfo->list, &targets); 308 mutex_unlock(&targets_mutex); 309 310 return 0; 311 } 312 313 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info) 314 { 315 struct bpf_iter_target_info *tinfo; 316 bool found = false; 317 318 mutex_lock(&targets_mutex); 319 list_for_each_entry(tinfo, &targets, list) { 320 if (reg_info == tinfo->reg_info) { 321 list_del(&tinfo->list); 322 kfree(tinfo); 323 found = true; 324 break; 325 } 326 } 327 mutex_unlock(&targets_mutex); 328 329 WARN_ON(found == false); 330 } 331 332 static void cache_btf_id(struct bpf_iter_target_info *tinfo, 333 struct bpf_prog *prog) 334 { 335 tinfo->btf_id = prog->aux->attach_btf_id; 336 } 337 338 int bpf_iter_prog_supported(struct bpf_prog *prog) 339 { 340 const char *attach_fname = prog->aux->attach_func_name; 341 struct bpf_iter_target_info *tinfo = NULL, *iter; 342 u32 prog_btf_id = prog->aux->attach_btf_id; 343 const char *prefix = BPF_ITER_FUNC_PREFIX; 344 int prefix_len = strlen(prefix); 345 346 if (strncmp(attach_fname, prefix, prefix_len)) 347 return -EINVAL; 348 349 mutex_lock(&targets_mutex); 350 list_for_each_entry(iter, &targets, list) { 351 if (iter->btf_id && iter->btf_id == prog_btf_id) { 352 tinfo = iter; 353 break; 354 } 355 if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) { 356 cache_btf_id(iter, prog); 357 tinfo = iter; 358 break; 359 } 360 } 361 mutex_unlock(&targets_mutex); 362 363 if (!tinfo) 364 return -EINVAL; 365 366 return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info, 367 tinfo->reg_info->ctx_arg_info_size); 368 } 369 370 const struct bpf_func_proto * 371 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 372 { 373 const struct bpf_iter_target_info *tinfo; 374 const struct bpf_func_proto *fn = NULL; 375 376 mutex_lock(&targets_mutex); 377 list_for_each_entry(tinfo, &targets, list) { 378 if (tinfo->btf_id == prog->aux->attach_btf_id) { 379 const struct bpf_iter_reg *reg_info; 380 381 reg_info = tinfo->reg_info; 382 if (reg_info->get_func_proto) 383 fn = reg_info->get_func_proto(func_id, prog); 384 break; 385 } 386 } 387 mutex_unlock(&targets_mutex); 388 389 return fn; 390 } 391 392 static void bpf_iter_link_release(struct bpf_link *link) 393 { 394 struct bpf_iter_link *iter_link = 395 container_of(link, struct bpf_iter_link, link); 396 397 if (iter_link->tinfo->reg_info->detach_target) 398 iter_link->tinfo->reg_info->detach_target(&iter_link->aux); 399 } 400 401 static void bpf_iter_link_dealloc(struct bpf_link *link) 402 { 403 struct bpf_iter_link *iter_link = 404 container_of(link, struct bpf_iter_link, link); 405 406 kfree(iter_link); 407 } 408 409 static int bpf_iter_link_replace(struct bpf_link *link, 410 struct bpf_prog *new_prog, 411 struct bpf_prog *old_prog) 412 { 413 int ret = 0; 414 415 mutex_lock(&link_mutex); 416 if (old_prog && link->prog != old_prog) { 417 ret = -EPERM; 418 goto out_unlock; 419 } 420 421 if (link->prog->type != new_prog->type || 422 link->prog->expected_attach_type != new_prog->expected_attach_type || 423 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { 424 ret = -EINVAL; 425 goto out_unlock; 426 } 427 428 old_prog = xchg(&link->prog, new_prog); 429 bpf_prog_put(old_prog); 430 431 out_unlock: 432 mutex_unlock(&link_mutex); 433 return ret; 434 } 435 436 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link, 437 struct seq_file *seq) 438 { 439 struct bpf_iter_link *iter_link = 440 container_of(link, struct bpf_iter_link, link); 441 bpf_iter_show_fdinfo_t show_fdinfo; 442 443 seq_printf(seq, 444 "target_name:\t%s\n", 445 iter_link->tinfo->reg_info->target); 446 447 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo; 448 if (show_fdinfo) 449 show_fdinfo(&iter_link->aux, seq); 450 } 451 452 static int bpf_iter_link_fill_link_info(const struct bpf_link *link, 453 struct bpf_link_info *info) 454 { 455 struct bpf_iter_link *iter_link = 456 container_of(link, struct bpf_iter_link, link); 457 char __user *ubuf = u64_to_user_ptr(info->iter.target_name); 458 bpf_iter_fill_link_info_t fill_link_info; 459 u32 ulen = info->iter.target_name_len; 460 const char *target_name; 461 u32 target_len; 462 463 if (!ulen ^ !ubuf) 464 return -EINVAL; 465 466 target_name = iter_link->tinfo->reg_info->target; 467 target_len = strlen(target_name); 468 info->iter.target_name_len = target_len + 1; 469 470 if (ubuf) { 471 if (ulen >= target_len + 1) { 472 if (copy_to_user(ubuf, target_name, target_len + 1)) 473 return -EFAULT; 474 } else { 475 char zero = '\0'; 476 477 if (copy_to_user(ubuf, target_name, ulen - 1)) 478 return -EFAULT; 479 if (put_user(zero, ubuf + ulen - 1)) 480 return -EFAULT; 481 return -ENOSPC; 482 } 483 } 484 485 fill_link_info = iter_link->tinfo->reg_info->fill_link_info; 486 if (fill_link_info) 487 return fill_link_info(&iter_link->aux, info); 488 489 return 0; 490 } 491 492 static const struct bpf_link_ops bpf_iter_link_lops = { 493 .release = bpf_iter_link_release, 494 .dealloc = bpf_iter_link_dealloc, 495 .update_prog = bpf_iter_link_replace, 496 .show_fdinfo = bpf_iter_link_show_fdinfo, 497 .fill_link_info = bpf_iter_link_fill_link_info, 498 }; 499 500 bool bpf_link_is_iter(struct bpf_link *link) 501 { 502 return link->ops == &bpf_iter_link_lops; 503 } 504 505 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, 506 struct bpf_prog *prog) 507 { 508 struct bpf_iter_target_info *tinfo = NULL, *iter; 509 struct bpf_link_primer link_primer; 510 union bpf_iter_link_info linfo; 511 struct bpf_iter_link *link; 512 u32 prog_btf_id, linfo_len; 513 bpfptr_t ulinfo; 514 int err; 515 516 if (attr->link_create.target_fd || attr->link_create.flags) 517 return -EINVAL; 518 519 memset(&linfo, 0, sizeof(union bpf_iter_link_info)); 520 521 ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel); 522 linfo_len = attr->link_create.iter_info_len; 523 if (bpfptr_is_null(ulinfo) ^ !linfo_len) 524 return -EINVAL; 525 526 if (!bpfptr_is_null(ulinfo)) { 527 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo), 528 linfo_len); 529 if (err) 530 return err; 531 linfo_len = min_t(u32, linfo_len, sizeof(linfo)); 532 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len)) 533 return -EFAULT; 534 } 535 536 prog_btf_id = prog->aux->attach_btf_id; 537 mutex_lock(&targets_mutex); 538 list_for_each_entry(iter, &targets, list) { 539 if (iter->btf_id == prog_btf_id) { 540 tinfo = iter; 541 break; 542 } 543 } 544 mutex_unlock(&targets_mutex); 545 if (!tinfo) 546 return -ENOENT; 547 548 /* Only allow sleepable program for resched-able iterator */ 549 if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) 550 return -EINVAL; 551 552 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN); 553 if (!link) 554 return -ENOMEM; 555 556 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog); 557 link->tinfo = tinfo; 558 559 err = bpf_link_prime(&link->link, &link_primer); 560 if (err) { 561 kfree(link); 562 return err; 563 } 564 565 if (tinfo->reg_info->attach_target) { 566 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux); 567 if (err) { 568 bpf_link_cleanup(&link_primer); 569 return err; 570 } 571 } 572 573 return bpf_link_settle(&link_primer); 574 } 575 576 static void init_seq_meta(struct bpf_iter_priv_data *priv_data, 577 struct bpf_iter_target_info *tinfo, 578 const struct bpf_iter_seq_info *seq_info, 579 struct bpf_prog *prog) 580 { 581 priv_data->tinfo = tinfo; 582 priv_data->seq_info = seq_info; 583 priv_data->prog = prog; 584 priv_data->session_id = atomic64_inc_return(&session_id); 585 priv_data->seq_num = 0; 586 priv_data->done_stop = false; 587 } 588 589 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 590 const struct bpf_iter_seq_info *seq_info) 591 { 592 struct bpf_iter_priv_data *priv_data; 593 struct bpf_iter_target_info *tinfo; 594 struct bpf_prog *prog; 595 u32 total_priv_dsize; 596 struct seq_file *seq; 597 int err = 0; 598 599 mutex_lock(&link_mutex); 600 prog = link->link.prog; 601 bpf_prog_inc(prog); 602 mutex_unlock(&link_mutex); 603 604 tinfo = link->tinfo; 605 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) + 606 seq_info->seq_priv_size; 607 priv_data = __seq_open_private(file, seq_info->seq_ops, 608 total_priv_dsize); 609 if (!priv_data) { 610 err = -ENOMEM; 611 goto release_prog; 612 } 613 614 if (seq_info->init_seq_private) { 615 err = seq_info->init_seq_private(priv_data->target_private, &link->aux); 616 if (err) 617 goto release_seq_file; 618 } 619 620 init_seq_meta(priv_data, tinfo, seq_info, prog); 621 seq = file->private_data; 622 seq->private = priv_data->target_private; 623 624 return 0; 625 626 release_seq_file: 627 seq_release_private(file->f_inode, file); 628 file->private_data = NULL; 629 release_prog: 630 bpf_prog_put(prog); 631 return err; 632 } 633 634 int bpf_iter_new_fd(struct bpf_link *link) 635 { 636 struct bpf_iter_link *iter_link; 637 struct file *file; 638 unsigned int flags; 639 int err, fd; 640 641 if (link->ops != &bpf_iter_link_lops) 642 return -EINVAL; 643 644 flags = O_RDONLY | O_CLOEXEC; 645 fd = get_unused_fd_flags(flags); 646 if (fd < 0) 647 return fd; 648 649 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags); 650 if (IS_ERR(file)) { 651 err = PTR_ERR(file); 652 goto free_fd; 653 } 654 655 iter_link = container_of(link, struct bpf_iter_link, link); 656 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link)); 657 if (err) 658 goto free_file; 659 660 fd_install(fd, file); 661 return fd; 662 663 free_file: 664 fput(file); 665 free_fd: 666 put_unused_fd(fd); 667 return err; 668 } 669 670 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) 671 { 672 struct bpf_iter_priv_data *iter_priv; 673 struct seq_file *seq; 674 void *seq_priv; 675 676 seq = meta->seq; 677 if (seq->file->f_op != &bpf_iter_fops) 678 return NULL; 679 680 seq_priv = seq->private; 681 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data, 682 target_private); 683 684 if (in_stop && iter_priv->done_stop) 685 return NULL; 686 687 meta->session_id = iter_priv->session_id; 688 meta->seq_num = iter_priv->seq_num; 689 690 return iter_priv->prog; 691 } 692 693 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) 694 { 695 struct bpf_run_ctx run_ctx, *old_run_ctx; 696 int ret; 697 698 if (prog->sleepable) { 699 rcu_read_lock_trace(); 700 migrate_disable(); 701 might_fault(); 702 old_run_ctx = bpf_set_run_ctx(&run_ctx); 703 ret = bpf_prog_run(prog, ctx); 704 bpf_reset_run_ctx(old_run_ctx); 705 migrate_enable(); 706 rcu_read_unlock_trace(); 707 } else { 708 rcu_read_lock(); 709 migrate_disable(); 710 old_run_ctx = bpf_set_run_ctx(&run_ctx); 711 ret = bpf_prog_run(prog, ctx); 712 bpf_reset_run_ctx(old_run_ctx); 713 migrate_enable(); 714 rcu_read_unlock(); 715 } 716 717 /* bpf program can only return 0 or 1: 718 * 0 : okay 719 * 1 : retry the same object 720 * The bpf_iter_run_prog() return value 721 * will be seq_ops->show() return value. 722 */ 723 return ret == 0 ? 0 : -EAGAIN; 724 } 725 726 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn, 727 void *, callback_ctx, u64, flags) 728 { 729 return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags); 730 } 731 732 const struct bpf_func_proto bpf_for_each_map_elem_proto = { 733 .func = bpf_for_each_map_elem, 734 .gpl_only = false, 735 .ret_type = RET_INTEGER, 736 .arg1_type = ARG_CONST_MAP_PTR, 737 .arg2_type = ARG_PTR_TO_FUNC, 738 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 739 .arg4_type = ARG_ANYTHING, 740 }; 741 742 BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, 743 u64, flags) 744 { 745 bpf_callback_t callback = (bpf_callback_t)callback_fn; 746 u64 ret; 747 u32 i; 748 749 /* Note: these safety checks are also verified when bpf_loop 750 * is inlined, be careful to modify this code in sync. See 751 * function verifier.c:inline_bpf_loop. 752 */ 753 if (flags) 754 return -EINVAL; 755 if (nr_loops > BPF_MAX_LOOPS) 756 return -E2BIG; 757 758 for (i = 0; i < nr_loops; i++) { 759 ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); 760 /* return value: 0 - continue, 1 - stop and return */ 761 if (ret) 762 return i + 1; 763 } 764 765 return i; 766 } 767 768 const struct bpf_func_proto bpf_loop_proto = { 769 .func = bpf_loop, 770 .gpl_only = false, 771 .ret_type = RET_INTEGER, 772 .arg1_type = ARG_ANYTHING, 773 .arg2_type = ARG_PTR_TO_FUNC, 774 .arg3_type = ARG_PTR_TO_STACK_OR_NULL, 775 .arg4_type = ARG_ANYTHING, 776 }; 777 778 struct bpf_iter_num_kern { 779 int cur; /* current value, inclusive */ 780 int end; /* final value, exclusive */ 781 } __aligned(8); 782 783 __bpf_kfunc_start_defs(); 784 785 __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) 786 { 787 struct bpf_iter_num_kern *s = (void *)it; 788 789 BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num)); 790 BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num)); 791 792 /* start == end is legit, it's an empty range and we'll just get NULL 793 * on first (and any subsequent) bpf_iter_num_next() call 794 */ 795 if (start > end) { 796 s->cur = s->end = 0; 797 return -EINVAL; 798 } 799 800 /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ 801 if ((s64)end - (s64)start > BPF_MAX_LOOPS) { 802 s->cur = s->end = 0; 803 return -E2BIG; 804 } 805 806 /* user will call bpf_iter_num_next() first, 807 * which will set s->cur to exactly start value; 808 * underflow shouldn't matter 809 */ 810 s->cur = start - 1; 811 s->end = end; 812 813 return 0; 814 } 815 816 __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) 817 { 818 struct bpf_iter_num_kern *s = (void *)it; 819 820 /* check failed initialization or if we are done (same behavior); 821 * need to be careful about overflow, so convert to s64 for checks, 822 * e.g., if s->cur == s->end == INT_MAX, we can't just do 823 * s->cur + 1 >= s->end 824 */ 825 if ((s64)(s->cur + 1) >= s->end) { 826 s->cur = s->end = 0; 827 return NULL; 828 } 829 830 s->cur++; 831 832 return &s->cur; 833 } 834 835 __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) 836 { 837 struct bpf_iter_num_kern *s = (void *)it; 838 839 s->cur = s->end = 0; 840 } 841 842 __bpf_kfunc_end_defs(); 843