1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/rculist.h> 4 #include <linux/list.h> 5 #include <linux/hash.h> 6 #include <linux/types.h> 7 #include <linux/spinlock.h> 8 #include <linux/bpf.h> 9 #include <linux/btf_ids.h> 10 #include <linux/bpf_local_storage.h> 11 #include <net/bpf_sk_storage.h> 12 #include <net/sock.h> 13 #include <uapi/linux/sock_diag.h> 14 #include <uapi/linux/btf.h> 15 16 DEFINE_BPF_STORAGE_CACHE(sk_cache); 17 18 static int omem_charge(struct sock *sk, unsigned int size) 19 { 20 /* same check as in sock_kmalloc() */ 21 if (size <= sysctl_optmem_max && 22 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { 23 atomic_add(size, &sk->sk_omem_alloc); 24 return 0; 25 } 26 27 return -ENOMEM; 28 } 29 30 static struct bpf_local_storage_data * 31 sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit) 32 { 33 struct bpf_local_storage *sk_storage; 34 struct bpf_local_storage_map *smap; 35 36 sk_storage = rcu_dereference(sk->sk_bpf_storage); 37 if (!sk_storage) 38 return NULL; 39 40 smap = (struct bpf_local_storage_map *)map; 41 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit); 42 } 43 44 static int sk_storage_delete(struct sock *sk, struct bpf_map *map) 45 { 46 struct bpf_local_storage_data *sdata; 47 48 sdata = sk_storage_lookup(sk, map, false); 49 if (!sdata) 50 return -ENOENT; 51 52 bpf_selem_unlink(SELEM(sdata)); 53 54 return 0; 55 } 56 57 /* Called by __sk_destruct() & bpf_sk_storage_clone() */ 58 void bpf_sk_storage_free(struct sock *sk) 59 { 60 struct bpf_local_storage_elem *selem; 61 struct bpf_local_storage *sk_storage; 62 bool free_sk_storage = false; 63 struct hlist_node *n; 64 65 rcu_read_lock(); 66 sk_storage = rcu_dereference(sk->sk_bpf_storage); 67 if (!sk_storage) { 68 rcu_read_unlock(); 69 return; 70 } 71 72 /* Netiher the bpf_prog nor the bpf-map's syscall 73 * could be modifying the sk_storage->list now. 74 * Thus, no elem can be added-to or deleted-from the 75 * sk_storage->list by the bpf_prog or by the bpf-map's syscall. 76 * 77 * It is racing with bpf_local_storage_map_free() alone 78 * when unlinking elem from the sk_storage->list and 79 * the map's bucket->list. 80 */ 81 raw_spin_lock_bh(&sk_storage->lock); 82 hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) { 83 /* Always unlink from map before unlinking from 84 * sk_storage. 85 */ 86 bpf_selem_unlink_map(selem); 87 free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage, 88 selem, true); 89 } 90 raw_spin_unlock_bh(&sk_storage->lock); 91 rcu_read_unlock(); 92 93 if (free_sk_storage) 94 kfree_rcu(sk_storage, rcu); 95 } 96 97 static void sk_storage_map_free(struct bpf_map *map) 98 { 99 struct bpf_local_storage_map *smap; 100 101 smap = (struct bpf_local_storage_map *)map; 102 bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx); 103 bpf_local_storage_map_free(smap); 104 } 105 106 static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr) 107 { 108 struct bpf_local_storage_map *smap; 109 110 smap = bpf_local_storage_map_alloc(attr); 111 if (IS_ERR(smap)) 112 return ERR_CAST(smap); 113 114 smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache); 115 return &smap->map; 116 } 117 118 static int notsupp_get_next_key(struct bpf_map *map, void *key, 119 void *next_key) 120 { 121 return -ENOTSUPP; 122 } 123 124 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key) 125 { 126 struct bpf_local_storage_data *sdata; 127 struct socket *sock; 128 int fd, err; 129 130 fd = *(int *)key; 131 sock = sockfd_lookup(fd, &err); 132 if (sock) { 133 sdata = sk_storage_lookup(sock->sk, map, true); 134 sockfd_put(sock); 135 return sdata ? sdata->data : NULL; 136 } 137 138 return ERR_PTR(err); 139 } 140 141 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, 142 void *value, u64 map_flags) 143 { 144 struct bpf_local_storage_data *sdata; 145 struct socket *sock; 146 int fd, err; 147 148 fd = *(int *)key; 149 sock = sockfd_lookup(fd, &err); 150 if (sock) { 151 sdata = bpf_local_storage_update( 152 sock->sk, (struct bpf_local_storage_map *)map, value, 153 map_flags); 154 sockfd_put(sock); 155 return PTR_ERR_OR_ZERO(sdata); 156 } 157 158 return err; 159 } 160 161 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) 162 { 163 struct socket *sock; 164 int fd, err; 165 166 fd = *(int *)key; 167 sock = sockfd_lookup(fd, &err); 168 if (sock) { 169 err = sk_storage_delete(sock->sk, map); 170 sockfd_put(sock); 171 return err; 172 } 173 174 return err; 175 } 176 177 static struct bpf_local_storage_elem * 178 bpf_sk_storage_clone_elem(struct sock *newsk, 179 struct bpf_local_storage_map *smap, 180 struct bpf_local_storage_elem *selem) 181 { 182 struct bpf_local_storage_elem *copy_selem; 183 184 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true); 185 if (!copy_selem) 186 return NULL; 187 188 if (map_value_has_spin_lock(&smap->map)) 189 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data, 190 SDATA(selem)->data, true); 191 else 192 copy_map_value(&smap->map, SDATA(copy_selem)->data, 193 SDATA(selem)->data); 194 195 return copy_selem; 196 } 197 198 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) 199 { 200 struct bpf_local_storage *new_sk_storage = NULL; 201 struct bpf_local_storage *sk_storage; 202 struct bpf_local_storage_elem *selem; 203 int ret = 0; 204 205 RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); 206 207 rcu_read_lock(); 208 sk_storage = rcu_dereference(sk->sk_bpf_storage); 209 210 if (!sk_storage || hlist_empty(&sk_storage->list)) 211 goto out; 212 213 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 214 struct bpf_local_storage_elem *copy_selem; 215 struct bpf_local_storage_map *smap; 216 struct bpf_map *map; 217 218 smap = rcu_dereference(SDATA(selem)->smap); 219 if (!(smap->map.map_flags & BPF_F_CLONE)) 220 continue; 221 222 /* Note that for lockless listeners adding new element 223 * here can race with cleanup in bpf_local_storage_map_free. 224 * Try to grab map refcnt to make sure that it's still 225 * alive and prevent concurrent removal. 226 */ 227 map = bpf_map_inc_not_zero(&smap->map); 228 if (IS_ERR(map)) 229 continue; 230 231 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem); 232 if (!copy_selem) { 233 ret = -ENOMEM; 234 bpf_map_put(map); 235 goto out; 236 } 237 238 if (new_sk_storage) { 239 bpf_selem_link_map(smap, copy_selem); 240 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); 241 } else { 242 ret = bpf_local_storage_alloc(newsk, smap, copy_selem); 243 if (ret) { 244 kfree(copy_selem); 245 atomic_sub(smap->elem_size, 246 &newsk->sk_omem_alloc); 247 bpf_map_put(map); 248 goto out; 249 } 250 251 new_sk_storage = 252 rcu_dereference(copy_selem->local_storage); 253 } 254 bpf_map_put(map); 255 } 256 257 out: 258 rcu_read_unlock(); 259 260 /* In case of an error, don't free anything explicitly here, the 261 * caller is responsible to call bpf_sk_storage_free. 262 */ 263 264 return ret; 265 } 266 267 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, 268 void *, value, u64, flags) 269 { 270 struct bpf_local_storage_data *sdata; 271 272 if (flags > BPF_SK_STORAGE_GET_F_CREATE) 273 return (unsigned long)NULL; 274 275 sdata = sk_storage_lookup(sk, map, true); 276 if (sdata) 277 return (unsigned long)sdata->data; 278 279 if (flags == BPF_SK_STORAGE_GET_F_CREATE && 280 /* Cannot add new elem to a going away sk. 281 * Otherwise, the new elem may become a leak 282 * (and also other memory issues during map 283 * destruction). 284 */ 285 refcount_inc_not_zero(&sk->sk_refcnt)) { 286 sdata = bpf_local_storage_update( 287 sk, (struct bpf_local_storage_map *)map, value, 288 BPF_NOEXIST); 289 /* sk must be a fullsock (guaranteed by verifier), 290 * so sock_gen_put() is unnecessary. 291 */ 292 sock_put(sk); 293 return IS_ERR(sdata) ? 294 (unsigned long)NULL : (unsigned long)sdata->data; 295 } 296 297 return (unsigned long)NULL; 298 } 299 300 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) 301 { 302 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 303 int err; 304 305 err = sk_storage_delete(sk, map); 306 sock_put(sk); 307 return err; 308 } 309 310 return -ENOENT; 311 } 312 313 static int sk_storage_charge(struct bpf_local_storage_map *smap, 314 void *owner, u32 size) 315 { 316 return omem_charge(owner, size); 317 } 318 319 static void sk_storage_uncharge(struct bpf_local_storage_map *smap, 320 void *owner, u32 size) 321 { 322 struct sock *sk = owner; 323 324 atomic_sub(size, &sk->sk_omem_alloc); 325 } 326 327 static struct bpf_local_storage __rcu ** 328 sk_storage_ptr(void *owner) 329 { 330 struct sock *sk = owner; 331 332 return &sk->sk_bpf_storage; 333 } 334 335 static int sk_storage_map_btf_id; 336 const struct bpf_map_ops sk_storage_map_ops = { 337 .map_meta_equal = bpf_map_meta_equal, 338 .map_alloc_check = bpf_local_storage_map_alloc_check, 339 .map_alloc = sk_storage_map_alloc, 340 .map_free = sk_storage_map_free, 341 .map_get_next_key = notsupp_get_next_key, 342 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem, 343 .map_update_elem = bpf_fd_sk_storage_update_elem, 344 .map_delete_elem = bpf_fd_sk_storage_delete_elem, 345 .map_check_btf = bpf_local_storage_map_check_btf, 346 .map_btf_name = "bpf_local_storage_map", 347 .map_btf_id = &sk_storage_map_btf_id, 348 .map_local_storage_charge = sk_storage_charge, 349 .map_local_storage_uncharge = sk_storage_uncharge, 350 .map_owner_storage_ptr = sk_storage_ptr, 351 }; 352 353 const struct bpf_func_proto bpf_sk_storage_get_proto = { 354 .func = bpf_sk_storage_get, 355 .gpl_only = false, 356 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 357 .arg1_type = ARG_CONST_MAP_PTR, 358 .arg2_type = ARG_PTR_TO_SOCKET, 359 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 360 .arg4_type = ARG_ANYTHING, 361 }; 362 363 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = { 364 .func = bpf_sk_storage_get, 365 .gpl_only = false, 366 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 367 .arg1_type = ARG_CONST_MAP_PTR, 368 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */ 369 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 370 .arg4_type = ARG_ANYTHING, 371 }; 372 373 const struct bpf_func_proto bpf_sk_storage_delete_proto = { 374 .func = bpf_sk_storage_delete, 375 .gpl_only = false, 376 .ret_type = RET_INTEGER, 377 .arg1_type = ARG_CONST_MAP_PTR, 378 .arg2_type = ARG_PTR_TO_SOCKET, 379 }; 380 381 BTF_ID_LIST(sk_storage_btf_ids) 382 BTF_ID_UNUSED 383 BTF_ID(struct, sock) 384 385 const struct bpf_func_proto sk_storage_get_btf_proto = { 386 .func = bpf_sk_storage_get, 387 .gpl_only = false, 388 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 389 .arg1_type = ARG_CONST_MAP_PTR, 390 .arg2_type = ARG_PTR_TO_BTF_ID, 391 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 392 .arg4_type = ARG_ANYTHING, 393 .btf_id = sk_storage_btf_ids, 394 }; 395 396 const struct bpf_func_proto sk_storage_delete_btf_proto = { 397 .func = bpf_sk_storage_delete, 398 .gpl_only = false, 399 .ret_type = RET_INTEGER, 400 .arg1_type = ARG_CONST_MAP_PTR, 401 .arg2_type = ARG_PTR_TO_BTF_ID, 402 .btf_id = sk_storage_btf_ids, 403 }; 404 405 struct bpf_sk_storage_diag { 406 u32 nr_maps; 407 struct bpf_map *maps[]; 408 }; 409 410 /* The reply will be like: 411 * INET_DIAG_BPF_SK_STORAGES (nla_nest) 412 * SK_DIAG_BPF_STORAGE (nla_nest) 413 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 414 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 415 * SK_DIAG_BPF_STORAGE (nla_nest) 416 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 417 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 418 * .... 419 */ 420 static int nla_value_size(u32 value_size) 421 { 422 /* SK_DIAG_BPF_STORAGE (nla_nest) 423 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 424 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 425 */ 426 return nla_total_size(0) + nla_total_size(sizeof(u32)) + 427 nla_total_size_64bit(value_size); 428 } 429 430 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag) 431 { 432 u32 i; 433 434 if (!diag) 435 return; 436 437 for (i = 0; i < diag->nr_maps; i++) 438 bpf_map_put(diag->maps[i]); 439 440 kfree(diag); 441 } 442 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free); 443 444 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag, 445 const struct bpf_map *map) 446 { 447 u32 i; 448 449 for (i = 0; i < diag->nr_maps; i++) { 450 if (diag->maps[i] == map) 451 return true; 452 } 453 454 return false; 455 } 456 457 struct bpf_sk_storage_diag * 458 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs) 459 { 460 struct bpf_sk_storage_diag *diag; 461 struct nlattr *nla; 462 u32 nr_maps = 0; 463 int rem, err; 464 465 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as 466 * the map_alloc_check() side also does. 467 */ 468 if (!bpf_capable()) 469 return ERR_PTR(-EPERM); 470 471 nla_for_each_nested(nla, nla_stgs, rem) { 472 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD) 473 nr_maps++; 474 } 475 476 diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps, 477 GFP_KERNEL); 478 if (!diag) 479 return ERR_PTR(-ENOMEM); 480 481 nla_for_each_nested(nla, nla_stgs, rem) { 482 struct bpf_map *map; 483 int map_fd; 484 485 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD) 486 continue; 487 488 map_fd = nla_get_u32(nla); 489 map = bpf_map_get(map_fd); 490 if (IS_ERR(map)) { 491 err = PTR_ERR(map); 492 goto err_free; 493 } 494 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) { 495 bpf_map_put(map); 496 err = -EINVAL; 497 goto err_free; 498 } 499 if (diag_check_dup(diag, map)) { 500 bpf_map_put(map); 501 err = -EEXIST; 502 goto err_free; 503 } 504 diag->maps[diag->nr_maps++] = map; 505 } 506 507 return diag; 508 509 err_free: 510 bpf_sk_storage_diag_free(diag); 511 return ERR_PTR(err); 512 } 513 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc); 514 515 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb) 516 { 517 struct nlattr *nla_stg, *nla_value; 518 struct bpf_local_storage_map *smap; 519 520 /* It cannot exceed max nlattr's payload */ 521 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE); 522 523 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE); 524 if (!nla_stg) 525 return -EMSGSIZE; 526 527 smap = rcu_dereference(sdata->smap); 528 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id)) 529 goto errout; 530 531 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE, 532 smap->map.value_size, 533 SK_DIAG_BPF_STORAGE_PAD); 534 if (!nla_value) 535 goto errout; 536 537 if (map_value_has_spin_lock(&smap->map)) 538 copy_map_value_locked(&smap->map, nla_data(nla_value), 539 sdata->data, true); 540 else 541 copy_map_value(&smap->map, nla_data(nla_value), sdata->data); 542 543 nla_nest_end(skb, nla_stg); 544 return 0; 545 546 errout: 547 nla_nest_cancel(skb, nla_stg); 548 return -EMSGSIZE; 549 } 550 551 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb, 552 int stg_array_type, 553 unsigned int *res_diag_size) 554 { 555 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 556 unsigned int diag_size = nla_total_size(0); 557 struct bpf_local_storage *sk_storage; 558 struct bpf_local_storage_elem *selem; 559 struct bpf_local_storage_map *smap; 560 struct nlattr *nla_stgs; 561 unsigned int saved_len; 562 int err = 0; 563 564 rcu_read_lock(); 565 566 sk_storage = rcu_dereference(sk->sk_bpf_storage); 567 if (!sk_storage || hlist_empty(&sk_storage->list)) { 568 rcu_read_unlock(); 569 return 0; 570 } 571 572 nla_stgs = nla_nest_start(skb, stg_array_type); 573 if (!nla_stgs) 574 /* Continue to learn diag_size */ 575 err = -EMSGSIZE; 576 577 saved_len = skb->len; 578 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 579 smap = rcu_dereference(SDATA(selem)->smap); 580 diag_size += nla_value_size(smap->map.value_size); 581 582 if (nla_stgs && diag_get(SDATA(selem), skb)) 583 /* Continue to learn diag_size */ 584 err = -EMSGSIZE; 585 } 586 587 rcu_read_unlock(); 588 589 if (nla_stgs) { 590 if (saved_len == skb->len) 591 nla_nest_cancel(skb, nla_stgs); 592 else 593 nla_nest_end(skb, nla_stgs); 594 } 595 596 if (diag_size == nla_total_size(0)) { 597 *res_diag_size = 0; 598 return 0; 599 } 600 601 *res_diag_size = diag_size; 602 return err; 603 } 604 605 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag, 606 struct sock *sk, struct sk_buff *skb, 607 int stg_array_type, 608 unsigned int *res_diag_size) 609 { 610 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 611 unsigned int diag_size = nla_total_size(0); 612 struct bpf_local_storage *sk_storage; 613 struct bpf_local_storage_data *sdata; 614 struct nlattr *nla_stgs; 615 unsigned int saved_len; 616 int err = 0; 617 u32 i; 618 619 *res_diag_size = 0; 620 621 /* No map has been specified. Dump all. */ 622 if (!diag->nr_maps) 623 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type, 624 res_diag_size); 625 626 rcu_read_lock(); 627 sk_storage = rcu_dereference(sk->sk_bpf_storage); 628 if (!sk_storage || hlist_empty(&sk_storage->list)) { 629 rcu_read_unlock(); 630 return 0; 631 } 632 633 nla_stgs = nla_nest_start(skb, stg_array_type); 634 if (!nla_stgs) 635 /* Continue to learn diag_size */ 636 err = -EMSGSIZE; 637 638 saved_len = skb->len; 639 for (i = 0; i < diag->nr_maps; i++) { 640 sdata = bpf_local_storage_lookup(sk_storage, 641 (struct bpf_local_storage_map *)diag->maps[i], 642 false); 643 644 if (!sdata) 645 continue; 646 647 diag_size += nla_value_size(diag->maps[i]->value_size); 648 649 if (nla_stgs && diag_get(sdata, skb)) 650 /* Continue to learn diag_size */ 651 err = -EMSGSIZE; 652 } 653 rcu_read_unlock(); 654 655 if (nla_stgs) { 656 if (saved_len == skb->len) 657 nla_nest_cancel(skb, nla_stgs); 658 else 659 nla_nest_end(skb, nla_stgs); 660 } 661 662 if (diag_size == nla_total_size(0)) { 663 *res_diag_size = 0; 664 return 0; 665 } 666 667 *res_diag_size = diag_size; 668 return err; 669 } 670 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put); 671 672 struct bpf_iter_seq_sk_storage_map_info { 673 struct bpf_map *map; 674 unsigned int bucket_id; 675 unsigned skip_elems; 676 }; 677 678 static struct bpf_local_storage_elem * 679 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, 680 struct bpf_local_storage_elem *prev_selem) 681 { 682 struct bpf_local_storage *sk_storage; 683 struct bpf_local_storage_elem *selem; 684 u32 skip_elems = info->skip_elems; 685 struct bpf_local_storage_map *smap; 686 u32 bucket_id = info->bucket_id; 687 u32 i, count, n_buckets; 688 struct bpf_local_storage_map_bucket *b; 689 690 smap = (struct bpf_local_storage_map *)info->map; 691 n_buckets = 1U << smap->bucket_log; 692 if (bucket_id >= n_buckets) 693 return NULL; 694 695 /* try to find next selem in the same bucket */ 696 selem = prev_selem; 697 count = 0; 698 while (selem) { 699 selem = hlist_entry_safe(selem->map_node.next, 700 struct bpf_local_storage_elem, map_node); 701 if (!selem) { 702 /* not found, unlock and go to the next bucket */ 703 b = &smap->buckets[bucket_id++]; 704 raw_spin_unlock_bh(&b->lock); 705 skip_elems = 0; 706 break; 707 } 708 sk_storage = rcu_dereference_raw(selem->local_storage); 709 if (sk_storage) { 710 info->skip_elems = skip_elems + count; 711 return selem; 712 } 713 count++; 714 } 715 716 for (i = bucket_id; i < (1U << smap->bucket_log); i++) { 717 b = &smap->buckets[i]; 718 raw_spin_lock_bh(&b->lock); 719 count = 0; 720 hlist_for_each_entry(selem, &b->list, map_node) { 721 sk_storage = rcu_dereference_raw(selem->local_storage); 722 if (sk_storage && count >= skip_elems) { 723 info->bucket_id = i; 724 info->skip_elems = count; 725 return selem; 726 } 727 count++; 728 } 729 raw_spin_unlock_bh(&b->lock); 730 skip_elems = 0; 731 } 732 733 info->bucket_id = i; 734 info->skip_elems = 0; 735 return NULL; 736 } 737 738 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos) 739 { 740 struct bpf_local_storage_elem *selem; 741 742 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL); 743 if (!selem) 744 return NULL; 745 746 if (*pos == 0) 747 ++*pos; 748 return selem; 749 } 750 751 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v, 752 loff_t *pos) 753 { 754 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 755 756 ++*pos; 757 ++info->skip_elems; 758 return bpf_sk_storage_map_seq_find_next(seq->private, v); 759 } 760 761 struct bpf_iter__bpf_sk_storage_map { 762 __bpf_md_ptr(struct bpf_iter_meta *, meta); 763 __bpf_md_ptr(struct bpf_map *, map); 764 __bpf_md_ptr(struct sock *, sk); 765 __bpf_md_ptr(void *, value); 766 }; 767 768 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, 769 struct bpf_map *map, struct sock *sk, 770 void *value) 771 772 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, 773 struct bpf_local_storage_elem *selem) 774 { 775 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 776 struct bpf_iter__bpf_sk_storage_map ctx = {}; 777 struct bpf_local_storage *sk_storage; 778 struct bpf_iter_meta meta; 779 struct bpf_prog *prog; 780 int ret = 0; 781 782 meta.seq = seq; 783 prog = bpf_iter_get_info(&meta, selem == NULL); 784 if (prog) { 785 ctx.meta = &meta; 786 ctx.map = info->map; 787 if (selem) { 788 sk_storage = rcu_dereference_raw(selem->local_storage); 789 ctx.sk = sk_storage->owner; 790 ctx.value = SDATA(selem)->data; 791 } 792 ret = bpf_iter_run_prog(prog, &ctx); 793 } 794 795 return ret; 796 } 797 798 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v) 799 { 800 return __bpf_sk_storage_map_seq_show(seq, v); 801 } 802 803 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) 804 { 805 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 806 struct bpf_local_storage_map *smap; 807 struct bpf_local_storage_map_bucket *b; 808 809 if (!v) { 810 (void)__bpf_sk_storage_map_seq_show(seq, v); 811 } else { 812 smap = (struct bpf_local_storage_map *)info->map; 813 b = &smap->buckets[info->bucket_id]; 814 raw_spin_unlock_bh(&b->lock); 815 } 816 } 817 818 static int bpf_iter_init_sk_storage_map(void *priv_data, 819 struct bpf_iter_aux_info *aux) 820 { 821 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; 822 823 seq_info->map = aux->map; 824 return 0; 825 } 826 827 static int bpf_iter_attach_map(struct bpf_prog *prog, 828 union bpf_iter_link_info *linfo, 829 struct bpf_iter_aux_info *aux) 830 { 831 struct bpf_map *map; 832 int err = -EINVAL; 833 834 if (!linfo->map.map_fd) 835 return -EBADF; 836 837 map = bpf_map_get_with_uref(linfo->map.map_fd); 838 if (IS_ERR(map)) 839 return PTR_ERR(map); 840 841 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 842 goto put_map; 843 844 if (prog->aux->max_rdonly_access > map->value_size) { 845 err = -EACCES; 846 goto put_map; 847 } 848 849 aux->map = map; 850 return 0; 851 852 put_map: 853 bpf_map_put_with_uref(map); 854 return err; 855 } 856 857 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) 858 { 859 bpf_map_put_with_uref(aux->map); 860 } 861 862 static const struct seq_operations bpf_sk_storage_map_seq_ops = { 863 .start = bpf_sk_storage_map_seq_start, 864 .next = bpf_sk_storage_map_seq_next, 865 .stop = bpf_sk_storage_map_seq_stop, 866 .show = bpf_sk_storage_map_seq_show, 867 }; 868 869 static const struct bpf_iter_seq_info iter_seq_info = { 870 .seq_ops = &bpf_sk_storage_map_seq_ops, 871 .init_seq_private = bpf_iter_init_sk_storage_map, 872 .fini_seq_private = NULL, 873 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info), 874 }; 875 876 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { 877 .target = "bpf_sk_storage_map", 878 .attach_target = bpf_iter_attach_map, 879 .detach_target = bpf_iter_detach_map, 880 .show_fdinfo = bpf_iter_map_show_fdinfo, 881 .fill_link_info = bpf_iter_map_fill_link_info, 882 .ctx_arg_info_size = 2, 883 .ctx_arg_info = { 884 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), 885 PTR_TO_BTF_ID_OR_NULL }, 886 { offsetof(struct bpf_iter__bpf_sk_storage_map, value), 887 PTR_TO_RDWR_BUF_OR_NULL }, 888 }, 889 .seq_info = &iter_seq_info, 890 }; 891 892 static int __init bpf_sk_storage_map_iter_init(void) 893 { 894 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id = 895 btf_sock_ids[BTF_SOCK_TYPE_SOCK]; 896 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info); 897 } 898 late_initcall(bpf_sk_storage_map_iter_init); 899