1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/rculist.h> 4 #include <linux/list.h> 5 #include <linux/hash.h> 6 #include <linux/types.h> 7 #include <linux/spinlock.h> 8 #include <linux/bpf.h> 9 #include <linux/btf.h> 10 #include <linux/btf_ids.h> 11 #include <linux/bpf_local_storage.h> 12 #include <net/bpf_sk_storage.h> 13 #include <net/sock.h> 14 #include <uapi/linux/sock_diag.h> 15 #include <uapi/linux/btf.h> 16 #include <linux/rcupdate_trace.h> 17 18 DEFINE_BPF_STORAGE_CACHE(sk_cache); 19 20 static struct bpf_local_storage_data * 21 bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit) 22 { 23 struct bpf_local_storage *sk_storage; 24 struct bpf_local_storage_map *smap; 25 26 sk_storage = 27 rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held()); 28 if (!sk_storage) 29 return NULL; 30 31 smap = (struct bpf_local_storage_map *)map; 32 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit); 33 } 34 35 static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map) 36 { 37 struct bpf_local_storage_data *sdata; 38 39 sdata = bpf_sk_storage_lookup(sk, map, false); 40 if (!sdata) 41 return -ENOENT; 42 43 return bpf_selem_unlink(SELEM(sdata)); 44 } 45 46 /* Called by __sk_destruct() & bpf_sk_storage_clone() */ 47 void bpf_sk_storage_free(struct sock *sk) 48 { 49 struct bpf_local_storage *sk_storage; 50 u32 uncharge; 51 52 rcu_read_lock_dont_migrate(); 53 sk_storage = rcu_dereference(sk->sk_bpf_storage); 54 if (!sk_storage) 55 goto out; 56 57 uncharge = bpf_local_storage_destroy(sk_storage); 58 if (uncharge) 59 atomic_sub(uncharge, &sk->sk_omem_alloc); 60 out: 61 rcu_read_unlock_migrate(); 62 } 63 64 static void bpf_sk_storage_map_free(struct bpf_map *map) 65 { 66 bpf_local_storage_map_free(map, &sk_cache); 67 } 68 69 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr) 70 { 71 return bpf_local_storage_map_alloc(attr, &sk_cache); 72 } 73 74 static int notsupp_get_next_key(struct bpf_map *map, void *key, 75 void *next_key) 76 { 77 return -ENOTSUPP; 78 } 79 80 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key) 81 { 82 struct bpf_local_storage_data *sdata; 83 struct socket *sock; 84 int fd, err; 85 86 fd = *(int *)key; 87 sock = sockfd_lookup(fd, &err); 88 if (sock) { 89 sdata = bpf_sk_storage_lookup(sock->sk, map, true); 90 sockfd_put(sock); 91 return sdata ? sdata->data : NULL; 92 } 93 94 return ERR_PTR(err); 95 } 96 97 static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, 98 void *value, u64 map_flags) 99 { 100 struct bpf_local_storage_data *sdata; 101 struct socket *sock; 102 int fd, err; 103 104 fd = *(int *)key; 105 sock = sockfd_lookup(fd, &err); 106 if (sock) { 107 sdata = bpf_local_storage_update( 108 sock->sk, (struct bpf_local_storage_map *)map, value, 109 map_flags, false); 110 sockfd_put(sock); 111 return PTR_ERR_OR_ZERO(sdata); 112 } 113 114 return err; 115 } 116 117 static long bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) 118 { 119 struct socket *sock; 120 int fd, err; 121 122 fd = *(int *)key; 123 sock = sockfd_lookup(fd, &err); 124 if (sock) { 125 err = bpf_sk_storage_del(sock->sk, map); 126 sockfd_put(sock); 127 return err; 128 } 129 130 return err; 131 } 132 133 static struct bpf_local_storage_elem * 134 bpf_sk_storage_clone_elem(struct sock *newsk, 135 struct bpf_local_storage_map *smap, 136 struct bpf_local_storage_elem *selem) 137 { 138 struct bpf_local_storage_elem *copy_selem; 139 140 copy_selem = bpf_selem_alloc(smap, newsk, NULL, false); 141 if (!copy_selem) 142 return NULL; 143 144 if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)) 145 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data, 146 SDATA(selem)->data, true); 147 else 148 copy_map_value(&smap->map, SDATA(copy_selem)->data, 149 SDATA(selem)->data); 150 151 return copy_selem; 152 } 153 154 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) 155 { 156 struct bpf_local_storage *new_sk_storage = NULL; 157 struct bpf_local_storage *sk_storage; 158 struct bpf_local_storage_elem *selem; 159 int ret = 0; 160 161 rcu_read_lock_dont_migrate(); 162 sk_storage = rcu_dereference(sk->sk_bpf_storage); 163 164 if (!sk_storage || hlist_empty(&sk_storage->list)) 165 goto out; 166 167 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 168 struct bpf_local_storage_elem *copy_selem; 169 struct bpf_local_storage_map *smap; 170 struct bpf_map *map; 171 172 smap = rcu_dereference(SDATA(selem)->smap); 173 if (!smap || !(smap->map.map_flags & BPF_F_CLONE)) 174 continue; 175 176 /* Note that for lockless listeners adding new element 177 * here can race with cleanup in bpf_local_storage_map_free. 178 * Try to grab map refcnt to make sure that it's still 179 * alive and prevent concurrent removal. 180 */ 181 map = bpf_map_inc_not_zero(&smap->map); 182 if (IS_ERR(map)) 183 continue; 184 185 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem); 186 if (!copy_selem) { 187 ret = -ENOMEM; 188 bpf_map_put(map); 189 goto out; 190 } 191 192 if (new_sk_storage) { 193 ret = bpf_selem_link_map(smap, new_sk_storage, copy_selem); 194 if (ret) { 195 bpf_selem_free(copy_selem, true); 196 atomic_sub(smap->elem_size, 197 &newsk->sk_omem_alloc); 198 bpf_map_put(map); 199 goto out; 200 } 201 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); 202 } else { 203 ret = bpf_local_storage_alloc(newsk, smap, copy_selem); 204 if (ret) { 205 bpf_selem_free(copy_selem, true); 206 atomic_sub(smap->elem_size, 207 &newsk->sk_omem_alloc); 208 bpf_map_put(map); 209 goto out; 210 } 211 212 new_sk_storage = 213 rcu_dereference(copy_selem->local_storage); 214 } 215 bpf_map_put(map); 216 } 217 218 out: 219 rcu_read_unlock_migrate(); 220 221 /* In case of an error, don't free anything explicitly here, the 222 * caller is responsible to call bpf_sk_storage_free. 223 */ 224 225 return ret; 226 } 227 228 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, 229 void *, value, u64, flags) 230 { 231 struct bpf_local_storage_data *sdata; 232 233 WARN_ON_ONCE(!bpf_rcu_lock_held()); 234 if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE) 235 return (unsigned long)NULL; 236 237 sdata = bpf_sk_storage_lookup(sk, map, true); 238 if (sdata) 239 return (unsigned long)sdata->data; 240 241 if (flags == BPF_SK_STORAGE_GET_F_CREATE && 242 /* Cannot add new elem to a going away sk. 243 * Otherwise, the new elem may become a leak 244 * (and also other memory issues during map 245 * destruction). 246 */ 247 refcount_inc_not_zero(&sk->sk_refcnt)) { 248 sdata = bpf_local_storage_update( 249 sk, (struct bpf_local_storage_map *)map, value, 250 BPF_NOEXIST, false); 251 /* sk must be a fullsock (guaranteed by verifier), 252 * so sock_gen_put() is unnecessary. 253 */ 254 sock_put(sk); 255 return IS_ERR(sdata) ? 256 (unsigned long)NULL : (unsigned long)sdata->data; 257 } 258 259 return (unsigned long)NULL; 260 } 261 262 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) 263 { 264 WARN_ON_ONCE(!bpf_rcu_lock_held()); 265 if (!sk || !sk_fullsock(sk)) 266 return -EINVAL; 267 268 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 269 int err; 270 271 err = bpf_sk_storage_del(sk, map); 272 sock_put(sk); 273 return err; 274 } 275 276 return -ENOENT; 277 } 278 279 static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap, 280 void *owner, u32 size) 281 { 282 struct sock *sk = (struct sock *)owner; 283 int optmem_max; 284 285 optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 286 /* same check as in sock_kmalloc() */ 287 if (size <= optmem_max && 288 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 289 atomic_add(size, &sk->sk_omem_alloc); 290 return 0; 291 } 292 293 return -ENOMEM; 294 } 295 296 static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap, 297 void *owner, u32 size) 298 { 299 struct sock *sk = owner; 300 301 atomic_sub(size, &sk->sk_omem_alloc); 302 } 303 304 static struct bpf_local_storage __rcu ** 305 bpf_sk_storage_ptr(void *owner) 306 { 307 struct sock *sk = owner; 308 309 return &sk->sk_bpf_storage; 310 } 311 312 const struct bpf_map_ops sk_storage_map_ops = { 313 .map_meta_equal = bpf_map_meta_equal, 314 .map_alloc_check = bpf_local_storage_map_alloc_check, 315 .map_alloc = bpf_sk_storage_map_alloc, 316 .map_free = bpf_sk_storage_map_free, 317 .map_get_next_key = notsupp_get_next_key, 318 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem, 319 .map_update_elem = bpf_fd_sk_storage_update_elem, 320 .map_delete_elem = bpf_fd_sk_storage_delete_elem, 321 .map_check_btf = bpf_local_storage_map_check_btf, 322 .map_btf_id = &bpf_local_storage_map_btf_id[0], 323 .map_local_storage_charge = bpf_sk_storage_charge, 324 .map_local_storage_uncharge = bpf_sk_storage_uncharge, 325 .map_owner_storage_ptr = bpf_sk_storage_ptr, 326 .map_mem_usage = bpf_local_storage_map_mem_usage, 327 }; 328 329 const struct bpf_func_proto bpf_sk_storage_get_proto = { 330 .func = bpf_sk_storage_get, 331 .gpl_only = false, 332 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 333 .arg1_type = ARG_CONST_MAP_PTR, 334 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, 335 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 336 .arg4_type = ARG_ANYTHING, 337 }; 338 339 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = { 340 .func = bpf_sk_storage_get, 341 .gpl_only = false, 342 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 343 .arg1_type = ARG_CONST_MAP_PTR, 344 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */ 345 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 346 .arg4_type = ARG_ANYTHING, 347 }; 348 349 const struct bpf_func_proto bpf_sk_storage_delete_proto = { 350 .func = bpf_sk_storage_delete, 351 .gpl_only = false, 352 .ret_type = RET_INTEGER, 353 .arg1_type = ARG_CONST_MAP_PTR, 354 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, 355 }; 356 357 static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog) 358 { 359 if (prog->aux->dst_prog) 360 return false; 361 362 /* Ensure the tracing program is not tracing 363 * any bpf_sk_storage*() function and also 364 * use the bpf_sk_storage_(get|delete) helper. 365 */ 366 switch (prog->expected_attach_type) { 367 case BPF_TRACE_ITER: 368 case BPF_TRACE_RAW_TP: 369 /* bpf_sk_storage has no trace point */ 370 return true; 371 case BPF_TRACE_FENTRY: 372 case BPF_TRACE_FEXIT: 373 case BPF_TRACE_FSESSION: 374 return !!strncmp(prog->aux->attach_func_name, "bpf_sk_storage", 375 strlen("bpf_sk_storage")); 376 default: 377 return false; 378 } 379 380 return false; 381 } 382 383 BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, 384 void *, value, u64, flags) 385 { 386 WARN_ON_ONCE(!bpf_rcu_lock_held()); 387 if (in_hardirq() || in_nmi()) 388 return (unsigned long)NULL; 389 390 return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags); 391 } 392 393 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map, 394 struct sock *, sk) 395 { 396 WARN_ON_ONCE(!bpf_rcu_lock_held()); 397 if (in_hardirq() || in_nmi()) 398 return -EPERM; 399 400 return ____bpf_sk_storage_delete(map, sk); 401 } 402 403 const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = { 404 .func = bpf_sk_storage_get_tracing, 405 .gpl_only = false, 406 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 407 .arg1_type = ARG_CONST_MAP_PTR, 408 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 409 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 410 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 411 .arg4_type = ARG_ANYTHING, 412 .allowed = bpf_sk_storage_tracing_allowed, 413 }; 414 415 const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = { 416 .func = bpf_sk_storage_delete_tracing, 417 .gpl_only = false, 418 .ret_type = RET_INTEGER, 419 .arg1_type = ARG_CONST_MAP_PTR, 420 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 421 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 422 .allowed = bpf_sk_storage_tracing_allowed, 423 }; 424 425 struct bpf_sk_storage_diag { 426 u32 nr_maps; 427 struct bpf_map *maps[]; 428 }; 429 430 /* The reply will be like: 431 * INET_DIAG_BPF_SK_STORAGES (nla_nest) 432 * SK_DIAG_BPF_STORAGE (nla_nest) 433 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 434 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 435 * SK_DIAG_BPF_STORAGE (nla_nest) 436 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 437 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 438 * .... 439 */ 440 static int nla_value_size(u32 value_size) 441 { 442 /* SK_DIAG_BPF_STORAGE (nla_nest) 443 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 444 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 445 */ 446 return nla_total_size(0) + nla_total_size(sizeof(u32)) + 447 nla_total_size_64bit(value_size); 448 } 449 450 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag) 451 { 452 u32 i; 453 454 if (!diag) 455 return; 456 457 for (i = 0; i < diag->nr_maps; i++) 458 bpf_map_put(diag->maps[i]); 459 460 kfree(diag); 461 } 462 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free); 463 464 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag, 465 const struct bpf_map *map) 466 { 467 u32 i; 468 469 for (i = 0; i < diag->nr_maps; i++) { 470 if (diag->maps[i] == map) 471 return true; 472 } 473 474 return false; 475 } 476 477 struct bpf_sk_storage_diag * 478 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs) 479 { 480 struct bpf_sk_storage_diag *diag; 481 struct nlattr *nla; 482 u32 nr_maps = 0; 483 int rem, err; 484 485 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as 486 * the map_alloc_check() side also does. 487 */ 488 if (!bpf_capable()) 489 return ERR_PTR(-EPERM); 490 491 nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD, 492 nla_stgs, rem) { 493 if (nla_len(nla) != sizeof(u32)) 494 return ERR_PTR(-EINVAL); 495 nr_maps++; 496 } 497 498 diag = kzalloc_flex(*diag, maps, nr_maps); 499 if (!diag) 500 return ERR_PTR(-ENOMEM); 501 502 nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD, 503 nla_stgs, rem) { 504 int map_fd = nla_get_u32(nla); 505 struct bpf_map *map = bpf_map_get(map_fd); 506 507 if (IS_ERR(map)) { 508 err = PTR_ERR(map); 509 goto err_free; 510 } 511 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) { 512 bpf_map_put(map); 513 err = -EINVAL; 514 goto err_free; 515 } 516 if (diag_check_dup(diag, map)) { 517 bpf_map_put(map); 518 err = -EEXIST; 519 goto err_free; 520 } 521 diag->maps[diag->nr_maps++] = map; 522 } 523 524 return diag; 525 526 err_free: 527 bpf_sk_storage_diag_free(diag); 528 return ERR_PTR(err); 529 } 530 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc); 531 532 static int diag_get(struct bpf_local_storage_map *smap, 533 struct bpf_local_storage_data *sdata, struct sk_buff *skb) 534 { 535 struct nlattr *nla_stg, *nla_value; 536 537 /* It cannot exceed max nlattr's payload */ 538 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE); 539 540 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE); 541 if (!nla_stg) 542 return -EMSGSIZE; 543 544 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id)) 545 goto errout; 546 547 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE, 548 smap->map.value_size, 549 SK_DIAG_BPF_STORAGE_PAD); 550 if (!nla_value) 551 goto errout; 552 553 if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)) 554 copy_map_value_locked(&smap->map, nla_data(nla_value), 555 sdata->data, true); 556 else 557 copy_map_value(&smap->map, nla_data(nla_value), sdata->data); 558 check_and_init_map_value(&smap->map, nla_data(nla_value)); 559 560 nla_nest_end(skb, nla_stg); 561 return 0; 562 563 errout: 564 nla_nest_cancel(skb, nla_stg); 565 return -EMSGSIZE; 566 } 567 568 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb, 569 int stg_array_type, 570 unsigned int *res_diag_size) 571 { 572 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 573 unsigned int diag_size = nla_total_size(0); 574 struct bpf_local_storage *sk_storage; 575 struct bpf_local_storage_elem *selem; 576 struct bpf_local_storage_map *smap; 577 struct nlattr *nla_stgs; 578 unsigned int saved_len; 579 int err = 0; 580 581 rcu_read_lock(); 582 583 sk_storage = rcu_dereference(sk->sk_bpf_storage); 584 if (!sk_storage || hlist_empty(&sk_storage->list)) { 585 rcu_read_unlock(); 586 return 0; 587 } 588 589 nla_stgs = nla_nest_start(skb, stg_array_type); 590 if (!nla_stgs) 591 /* Continue to learn diag_size */ 592 err = -EMSGSIZE; 593 594 saved_len = skb->len; 595 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 596 smap = rcu_dereference(SDATA(selem)->smap); 597 if (!smap) 598 continue; 599 diag_size += nla_value_size(smap->map.value_size); 600 601 if (nla_stgs && diag_get(smap, SDATA(selem), skb)) 602 /* Continue to learn diag_size */ 603 err = -EMSGSIZE; 604 } 605 606 rcu_read_unlock(); 607 608 if (nla_stgs) { 609 if (saved_len == skb->len) 610 nla_nest_cancel(skb, nla_stgs); 611 else 612 nla_nest_end(skb, nla_stgs); 613 } 614 615 if (diag_size == nla_total_size(0)) { 616 *res_diag_size = 0; 617 return 0; 618 } 619 620 *res_diag_size = diag_size; 621 return err; 622 } 623 624 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag, 625 struct sock *sk, struct sk_buff *skb, 626 int stg_array_type, 627 unsigned int *res_diag_size) 628 { 629 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 630 unsigned int diag_size = nla_total_size(0); 631 struct bpf_local_storage *sk_storage; 632 struct bpf_local_storage_data *sdata; 633 struct nlattr *nla_stgs; 634 unsigned int saved_len; 635 int err = 0; 636 u32 i; 637 638 *res_diag_size = 0; 639 640 /* No map has been specified. Dump all. */ 641 if (!diag->nr_maps) 642 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type, 643 res_diag_size); 644 645 rcu_read_lock(); 646 sk_storage = rcu_dereference(sk->sk_bpf_storage); 647 if (!sk_storage || hlist_empty(&sk_storage->list)) { 648 rcu_read_unlock(); 649 return 0; 650 } 651 652 nla_stgs = nla_nest_start(skb, stg_array_type); 653 if (!nla_stgs) 654 /* Continue to learn diag_size */ 655 err = -EMSGSIZE; 656 657 saved_len = skb->len; 658 for (i = 0; i < diag->nr_maps; i++) { 659 sdata = bpf_local_storage_lookup(sk_storage, 660 (struct bpf_local_storage_map *)diag->maps[i], 661 false); 662 663 if (!sdata) 664 continue; 665 666 diag_size += nla_value_size(diag->maps[i]->value_size); 667 668 if (nla_stgs && diag_get((struct bpf_local_storage_map *)diag->maps[i], sdata, skb)) 669 /* Continue to learn diag_size */ 670 err = -EMSGSIZE; 671 } 672 rcu_read_unlock(); 673 674 if (nla_stgs) { 675 if (saved_len == skb->len) 676 nla_nest_cancel(skb, nla_stgs); 677 else 678 nla_nest_end(skb, nla_stgs); 679 } 680 681 if (diag_size == nla_total_size(0)) { 682 *res_diag_size = 0; 683 return 0; 684 } 685 686 *res_diag_size = diag_size; 687 return err; 688 } 689 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put); 690 691 struct bpf_iter_seq_sk_storage_map_info { 692 struct bpf_map *map; 693 unsigned int bucket_id; 694 unsigned skip_elems; 695 }; 696 697 static struct bpf_local_storage_elem * 698 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, 699 struct bpf_local_storage_elem *prev_selem) 700 __acquires(RCU) __releases(RCU) 701 { 702 struct bpf_local_storage *sk_storage; 703 struct bpf_local_storage_elem *selem; 704 u32 skip_elems = info->skip_elems; 705 struct bpf_local_storage_map *smap; 706 u32 bucket_id = info->bucket_id; 707 u32 i, count, n_buckets; 708 struct bpf_local_storage_map_bucket *b; 709 710 smap = (struct bpf_local_storage_map *)info->map; 711 n_buckets = 1U << smap->bucket_log; 712 if (bucket_id >= n_buckets) 713 return NULL; 714 715 /* try to find next selem in the same bucket */ 716 selem = prev_selem; 717 count = 0; 718 while (selem) { 719 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)), 720 struct bpf_local_storage_elem, map_node); 721 if (!selem) { 722 /* not found, unlock and go to the next bucket */ 723 b = &smap->buckets[bucket_id++]; 724 rcu_read_unlock(); 725 skip_elems = 0; 726 break; 727 } 728 sk_storage = rcu_dereference(selem->local_storage); 729 if (sk_storage) { 730 info->skip_elems = skip_elems + count; 731 return selem; 732 } 733 count++; 734 } 735 736 for (i = bucket_id; i < (1U << smap->bucket_log); i++) { 737 b = &smap->buckets[i]; 738 rcu_read_lock(); 739 count = 0; 740 hlist_for_each_entry_rcu(selem, &b->list, map_node) { 741 sk_storage = rcu_dereference(selem->local_storage); 742 if (sk_storage && count >= skip_elems) { 743 info->bucket_id = i; 744 info->skip_elems = count; 745 return selem; 746 } 747 count++; 748 } 749 rcu_read_unlock(); 750 skip_elems = 0; 751 } 752 753 info->bucket_id = i; 754 info->skip_elems = 0; 755 return NULL; 756 } 757 758 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos) 759 { 760 struct bpf_local_storage_elem *selem; 761 762 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL); 763 if (!selem) 764 return NULL; 765 766 if (*pos == 0) 767 ++*pos; 768 return selem; 769 } 770 771 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v, 772 loff_t *pos) 773 { 774 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 775 776 ++*pos; 777 ++info->skip_elems; 778 return bpf_sk_storage_map_seq_find_next(seq->private, v); 779 } 780 781 struct bpf_iter__bpf_sk_storage_map { 782 __bpf_md_ptr(struct bpf_iter_meta *, meta); 783 __bpf_md_ptr(struct bpf_map *, map); 784 __bpf_md_ptr(struct sock *, sk); 785 __bpf_md_ptr(void *, value); 786 }; 787 788 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, 789 struct bpf_map *map, struct sock *sk, 790 void *value) 791 792 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, 793 struct bpf_local_storage_elem *selem) 794 { 795 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 796 struct bpf_iter__bpf_sk_storage_map ctx = {}; 797 struct bpf_local_storage *sk_storage; 798 struct bpf_iter_meta meta; 799 struct bpf_prog *prog; 800 int ret = 0; 801 802 meta.seq = seq; 803 prog = bpf_iter_get_info(&meta, selem == NULL); 804 if (prog) { 805 ctx.meta = &meta; 806 ctx.map = info->map; 807 if (selem) { 808 sk_storage = rcu_dereference(selem->local_storage); 809 ctx.sk = sk_storage->owner; 810 ctx.value = SDATA(selem)->data; 811 } 812 ret = bpf_iter_run_prog(prog, &ctx); 813 } 814 815 return ret; 816 } 817 818 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v) 819 { 820 return __bpf_sk_storage_map_seq_show(seq, v); 821 } 822 823 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) 824 __releases(RCU) 825 { 826 if (!v) 827 (void)__bpf_sk_storage_map_seq_show(seq, v); 828 else 829 rcu_read_unlock(); 830 } 831 832 static int bpf_iter_init_sk_storage_map(void *priv_data, 833 struct bpf_iter_aux_info *aux) 834 { 835 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; 836 837 bpf_map_inc_with_uref(aux->map); 838 seq_info->map = aux->map; 839 return 0; 840 } 841 842 static void bpf_iter_fini_sk_storage_map(void *priv_data) 843 { 844 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; 845 846 bpf_map_put_with_uref(seq_info->map); 847 } 848 849 static int bpf_iter_attach_map(struct bpf_prog *prog, 850 union bpf_iter_link_info *linfo, 851 struct bpf_iter_aux_info *aux) 852 { 853 struct bpf_map *map; 854 int err = -EINVAL; 855 856 if (!linfo->map.map_fd) 857 return -EBADF; 858 859 map = bpf_map_get_with_uref(linfo->map.map_fd); 860 if (IS_ERR(map)) 861 return PTR_ERR(map); 862 863 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 864 goto put_map; 865 866 if (prog->aux->max_rdwr_access > map->value_size) { 867 err = -EACCES; 868 goto put_map; 869 } 870 871 aux->map = map; 872 return 0; 873 874 put_map: 875 bpf_map_put_with_uref(map); 876 return err; 877 } 878 879 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) 880 { 881 bpf_map_put_with_uref(aux->map); 882 } 883 884 static const struct seq_operations bpf_sk_storage_map_seq_ops = { 885 .start = bpf_sk_storage_map_seq_start, 886 .next = bpf_sk_storage_map_seq_next, 887 .stop = bpf_sk_storage_map_seq_stop, 888 .show = bpf_sk_storage_map_seq_show, 889 }; 890 891 static const struct bpf_iter_seq_info iter_seq_info = { 892 .seq_ops = &bpf_sk_storage_map_seq_ops, 893 .init_seq_private = bpf_iter_init_sk_storage_map, 894 .fini_seq_private = bpf_iter_fini_sk_storage_map, 895 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info), 896 }; 897 898 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { 899 .target = "bpf_sk_storage_map", 900 .attach_target = bpf_iter_attach_map, 901 .detach_target = bpf_iter_detach_map, 902 .show_fdinfo = bpf_iter_map_show_fdinfo, 903 .fill_link_info = bpf_iter_map_fill_link_info, 904 .ctx_arg_info_size = 2, 905 .ctx_arg_info = { 906 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), 907 PTR_TO_BTF_ID_OR_NULL }, 908 { offsetof(struct bpf_iter__bpf_sk_storage_map, value), 909 PTR_TO_BUF | PTR_MAYBE_NULL }, 910 }, 911 .seq_info = &iter_seq_info, 912 }; 913 914 static int __init bpf_sk_storage_map_iter_init(void) 915 { 916 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id = 917 btf_sock_ids[BTF_SOCK_TYPE_SOCK]; 918 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info); 919 } 920 late_initcall(bpf_sk_storage_map_iter_init); 921