1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 #include <linux/rculist.h> 4 #include <linux/list.h> 5 #include <linux/hash.h> 6 #include <linux/types.h> 7 #include <linux/spinlock.h> 8 #include <linux/bpf.h> 9 #include <linux/btf.h> 10 #include <linux/btf_ids.h> 11 #include <linux/bpf_local_storage.h> 12 #include <net/bpf_sk_storage.h> 13 #include <net/sock.h> 14 #include <uapi/linux/sock_diag.h> 15 #include <uapi/linux/btf.h> 16 #include <linux/rcupdate_trace.h> 17 18 DEFINE_BPF_STORAGE_CACHE(sk_cache); 19 20 static struct bpf_local_storage_data * 21 bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit) 22 { 23 struct bpf_local_storage *sk_storage; 24 struct bpf_local_storage_map *smap; 25 26 sk_storage = 27 rcu_dereference_check(sk->sk_bpf_storage, bpf_rcu_lock_held()); 28 if (!sk_storage) 29 return NULL; 30 31 smap = (struct bpf_local_storage_map *)map; 32 return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit); 33 } 34 35 static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map) 36 { 37 struct bpf_local_storage_data *sdata; 38 39 sdata = bpf_sk_storage_lookup(sk, map, false); 40 if (!sdata) 41 return -ENOENT; 42 43 return bpf_selem_unlink(SELEM(sdata)); 44 } 45 46 /* Called by __sk_destruct() & bpf_sk_storage_clone() */ 47 void bpf_sk_storage_free(struct sock *sk) 48 { 49 struct bpf_local_storage *sk_storage; 50 u32 uncharge; 51 52 rcu_read_lock_dont_migrate(); 53 sk_storage = rcu_dereference(sk->sk_bpf_storage); 54 if (!sk_storage) 55 goto out; 56 57 uncharge = bpf_local_storage_destroy(sk_storage); 58 if (uncharge) 59 atomic_sub(uncharge, &sk->sk_omem_alloc); 60 out: 61 rcu_read_unlock_migrate(); 62 } 63 64 static void bpf_sk_storage_map_free(struct bpf_map *map) 65 { 66 bpf_local_storage_map_free(map, &sk_cache); 67 } 68 69 static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr) 70 { 71 return bpf_local_storage_map_alloc(attr, &sk_cache); 72 } 73 74 static int notsupp_get_next_key(struct bpf_map *map, void *key, 75 void *next_key) 76 { 77 return -ENOTSUPP; 78 } 79 80 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key) 81 { 82 struct bpf_local_storage_data *sdata; 83 struct socket *sock; 84 int fd, err; 85 86 fd = *(int *)key; 87 sock = sockfd_lookup(fd, &err); 88 if (sock) { 89 sdata = bpf_sk_storage_lookup(sock->sk, map, true); 90 sockfd_put(sock); 91 return sdata ? sdata->data : NULL; 92 } 93 94 return ERR_PTR(err); 95 } 96 97 static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, 98 void *value, u64 map_flags) 99 { 100 struct bpf_local_storage_data *sdata; 101 struct socket *sock; 102 int fd, err; 103 104 fd = *(int *)key; 105 sock = sockfd_lookup(fd, &err); 106 if (sock) { 107 sdata = bpf_local_storage_update( 108 sock->sk, (struct bpf_local_storage_map *)map, value, 109 map_flags, false); 110 sockfd_put(sock); 111 return PTR_ERR_OR_ZERO(sdata); 112 } 113 114 return err; 115 } 116 117 static long bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key) 118 { 119 struct socket *sock; 120 int fd, err; 121 122 fd = *(int *)key; 123 sock = sockfd_lookup(fd, &err); 124 if (sock) { 125 err = bpf_sk_storage_del(sock->sk, map); 126 sockfd_put(sock); 127 return err; 128 } 129 130 return err; 131 } 132 133 static struct bpf_local_storage_elem * 134 bpf_sk_storage_clone_elem(struct sock *newsk, 135 struct bpf_local_storage_map *smap, 136 struct bpf_local_storage_elem *selem) 137 { 138 struct bpf_local_storage_elem *copy_selem; 139 140 copy_selem = bpf_selem_alloc(smap, newsk, NULL, false); 141 if (!copy_selem) 142 return NULL; 143 144 if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)) 145 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data, 146 SDATA(selem)->data, true); 147 else 148 copy_map_value(&smap->map, SDATA(copy_selem)->data, 149 SDATA(selem)->data); 150 151 return copy_selem; 152 } 153 154 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) 155 { 156 struct bpf_local_storage *new_sk_storage = NULL; 157 struct bpf_local_storage *sk_storage; 158 struct bpf_local_storage_elem *selem; 159 int ret = 0; 160 161 rcu_read_lock_dont_migrate(); 162 sk_storage = rcu_dereference(sk->sk_bpf_storage); 163 164 if (!sk_storage || hlist_empty(&sk_storage->list)) 165 goto out; 166 167 /* 168 * The maps lead to newsk as soon as an element is linked, but 169 * sk_clone() sets sk_refcnt only later. Until then it holds whatever 170 * the slab object held before: no reference can be had yet. 171 */ 172 refcount_set(&newsk->sk_refcnt, 0); 173 174 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 175 struct bpf_local_storage_elem *copy_selem; 176 struct bpf_local_storage_map *smap; 177 struct bpf_map *map; 178 179 smap = rcu_dereference(SDATA(selem)->smap); 180 if (!smap || !(smap->map.map_flags & BPF_F_CLONE)) 181 continue; 182 183 /* Note that for lockless listeners adding new element 184 * here can race with cleanup in bpf_local_storage_map_free. 185 * Try to grab map refcnt to make sure that it's still 186 * alive and prevent concurrent removal. 187 */ 188 map = bpf_map_inc_not_zero(&smap->map); 189 if (IS_ERR(map)) 190 continue; 191 192 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem); 193 if (!copy_selem) { 194 ret = -ENOMEM; 195 bpf_map_put(map); 196 goto out; 197 } 198 199 if (new_sk_storage) { 200 ret = bpf_selem_link_map(smap, new_sk_storage, copy_selem); 201 if (ret) { 202 bpf_selem_free(copy_selem, true); 203 atomic_sub(smap->elem_size, 204 &newsk->sk_omem_alloc); 205 bpf_map_put(map); 206 goto out; 207 } 208 bpf_selem_link_storage_nolock(new_sk_storage, copy_selem); 209 } else { 210 ret = bpf_local_storage_alloc(newsk, smap, copy_selem); 211 if (ret) { 212 bpf_selem_free(copy_selem, true); 213 atomic_sub(smap->elem_size, 214 &newsk->sk_omem_alloc); 215 bpf_map_put(map); 216 goto out; 217 } 218 219 new_sk_storage = 220 rcu_dereference(copy_selem->local_storage); 221 } 222 bpf_map_put(map); 223 } 224 225 out: 226 rcu_read_unlock_migrate(); 227 228 /* In case of an error, don't free anything explicitly here, the 229 * caller is responsible to call bpf_sk_storage_free. 230 */ 231 232 return ret; 233 } 234 235 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, 236 void *, value, u64, flags) 237 { 238 struct bpf_local_storage_data *sdata; 239 240 WARN_ON_ONCE(!bpf_rcu_lock_held()); 241 if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE) 242 return (unsigned long)NULL; 243 244 sdata = bpf_sk_storage_lookup(sk, map, true); 245 if (sdata) 246 return (unsigned long)sdata->data; 247 248 if (flags == BPF_SK_STORAGE_GET_F_CREATE && 249 /* Cannot add new elem to a going away sk. 250 * Otherwise, the new elem may become a leak 251 * (and also other memory issues during map 252 * destruction). 253 */ 254 refcount_inc_not_zero(&sk->sk_refcnt)) { 255 sdata = bpf_local_storage_update( 256 sk, (struct bpf_local_storage_map *)map, value, 257 BPF_NOEXIST, false); 258 /* sk must be a fullsock (guaranteed by verifier), 259 * so sock_gen_put() is unnecessary. 260 */ 261 sock_put(sk); 262 return IS_ERR(sdata) ? 263 (unsigned long)NULL : (unsigned long)sdata->data; 264 } 265 266 return (unsigned long)NULL; 267 } 268 269 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) 270 { 271 WARN_ON_ONCE(!bpf_rcu_lock_held()); 272 if (!sk || !sk_fullsock(sk)) 273 return -EINVAL; 274 275 if (refcount_inc_not_zero(&sk->sk_refcnt)) { 276 int err; 277 278 err = bpf_sk_storage_del(sk, map); 279 sock_put(sk); 280 return err; 281 } 282 283 return -ENOENT; 284 } 285 286 static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap, 287 void *owner, u32 size) 288 { 289 struct sock *sk = (struct sock *)owner; 290 int optmem_max; 291 292 optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); 293 /* same check as in sock_kmalloc() */ 294 if (size <= optmem_max && 295 atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { 296 atomic_add(size, &sk->sk_omem_alloc); 297 return 0; 298 } 299 300 return -ENOMEM; 301 } 302 303 static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap, 304 void *owner, u32 size) 305 { 306 struct sock *sk = owner; 307 308 atomic_sub(size, &sk->sk_omem_alloc); 309 } 310 311 static struct bpf_local_storage __rcu ** 312 bpf_sk_storage_ptr(void *owner) 313 { 314 struct sock *sk = owner; 315 316 return &sk->sk_bpf_storage; 317 } 318 319 const struct bpf_map_ops sk_storage_map_ops = { 320 .map_meta_equal = bpf_map_meta_equal, 321 .map_alloc_check = bpf_local_storage_map_alloc_check, 322 .map_alloc = bpf_sk_storage_map_alloc, 323 .map_free = bpf_sk_storage_map_free, 324 .map_get_next_key = notsupp_get_next_key, 325 .map_lookup_elem = bpf_fd_sk_storage_lookup_elem, 326 .map_update_elem = bpf_fd_sk_storage_update_elem, 327 .map_delete_elem = bpf_fd_sk_storage_delete_elem, 328 .map_check_btf = bpf_local_storage_map_check_btf, 329 .map_btf_id = &bpf_local_storage_map_btf_id[0], 330 .map_local_storage_charge = bpf_sk_storage_charge, 331 .map_local_storage_uncharge = bpf_sk_storage_uncharge, 332 .map_owner_storage_ptr = bpf_sk_storage_ptr, 333 .map_mem_usage = bpf_local_storage_map_mem_usage, 334 }; 335 336 const struct bpf_func_proto bpf_sk_storage_get_proto = { 337 .func = bpf_sk_storage_get, 338 .gpl_only = false, 339 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 340 .arg1_type = ARG_CONST_MAP_PTR, 341 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, 342 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 343 .arg4_type = ARG_ANYTHING, 344 }; 345 346 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = { 347 .func = bpf_sk_storage_get, 348 .gpl_only = false, 349 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 350 .arg1_type = ARG_CONST_MAP_PTR, 351 .arg2_type = ARG_PTR_TO_CTX, /* context is 'struct sock' */ 352 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 353 .arg4_type = ARG_ANYTHING, 354 }; 355 356 const struct bpf_func_proto bpf_sk_storage_delete_proto = { 357 .func = bpf_sk_storage_delete, 358 .gpl_only = false, 359 .ret_type = RET_INTEGER, 360 .arg1_type = ARG_CONST_MAP_PTR, 361 .arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, 362 }; 363 364 static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog) 365 { 366 if (prog->aux->dst_prog) 367 return false; 368 369 /* Ensure the tracing program is not tracing 370 * any bpf_sk_storage*() function and also 371 * use the bpf_sk_storage_(get|delete) helper. 372 */ 373 switch (prog->expected_attach_type) { 374 case BPF_TRACE_ITER: 375 case BPF_TRACE_RAW_TP: 376 /* bpf_sk_storage has no trace point */ 377 return true; 378 case BPF_TRACE_FENTRY: 379 case BPF_TRACE_FEXIT: 380 case BPF_TRACE_FSESSION: 381 return !!strncmp(prog->aux->attach_func_name, "bpf_sk_storage", 382 strlen("bpf_sk_storage")); 383 default: 384 return false; 385 } 386 387 return false; 388 } 389 390 BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, 391 void *, value, u64, flags) 392 { 393 WARN_ON_ONCE(!bpf_rcu_lock_held()); 394 if (in_hardirq() || in_nmi()) 395 return (unsigned long)NULL; 396 397 return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags); 398 } 399 400 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map, 401 struct sock *, sk) 402 { 403 WARN_ON_ONCE(!bpf_rcu_lock_held()); 404 if (in_hardirq() || in_nmi()) 405 return -EPERM; 406 407 return ____bpf_sk_storage_delete(map, sk); 408 } 409 410 const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = { 411 .func = bpf_sk_storage_get_tracing, 412 .gpl_only = false, 413 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 414 .arg1_type = ARG_CONST_MAP_PTR, 415 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 416 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 417 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 418 .arg4_type = ARG_ANYTHING, 419 .allowed = bpf_sk_storage_tracing_allowed, 420 }; 421 422 const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = { 423 .func = bpf_sk_storage_delete_tracing, 424 .gpl_only = false, 425 .ret_type = RET_INTEGER, 426 .arg1_type = ARG_CONST_MAP_PTR, 427 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 428 .arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 429 .allowed = bpf_sk_storage_tracing_allowed, 430 }; 431 432 struct bpf_sk_storage_diag { 433 u32 nr_maps; 434 struct bpf_map *maps[]; 435 }; 436 437 /* The reply will be like: 438 * INET_DIAG_BPF_SK_STORAGES (nla_nest) 439 * SK_DIAG_BPF_STORAGE (nla_nest) 440 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 441 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 442 * SK_DIAG_BPF_STORAGE (nla_nest) 443 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 444 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 445 * .... 446 */ 447 static int nla_value_size(u32 value_size) 448 { 449 /* SK_DIAG_BPF_STORAGE (nla_nest) 450 * SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32) 451 * SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit) 452 */ 453 return nla_total_size(0) + nla_total_size(sizeof(u32)) + 454 nla_total_size_64bit(value_size); 455 } 456 457 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag) 458 { 459 u32 i; 460 461 if (!diag) 462 return; 463 464 for (i = 0; i < diag->nr_maps; i++) 465 bpf_map_put(diag->maps[i]); 466 467 kfree(diag); 468 } 469 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free); 470 471 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag, 472 const struct bpf_map *map) 473 { 474 u32 i; 475 476 for (i = 0; i < diag->nr_maps; i++) { 477 if (diag->maps[i] == map) 478 return true; 479 } 480 481 return false; 482 } 483 484 struct bpf_sk_storage_diag * 485 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs) 486 { 487 struct bpf_sk_storage_diag *diag; 488 struct nlattr *nla; 489 u32 nr_maps = 0; 490 int rem, err; 491 492 /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as 493 * the map_alloc_check() side also does. 494 */ 495 if (!bpf_capable()) 496 return ERR_PTR(-EPERM); 497 498 nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD, 499 nla_stgs, rem) { 500 if (nla_len(nla) != sizeof(u32)) 501 return ERR_PTR(-EINVAL); 502 nr_maps++; 503 } 504 505 diag = kzalloc_flex(*diag, maps, nr_maps); 506 if (!diag) 507 return ERR_PTR(-ENOMEM); 508 509 nla_for_each_nested_type(nla, SK_DIAG_BPF_STORAGE_REQ_MAP_FD, 510 nla_stgs, rem) { 511 int map_fd = nla_get_u32(nla); 512 struct bpf_map *map = bpf_map_get(map_fd); 513 514 if (IS_ERR(map)) { 515 err = PTR_ERR(map); 516 goto err_free; 517 } 518 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) { 519 bpf_map_put(map); 520 err = -EINVAL; 521 goto err_free; 522 } 523 if (diag_check_dup(diag, map)) { 524 bpf_map_put(map); 525 err = -EEXIST; 526 goto err_free; 527 } 528 diag->maps[diag->nr_maps++] = map; 529 } 530 531 return diag; 532 533 err_free: 534 bpf_sk_storage_diag_free(diag); 535 return ERR_PTR(err); 536 } 537 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc); 538 539 static int diag_get(struct bpf_local_storage_map *smap, 540 struct bpf_local_storage_data *sdata, struct sk_buff *skb) 541 { 542 struct nlattr *nla_stg, *nla_value; 543 544 /* It cannot exceed max nlattr's payload */ 545 BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE); 546 547 nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE); 548 if (!nla_stg) 549 return -EMSGSIZE; 550 551 if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id)) 552 goto errout; 553 554 nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE, 555 smap->map.value_size, 556 SK_DIAG_BPF_STORAGE_PAD); 557 if (!nla_value) 558 goto errout; 559 560 if (btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)) 561 copy_map_value_locked(&smap->map, nla_data(nla_value), 562 sdata->data, true); 563 else 564 copy_map_value(&smap->map, nla_data(nla_value), sdata->data); 565 check_and_init_map_value(&smap->map, nla_data(nla_value)); 566 567 nla_nest_end(skb, nla_stg); 568 return 0; 569 570 errout: 571 nla_nest_cancel(skb, nla_stg); 572 return -EMSGSIZE; 573 } 574 575 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb, 576 int stg_array_type, 577 unsigned int *res_diag_size) 578 { 579 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 580 unsigned int diag_size = nla_total_size(0); 581 struct bpf_local_storage *sk_storage; 582 struct bpf_local_storage_elem *selem; 583 struct bpf_local_storage_map *smap; 584 struct nlattr *nla_stgs; 585 unsigned int saved_len; 586 int err = 0; 587 588 rcu_read_lock(); 589 590 sk_storage = rcu_dereference(sk->sk_bpf_storage); 591 if (!sk_storage || hlist_empty(&sk_storage->list)) { 592 rcu_read_unlock(); 593 return 0; 594 } 595 596 nla_stgs = nla_nest_start(skb, stg_array_type); 597 if (!nla_stgs) 598 /* Continue to learn diag_size */ 599 err = -EMSGSIZE; 600 601 saved_len = skb->len; 602 hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { 603 smap = rcu_dereference(SDATA(selem)->smap); 604 if (!smap) 605 continue; 606 diag_size += nla_value_size(smap->map.value_size); 607 608 if (nla_stgs && diag_get(smap, SDATA(selem), skb)) 609 /* Continue to learn diag_size */ 610 err = -EMSGSIZE; 611 } 612 613 rcu_read_unlock(); 614 615 if (nla_stgs) { 616 if (saved_len == skb->len) 617 nla_nest_cancel(skb, nla_stgs); 618 else 619 nla_nest_end(skb, nla_stgs); 620 } 621 622 if (diag_size == nla_total_size(0)) { 623 *res_diag_size = 0; 624 return 0; 625 } 626 627 *res_diag_size = diag_size; 628 return err; 629 } 630 631 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag, 632 struct sock *sk, struct sk_buff *skb, 633 int stg_array_type, 634 unsigned int *res_diag_size) 635 { 636 /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */ 637 unsigned int diag_size = nla_total_size(0); 638 struct bpf_local_storage *sk_storage; 639 struct bpf_local_storage_data *sdata; 640 struct nlattr *nla_stgs; 641 unsigned int saved_len; 642 int err = 0; 643 u32 i; 644 645 *res_diag_size = 0; 646 647 /* No map has been specified. Dump all. */ 648 if (!diag->nr_maps) 649 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type, 650 res_diag_size); 651 652 rcu_read_lock(); 653 sk_storage = rcu_dereference(sk->sk_bpf_storage); 654 if (!sk_storage || hlist_empty(&sk_storage->list)) { 655 rcu_read_unlock(); 656 return 0; 657 } 658 659 nla_stgs = nla_nest_start(skb, stg_array_type); 660 if (!nla_stgs) 661 /* Continue to learn diag_size */ 662 err = -EMSGSIZE; 663 664 saved_len = skb->len; 665 for (i = 0; i < diag->nr_maps; i++) { 666 sdata = bpf_local_storage_lookup(sk_storage, 667 (struct bpf_local_storage_map *)diag->maps[i], 668 false); 669 670 if (!sdata) 671 continue; 672 673 diag_size += nla_value_size(diag->maps[i]->value_size); 674 675 if (nla_stgs && diag_get((struct bpf_local_storage_map *)diag->maps[i], sdata, skb)) 676 /* Continue to learn diag_size */ 677 err = -EMSGSIZE; 678 } 679 rcu_read_unlock(); 680 681 if (nla_stgs) { 682 if (saved_len == skb->len) 683 nla_nest_cancel(skb, nla_stgs); 684 else 685 nla_nest_end(skb, nla_stgs); 686 } 687 688 if (diag_size == nla_total_size(0)) { 689 *res_diag_size = 0; 690 return 0; 691 } 692 693 *res_diag_size = diag_size; 694 return err; 695 } 696 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put); 697 698 struct bpf_iter_seq_sk_storage_map_info { 699 struct bpf_map *map; 700 unsigned int bucket_id; 701 unsigned skip_elems; 702 struct rcu_head *dead_sks; 703 }; 704 705 /* 706 * sock_put() in two halves. The end of a socket may want process context: 707 * rfcomm_sock_destruct() takes a mutex. If the iterator drops the last 708 * reference, it keeps the socket until it has left the RCU read side critical 709 * section. Nobody uses sk_rcu before sk_destruct(). 710 */ 711 static void 712 bpf_sk_storage_map_seq_put_sk(struct bpf_iter_seq_sk_storage_map_info *info, 713 struct sock *sk) 714 { 715 if (refcount_dec_and_test(&sk->sk_refcnt)) { 716 sk->sk_rcu.next = info->dead_sks; 717 info->dead_sks = &sk->sk_rcu; 718 } 719 } 720 721 static void 722 bpf_sk_storage_map_seq_free_sks(struct bpf_iter_seq_sk_storage_map_info *info) 723 { 724 struct sock *sk; 725 726 while (info->dead_sks) { 727 sk = container_of(info->dead_sks, struct sock, sk_rcu); 728 info->dead_sks = sk->sk_rcu.next; 729 sk_free(sk); 730 } 731 } 732 733 static struct bpf_local_storage_elem * 734 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, 735 struct bpf_local_storage_elem *prev_selem) 736 __acquires(RCU) __releases(RCU) 737 { 738 struct bpf_local_storage *sk_storage; 739 struct bpf_local_storage_elem *selem; 740 u32 skip_elems = info->skip_elems; 741 struct bpf_local_storage_map *smap; 742 u32 bucket_id = info->bucket_id; 743 u32 i, count, n_buckets; 744 struct bpf_local_storage_map_bucket *b; 745 746 smap = (struct bpf_local_storage_map *)info->map; 747 n_buckets = 1U << smap->bucket_log; 748 if (bucket_id >= n_buckets) 749 return NULL; 750 751 /* try to find next selem in the same bucket */ 752 selem = prev_selem; 753 count = 0; 754 while (selem) { 755 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)), 756 struct bpf_local_storage_elem, map_node); 757 if (!selem) { 758 /* not found, unlock and go to the next bucket */ 759 b = &smap->buckets[bucket_id++]; 760 rcu_read_unlock(); 761 bpf_sk_storage_map_seq_free_sks(info); 762 skip_elems = 0; 763 break; 764 } 765 sk_storage = rcu_dereference(selem->local_storage); 766 if (sk_storage) { 767 info->skip_elems = skip_elems + count; 768 return selem; 769 } 770 count++; 771 } 772 773 for (i = bucket_id; i < (1U << smap->bucket_log); i++) { 774 b = &smap->buckets[i]; 775 rcu_read_lock(); 776 count = 0; 777 hlist_for_each_entry_rcu(selem, &b->list, map_node) { 778 sk_storage = rcu_dereference(selem->local_storage); 779 if (sk_storage && count >= skip_elems) { 780 info->bucket_id = i; 781 info->skip_elems = count; 782 return selem; 783 } 784 count++; 785 } 786 rcu_read_unlock(); 787 skip_elems = 0; 788 } 789 790 info->bucket_id = i; 791 info->skip_elems = 0; 792 return NULL; 793 } 794 795 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos) 796 { 797 struct bpf_local_storage_elem *selem; 798 799 selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL); 800 if (!selem) 801 return NULL; 802 803 if (*pos == 0) 804 ++*pos; 805 return selem; 806 } 807 808 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v, 809 loff_t *pos) 810 { 811 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 812 813 ++*pos; 814 ++info->skip_elems; 815 return bpf_sk_storage_map_seq_find_next(seq->private, v); 816 } 817 818 struct bpf_iter__bpf_sk_storage_map { 819 __bpf_md_ptr(struct bpf_iter_meta *, meta); 820 __bpf_md_ptr(struct bpf_map *, map); 821 __bpf_md_ptr(struct sock *, sk); 822 __bpf_md_ptr(void *, value); 823 }; 824 825 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, 826 struct bpf_map *map, struct sock *sk, 827 void *value) 828 829 /* 830 * Take a reference on the socket which owns @selem. RCU keeps the element, not 831 * its owner. As long as the element is linked under the lock of the storage, 832 * the owner has not got past bpf_sk_storage_free(), unless that has given up 833 * on the lock. Then it still waits for owner_refcnt. 834 */ 835 static struct sock * 836 bpf_sk_storage_selem_get_sk(struct bpf_local_storage_elem *selem) 837 { 838 struct bpf_local_storage *sk_storage; 839 struct sock *sk = NULL; 840 unsigned long flags; 841 842 sk_storage = rcu_dereference(selem->local_storage); 843 /* reset since ->next() looked: the socket goes away */ 844 if (!sk_storage) 845 return NULL; 846 847 if (raw_res_spin_lock_irqsave(&sk_storage->lock, flags)) 848 return NULL; 849 850 if (!hlist_unhashed(&selem->snode) && 851 refcount_inc_not_zero(&sk_storage->owner_refcnt)) { 852 sk = sk_storage->owner; 853 if (!refcount_inc_not_zero(&sk->sk_refcnt)) 854 sk = NULL; 855 bpf_local_storage_unpin_owner(sk_storage); 856 } 857 raw_res_spin_unlock_irqrestore(&sk_storage->lock, flags); 858 859 return sk; 860 } 861 862 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, 863 struct bpf_local_storage_elem *selem) 864 { 865 struct bpf_iter_seq_sk_storage_map_info *info = seq->private; 866 struct bpf_iter__bpf_sk_storage_map ctx = {}; 867 struct bpf_iter_meta meta; 868 struct bpf_prog *prog; 869 int ret = 0; 870 871 meta.seq = seq; 872 prog = bpf_iter_get_info(&meta, selem == NULL); 873 if (prog) { 874 ctx.meta = &meta; 875 ctx.map = info->map; 876 if (selem) { 877 ctx.sk = bpf_sk_storage_selem_get_sk(selem); 878 if (!ctx.sk) 879 return SEQ_SKIP; 880 ctx.value = SDATA(selem)->data; 881 } 882 ret = bpf_iter_run_prog(prog, &ctx); 883 if (ctx.sk) 884 bpf_sk_storage_map_seq_put_sk(info, ctx.sk); 885 } 886 887 return ret; 888 } 889 890 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v) 891 { 892 return __bpf_sk_storage_map_seq_show(seq, v); 893 } 894 895 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) 896 __releases(RCU) 897 { 898 if (!v) 899 (void)__bpf_sk_storage_map_seq_show(seq, v); 900 else 901 rcu_read_unlock(); 902 bpf_sk_storage_map_seq_free_sks(seq->private); 903 } 904 905 static int bpf_iter_init_sk_storage_map(void *priv_data, 906 struct bpf_iter_aux_info *aux) 907 { 908 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; 909 910 bpf_map_inc_with_uref(aux->map); 911 seq_info->map = aux->map; 912 return 0; 913 } 914 915 static void bpf_iter_fini_sk_storage_map(void *priv_data) 916 { 917 struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data; 918 919 bpf_map_put_with_uref(seq_info->map); 920 } 921 922 static int bpf_iter_map_check_prog(const struct bpf_prog *prog, 923 const struct bpf_iter_aux_info *aux) 924 { 925 if (prog->aux->max_rdwr_access > aux->map->value_size) 926 return -EACCES; 927 928 return bpf_iter_map_check_value_write(prog, aux->map); 929 } 930 931 static int bpf_iter_attach_map(struct bpf_prog *prog, 932 union bpf_iter_link_info *linfo, 933 struct bpf_iter_aux_info *aux) 934 { 935 struct bpf_map *map; 936 int err = -EINVAL; 937 938 if (!linfo->map.map_fd) 939 return -EBADF; 940 941 map = bpf_map_get_with_uref(linfo->map.map_fd); 942 if (IS_ERR(map)) 943 return PTR_ERR(map); 944 945 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 946 goto put_map; 947 948 aux->map = map; 949 err = bpf_iter_map_check_prog(prog, aux); 950 if (err) 951 goto put_map; 952 953 return 0; 954 955 put_map: 956 bpf_map_put_with_uref(map); 957 return err; 958 } 959 960 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) 961 { 962 bpf_map_put_with_uref(aux->map); 963 } 964 965 static const struct seq_operations bpf_sk_storage_map_seq_ops = { 966 .start = bpf_sk_storage_map_seq_start, 967 .next = bpf_sk_storage_map_seq_next, 968 .stop = bpf_sk_storage_map_seq_stop, 969 .show = bpf_sk_storage_map_seq_show, 970 }; 971 972 static const struct bpf_iter_seq_info iter_seq_info = { 973 .seq_ops = &bpf_sk_storage_map_seq_ops, 974 .init_seq_private = bpf_iter_init_sk_storage_map, 975 .fini_seq_private = bpf_iter_fini_sk_storage_map, 976 .seq_priv_size = sizeof(struct bpf_iter_seq_sk_storage_map_info), 977 }; 978 979 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { 980 .target = "bpf_sk_storage_map", 981 .attach_target = bpf_iter_attach_map, 982 .detach_target = bpf_iter_detach_map, 983 .check_prog = bpf_iter_map_check_prog, 984 .show_fdinfo = bpf_iter_map_show_fdinfo, 985 .fill_link_info = bpf_iter_map_fill_link_info, 986 .ctx_arg_info_size = 2, 987 .ctx_arg_info = { 988 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk), 989 PTR_TO_BTF_ID_OR_NULL }, 990 { offsetof(struct bpf_iter__bpf_sk_storage_map, value), 991 PTR_TO_BUF | PTR_MAYBE_NULL }, 992 }, 993 .seq_info = &iter_seq_info, 994 }; 995 996 static int __init bpf_sk_storage_map_iter_init(void) 997 { 998 bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id = 999 btf_sock_ids[BTF_SOCK_TYPE_SOCK]; 1000 return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info); 1001 } 1002 late_initcall(bpf_sk_storage_map_iter_init); 1003