Lines Matching refs:storage
78 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup() local
80 storage = container_of(node, struct bpf_cgroup_storage, node); in cgroup_storage_lookup()
82 switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) { in cgroup_storage_lookup()
92 return storage; in cgroup_storage_lookup()
103 struct bpf_cgroup_storage *storage) in cgroup_storage_insert() argument
114 switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) { in cgroup_storage_insert()
126 rb_link_node(&storage->node, parent, new); in cgroup_storage_insert()
127 rb_insert_color(&storage->node, root); in cgroup_storage_insert()
135 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup_elem() local
137 storage = cgroup_storage_lookup(map, key, false); in cgroup_storage_lookup_elem()
138 if (!storage) in cgroup_storage_lookup_elem()
141 return &READ_ONCE(storage->buf)->data[0]; in cgroup_storage_lookup_elem()
147 struct bpf_cgroup_storage *storage; in cgroup_storage_update_elem() local
157 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, in cgroup_storage_update_elem()
159 if (!storage) in cgroup_storage_update_elem()
163 copy_map_value_locked(map, storage->buf->data, value, false); in cgroup_storage_update_elem()
176 new = xchg(&storage->buf, new); in cgroup_storage_update_elem()
186 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_copy() local
191 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_copy()
192 if (!storage) { in bpf_percpu_cgroup_storage_copy()
203 copy_map_value(_map, value, per_cpu_ptr(storage->percpu_buf, cpu)); in bpf_percpu_cgroup_storage_copy()
208 copy_map_value_long(_map, value + off, per_cpu_ptr(storage->percpu_buf, cpu)); in bpf_percpu_cgroup_storage_copy()
220 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_update() local
229 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_update()
230 if (!storage) { in bpf_percpu_cgroup_storage_update()
243 copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), value); in bpf_percpu_cgroup_storage_update()
249 copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val); in bpf_percpu_cgroup_storage_update()
260 struct bpf_cgroup_storage *storage; in cgroup_storage_get_next_key() local
268 storage = cgroup_storage_lookup(map, key, true); in cgroup_storage_get_next_key()
269 if (!storage) in cgroup_storage_get_next_key()
272 storage = list_next_entry(storage, list_map); in cgroup_storage_get_next_key()
273 if (!storage) in cgroup_storage_get_next_key()
276 storage = list_first_entry(&map->list, in cgroup_storage_get_next_key()
284 *next = storage->key; in cgroup_storage_get_next_key()
287 *next = storage->key.cgroup_inode_id; in cgroup_storage_get_next_key()
345 struct bpf_cgroup_storage *storage, *stmp; in cgroup_storage_map_free() local
349 list_for_each_entry_safe(storage, stmp, storages, list_map) { in cgroup_storage_map_free()
350 bpf_cgroup_storage_unlink(storage); in cgroup_storage_map_free()
351 bpf_cgroup_storage_free(storage); in cgroup_storage_map_free()
422 struct bpf_cgroup_storage *storage; in cgroup_storage_seq_show_elem() local
426 storage = cgroup_storage_lookup(map_to_storage(map), key, false); in cgroup_storage_seq_show_elem()
427 if (!storage) { in cgroup_storage_seq_show_elem()
437 &READ_ONCE(storage->buf)->data[0], m); in cgroup_storage_seq_show_elem()
444 per_cpu_ptr(storage->percpu_buf, cpu), in cgroup_storage_seq_show_elem()
507 struct bpf_cgroup_storage *storage; in bpf_cgroup_storage_alloc() local
518 storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage), in bpf_cgroup_storage_alloc()
520 if (!storage) in bpf_cgroup_storage_alloc()
524 storage->buf = bpf_map_kmalloc_node(map, size, gfp, in bpf_cgroup_storage_alloc()
526 if (!storage->buf) in bpf_cgroup_storage_alloc()
528 check_and_init_map_value(map, storage->buf->data); in bpf_cgroup_storage_alloc()
530 storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp); in bpf_cgroup_storage_alloc()
531 if (!storage->percpu_buf) in bpf_cgroup_storage_alloc()
535 storage->map = (struct bpf_cgroup_storage_map *)map; in bpf_cgroup_storage_alloc()
537 return storage; in bpf_cgroup_storage_alloc()
540 kfree(storage); in bpf_cgroup_storage_alloc()
546 struct bpf_cgroup_storage *storage = in free_shared_cgroup_storage_rcu() local
549 kfree(storage->buf); in free_shared_cgroup_storage_rcu()
550 kfree(storage); in free_shared_cgroup_storage_rcu()
555 struct bpf_cgroup_storage *storage = in free_percpu_cgroup_storage_rcu() local
558 free_percpu(storage->percpu_buf); in free_percpu_cgroup_storage_rcu()
559 kfree(storage); in free_percpu_cgroup_storage_rcu()
562 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_free() argument
567 if (!storage) in bpf_cgroup_storage_free()
570 map = &storage->map->map; in bpf_cgroup_storage_free()
573 call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); in bpf_cgroup_storage_free()
575 call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); in bpf_cgroup_storage_free()
578 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, in bpf_cgroup_storage_link() argument
584 if (!storage) in bpf_cgroup_storage_link()
587 storage->key.attach_type = type; in bpf_cgroup_storage_link()
588 storage->key.cgroup_inode_id = cgroup_id(cgroup); in bpf_cgroup_storage_link()
590 map = storage->map; in bpf_cgroup_storage_link()
593 WARN_ON(cgroup_storage_insert(map, storage)); in bpf_cgroup_storage_link()
594 list_add(&storage->list_map, &map->list); in bpf_cgroup_storage_link()
595 list_add(&storage->list_cg, &cgroup->bpf.storages); in bpf_cgroup_storage_link()
599 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_unlink() argument
604 if (!storage) in bpf_cgroup_storage_unlink()
607 map = storage->map; in bpf_cgroup_storage_unlink()
611 rb_erase(&storage->node, root); in bpf_cgroup_storage_unlink()
613 list_del(&storage->list_map); in bpf_cgroup_storage_unlink()
614 list_del(&storage->list_cg); in bpf_cgroup_storage_unlink()