54 static bool selem_linked_to_storage_lockless(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage_lockless()  argument
56 	return !hlist_unhashed_lockless(&selem->snode);  in selem_linked_to_storage_lockless()
59 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage()  argument
61 	return !hlist_unhashed(&selem->snode);  in selem_linked_to_storage()
64 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map()  argument
66 	return !hlist_unhashed(&selem->map_node);  in selem_linked_to_map()
73 	struct bpf_local_storage_elem *selem;  in bpf_selem_alloc()  local
78 	selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size,  in bpf_selem_alloc()
81 	if (selem) {  in bpf_selem_alloc()
82 		RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_alloc()
83 		atomic_set(&selem->state, 0);  in bpf_selem_alloc()
87 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
89 				bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);  in bpf_selem_alloc()
91 		return selem;  in bpf_selem_alloc()
128 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_trace_rcu()  local
131 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in bpf_selem_free_trace_rcu()
133 	smap = rcu_dereference_check(SDATA(selem)->smap, 1);  in bpf_selem_free_trace_rcu()
136 		bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free_trace_rcu()
141 	kfree(selem);  in bpf_selem_free_trace_rcu()
144 void bpf_selem_free(struct bpf_local_storage_elem *selem,  in bpf_selem_free()  argument
149 	smap = rcu_dereference_check(SDATA(selem)->smap, 1);  in bpf_selem_free()
153 			bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free()
154 		kfree_rcu(selem, rcu);  in bpf_selem_free()
158 	call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);  in bpf_selem_free()
163 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_list()  local
171 	hlist_for_each_entry_safe(selem, n, list, free_node)  in bpf_selem_free_list()
172 		bpf_selem_free(selem, reuse_now);  in bpf_selem_free_list()
175 static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock_misc()  argument
184 	    SDATA(selem))  in bpf_selem_unlink_storage_nolock_misc()
210 					    struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock()  argument
216 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
218 	free_local_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_storage_nolock()
221 	bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage,  in bpf_selem_unlink_storage_nolock()
224 	hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_storage_nolock()
226 	hlist_add_head(&selem->free_node, free_selem_list);  in bpf_selem_unlink_storage_nolock()
232 				   struct bpf_local_storage_elem *selem)  in bpf_selem_link_storage_nolock()  argument
236 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_link_storage_nolock()
239 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
240 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
243 static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map()  argument
251 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_map()
253 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
259 	hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map()
265 static void bpf_selem_unlink_map_nolock(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map_nolock()  argument
267 	hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map_nolock()
272 		       struct bpf_local_storage_elem *selem)  in bpf_selem_link_map()  argument
284 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map()
291 				      struct bpf_local_storage_elem *selem)  in bpf_selem_link_map_nolock()  argument
293 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map_nolock()
300 int bpf_selem_unlink(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink()  argument
311 	if (unlikely(!selem_linked_to_storage_lockless(selem)))  in bpf_selem_unlink()
315 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink()
322 	if (likely(selem_linked_to_storage(selem))) {  in bpf_selem_unlink()
327 		err = bpf_selem_unlink_map(selem);  in bpf_selem_unlink()
332 			local_storage, selem, &selem_free_list);  in bpf_selem_unlink()
350 static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_nofail()  argument
359 	local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held());  in bpf_selem_unlink_nofail()
360 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_nofail()
371 			if (likely(selem_linked_to_map(selem))) {  in bpf_selem_unlink_nofail()
372 				hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_nofail()
373 				bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_unlink_nofail()
389 			RCU_INIT_POINTER(SDATA(selem)->smap, NULL);  in bpf_selem_unlink_nofail()
395 			if (likely(selem_linked_to_storage(selem))) {  in bpf_selem_unlink_nofail()
396 				free_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_nofail()
405 						selem, smap, local_storage,  in bpf_selem_unlink_nofail()
407 				hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_nofail()
421 			RCU_INIT_POINTER(selem->local_storage, NULL);  in bpf_selem_unlink_nofail()
425 		atomic_or(in_map_free ? SELEM_MAP_UNLINKED : SELEM_STORAGE_UNLINKED, &selem->state);  in bpf_selem_unlink_nofail()
434 	    atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED)  in bpf_selem_unlink_nofail()
435 		bpf_selem_free(selem, true);  in bpf_selem_unlink_nofail()
443 				      struct bpf_local_storage_elem *selem)  in __bpf_local_storage_insert_cache()  argument
457 	if (selem_linked_to_storage(selem))  in __bpf_local_storage_insert_cache()
458 		rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));  in __bpf_local_storage_insert_cache()
551 	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;  in bpf_local_storage_update()  local
573 		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);  in bpf_local_storage_update()
574 		if (!selem)  in bpf_local_storage_update()
577 		err = bpf_local_storage_alloc(owner, smap, selem);  in bpf_local_storage_update()
579 			bpf_selem_free(selem, true);  in bpf_local_storage_update()
584 		return SDATA(selem);  in bpf_local_storage_update()
607 	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);  in bpf_local_storage_update()
634 		selem = SELEM(old_sdata);  in bpf_local_storage_update()
646 	bpf_selem_link_map_nolock(b, selem);  in bpf_local_storage_update()
649 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
667 	return err ? ERR_PTR(err) : SDATA(selem);  in bpf_local_storage_update()
735 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_destroy()  local
746 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_destroy()
747 		bpf_selem_unlink_nofail(selem, NULL);  in bpf_local_storage_destroy()
819 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_map_free()  local
847 		hlist_for_each_entry_rcu(selem, &b->list, map_node) {  in bpf_local_storage_map_free()
848 			bpf_selem_unlink_nofail(selem, b);  in bpf_local_storage_map_free()