118 	struct bpf_local_storage *local_storage;  in __bpf_local_storage_free_trace_rcu()  local
123 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in __bpf_local_storage_free_trace_rcu()
125 		kfree(local_storage);  in __bpf_local_storage_free_trace_rcu()
127 		kfree_rcu(local_storage, rcu);  in __bpf_local_storage_free_trace_rcu()
132 	struct bpf_local_storage *local_storage;  in bpf_local_storage_free_rcu()  local
134 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in bpf_local_storage_free_rcu()
135 	bpf_mem_cache_raw_free(local_storage);  in bpf_local_storage_free_rcu()
147 static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,  in __bpf_local_storage_free()  argument
151 		kfree_rcu(local_storage, rcu);  in __bpf_local_storage_free()
153 		call_rcu_tasks_trace(&local_storage->rcu,  in __bpf_local_storage_free()
157 static void bpf_local_storage_free(struct bpf_local_storage *local_storage,  in bpf_local_storage_free()  argument
161 	if (!local_storage)  in bpf_local_storage_free()
165 		__bpf_local_storage_free(local_storage, reuse_now);  in bpf_local_storage_free()
170 		call_rcu_tasks_trace(&local_storage->rcu,  in bpf_local_storage_free()
176 		bpf_mem_cache_free(&smap->storage_ma, local_storage);  in bpf_local_storage_free()
182 		call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu);  in bpf_local_storage_free()
285 static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
294 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
304 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
307 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
327 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
329 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
333 	if (rcu_access_pointer(local_storage->smap) == smap)  in bpf_selem_unlink_storage_nolock()
334 		RCU_INIT_POINTER(local_storage->smap, NULL);  in bpf_selem_unlink_storage_nolock()
339 static bool check_storage_bpf_ma(struct bpf_local_storage *local_storage,  in check_storage_bpf_ma()  argument
362 		n = rcu_dereference_check(hlist_first_rcu(&local_storage->list),  in check_storage_bpf_ma()
378 	struct bpf_local_storage *local_storage;  in bpf_selem_unlink_storage()  local
387 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_storage()
389 	storage_smap = rcu_dereference_check(local_storage->smap,  in bpf_selem_unlink_storage()
391 	bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, selem);  in bpf_selem_unlink_storage()
393 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_selem_unlink_storage()
396 			local_storage, selem, true, &selem_free_list);  in bpf_selem_unlink_storage()
397 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_selem_unlink_storage()
402 		bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);  in bpf_selem_unlink_storage()
405 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
408 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
409 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
452 void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,  in __bpf_local_storage_insert_cache()  argument
463 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in __bpf_local_storage_insert_cache()
465 		rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));  in __bpf_local_storage_insert_cache()
466 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in __bpf_local_storage_insert_cache()
562 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
577 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
579 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
605 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
623 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_update()
626 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
636 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
653 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
658 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
663 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()
737 void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)  in bpf_local_storage_destroy()  argument
746 	storage_smap = rcu_dereference_check(local_storage->smap, bpf_rcu_lock_held());  in bpf_local_storage_destroy()
747 	bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, NULL);  in bpf_local_storage_destroy()
758 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_destroy()
759 	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {  in bpf_local_storage_destroy()
771 			local_storage, selem, true, &free_selem_list);  in bpf_local_storage_destroy()
773 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_destroy()
778 		bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);  in bpf_local_storage_destroy()