513 int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)  in bpf_mem_alloc_init()  argument
526 	ma->percpu = percpu;  in bpf_mem_alloc_init()
541 		ma->objcg = objcg;  in bpf_mem_alloc_init()
552 		ma->cache = pc;  in bpf_mem_alloc_init()
562 	ma->objcg = objcg;  in bpf_mem_alloc_init()
577 	ma->caches = pcc;  in bpf_mem_alloc_init()
581 int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg)  in bpf_mem_alloc_percpu_init()  argument
589 	ma->caches = pcc;  in bpf_mem_alloc_percpu_init()
590 	ma->objcg = objcg;  in bpf_mem_alloc_percpu_init()
591 	ma->percpu = true;  in bpf_mem_alloc_percpu_init()
595 int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)  in bpf_mem_alloc_percpu_unit_init()  argument
610 	objcg = ma->objcg;  in bpf_mem_alloc_percpu_unit_init()
611 	pcc = ma->caches;  in bpf_mem_alloc_percpu_unit_init()
662 static void check_leaked_objs(struct bpf_mem_alloc *ma)  in check_leaked_objs()  argument
668 	if (ma->cache) {  in check_leaked_objs()
670 			c = per_cpu_ptr(ma->cache, cpu);  in check_leaked_objs()
674 	if (ma->caches) {  in check_leaked_objs()
676 			cc = per_cpu_ptr(ma->caches, cpu);  in check_leaked_objs()
685 static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)  in free_mem_alloc_no_barrier()  argument
688 	if (ma->dtor_ctx_free)  in free_mem_alloc_no_barrier()
689 		ma->dtor_ctx_free(ma->dtor_ctx);  in free_mem_alloc_no_barrier()
690 	check_leaked_objs(ma);  in free_mem_alloc_no_barrier()
691 	free_percpu(ma->cache);  in free_mem_alloc_no_barrier()
692 	free_percpu(ma->caches);  in free_mem_alloc_no_barrier()
693 	ma->cache = NULL;  in free_mem_alloc_no_barrier()
694 	ma->caches = NULL;  in free_mem_alloc_no_barrier()
697 static void free_mem_alloc(struct bpf_mem_alloc *ma)  in free_mem_alloc()  argument
713 	free_mem_alloc_no_barrier(ma);  in free_mem_alloc()
718 	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);  in free_mem_alloc_deferred()  local
720 	free_mem_alloc(ma);  in free_mem_alloc_deferred()
721 	kfree(ma);  in free_mem_alloc_deferred()
724 static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)  in destroy_mem_alloc()  argument
732 		free_mem_alloc_no_barrier(ma);  in destroy_mem_alloc()
736 	copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL);  in destroy_mem_alloc()
739 		free_mem_alloc(ma);  in destroy_mem_alloc()
744 	memset(ma, 0, sizeof(*ma));  in destroy_mem_alloc()
749 void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)  in bpf_mem_alloc_destroy()  argument
755 	if (ma->cache) {  in bpf_mem_alloc_destroy()
758 			c = per_cpu_ptr(ma->cache, cpu);  in bpf_mem_alloc_destroy()
765 		obj_cgroup_put(ma->objcg);  in bpf_mem_alloc_destroy()
766 		destroy_mem_alloc(ma, rcu_in_progress);  in bpf_mem_alloc_destroy()
768 	if (ma->caches) {  in bpf_mem_alloc_destroy()
771 			cc = per_cpu_ptr(ma->caches, cpu);  in bpf_mem_alloc_destroy()
781 		obj_cgroup_put(ma->objcg);  in bpf_mem_alloc_destroy()
782 		destroy_mem_alloc(ma, rcu_in_progress);  in bpf_mem_alloc_destroy()
896 void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)  in bpf_mem_alloc()  argument
904 	if (!ma->percpu)  in bpf_mem_alloc()
910 	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);  in bpf_mem_alloc()
914 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_free()  argument
927 	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);  in bpf_mem_free()
930 void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_free_rcu()  argument
943 	unit_free_rcu(this_cpu_ptr(ma->caches)->cache + idx, ptr);  in bpf_mem_free_rcu()
946 void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma)  in bpf_mem_cache_alloc()  argument
950 	ret = unit_alloc(this_cpu_ptr(ma->cache));  in bpf_mem_cache_alloc()
954 void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_cache_free()  argument
959 	unit_free(this_cpu_ptr(ma->cache), ptr);  in bpf_mem_cache_free()
962 void notrace bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr)  in bpf_mem_cache_free_rcu()  argument
967 	unit_free_rcu(this_cpu_ptr(ma->cache), ptr);  in bpf_mem_cache_free_rcu()
992 void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)  in bpf_mem_cache_alloc_flags()  argument
997 	c = this_cpu_ptr(ma->cache);  in bpf_mem_cache_alloc_flags()
1025 void bpf_mem_alloc_set_dtor(struct bpf_mem_alloc *ma, void (*dtor)(void *obj, void *ctx),  in bpf_mem_alloc_set_dtor()  argument
1032 	ma->dtor_ctx_free = dtor_ctx_free;  in bpf_mem_alloc_set_dtor()
1033 	ma->dtor_ctx = ctx;  in bpf_mem_alloc_set_dtor()
1035 	if (ma->cache) {  in bpf_mem_alloc_set_dtor()
1037 			c = per_cpu_ptr(ma->cache, cpu);  in bpf_mem_alloc_set_dtor()
1042 	if (ma->caches) {  in bpf_mem_alloc_set_dtor()
1044 			cc = per_cpu_ptr(ma->caches, cpu);  in bpf_mem_alloc_set_dtor()