Lines Matching full:c
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
148 static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags) in __alloc() argument
150 if (c->percpu_size) { in __alloc()
151 void __percpu **obj = kmalloc_node(c->percpu_size, flags, node); in __alloc()
152 void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags); in __alloc()
163 return kmalloc_node(c->unit_size, flags | __GFP_ZERO, node); in __alloc()
166 static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c) in get_memcg() argument
169 if (c->objcg) in get_memcg()
170 return get_mem_cgroup_from_objcg(c->objcg); in get_memcg()
177 static void inc_active(struct bpf_mem_cache *c, unsigned long *flags) in inc_active() argument
192 WARN_ON_ONCE(local_inc_return(&c->active) != 1); in inc_active()
195 static void dec_active(struct bpf_mem_cache *c, unsigned long *flags) in dec_active() argument
197 local_dec(&c->active); in dec_active()
202 static void add_obj_to_free_list(struct bpf_mem_cache *c, void *obj) in add_obj_to_free_list() argument
206 inc_active(c, &flags); in add_obj_to_free_list()
207 __llist_add(obj, &c->free_llist); in add_obj_to_free_list()
208 c->free_cnt++; in add_obj_to_free_list()
209 dec_active(c, &flags); in add_obj_to_free_list()
213 static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) in alloc_bulk() argument
224 * c->lock serializes concurrent llist_del_first() against in alloc_bulk()
227 scoped_guard(raw_spinlock_irqsave, &c->lock) { in alloc_bulk()
229 obj = llist_del_first(&c->free_by_rcu_ttrace); in alloc_bulk()
232 add_obj_to_free_list(c, obj); in alloc_bulk()
236 obj = llist_del_first(&c->waiting_for_gp_ttrace); in alloc_bulk()
239 add_obj_to_free_list(c, obj); in alloc_bulk()
245 memcg = get_memcg(c); in alloc_bulk()
253 obj = __alloc(c, node, gfp); in alloc_bulk()
256 add_obj_to_free_list(c, obj); in alloc_bulk()
270 static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool percpu) in free_all() argument
276 if (c->dtor) in free_all()
277 c->dtor((void *)pos + LLIST_NODE_SZ, c->dtor_ctx); in free_all()
284 static void __do_call_rcu_ttrace(struct bpf_mem_cache *c);
288 struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); in __free_rcu() local
291 scoped_guard(raw_spinlock_irqsave, &c->lock) in __free_rcu()
292 llnode = llist_del_all(&c->waiting_for_gp_ttrace); in __free_rcu()
294 free_all(c, llnode, !!c->percpu_size); in __free_rcu()
300 * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero. in __free_rcu()
302 if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1) in __free_rcu()
307 __do_call_rcu_ttrace(c); in __free_rcu()
310 static void enque_to_free(struct bpf_mem_cache *c, void *obj) in enque_to_free() argument
317 llist_add(llnode, &c->free_by_rcu_ttrace); in enque_to_free()
320 static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) in __do_call_rcu_ttrace() argument
328 atomic_set(&c->call_rcu_ttrace_in_progress, 1); in __do_call_rcu_ttrace()
330 WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); in __do_call_rcu_ttrace()
331 llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) in __do_call_rcu_ttrace()
332 llist_add(llnode, &c->waiting_for_gp_ttrace); in __do_call_rcu_ttrace()
334 if (unlikely(READ_ONCE(c->draining))) { in __do_call_rcu_ttrace()
335 __free_rcu(&c->rcu_ttrace); in __do_call_rcu_ttrace()
344 call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu); in __do_call_rcu_ttrace()
347 static void do_call_rcu_ttrace(struct bpf_mem_cache *c) in do_call_rcu_ttrace() argument
351 if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) { in do_call_rcu_ttrace()
352 if (unlikely(READ_ONCE(c->draining))) { in do_call_rcu_ttrace()
353 scoped_guard(raw_spinlock_irqsave, &c->lock) in do_call_rcu_ttrace()
354 llnode = llist_del_all(&c->free_by_rcu_ttrace); in do_call_rcu_ttrace()
355 free_all(c, llnode, !!c->percpu_size); in do_call_rcu_ttrace()
360 __do_call_rcu_ttrace(c); in do_call_rcu_ttrace()
363 static void free_bulk(struct bpf_mem_cache *c) in free_bulk() argument
365 struct bpf_mem_cache *tgt = c->tgt; in free_bulk()
370 WARN_ON_ONCE(tgt->unit_size != c->unit_size); in free_bulk()
371 WARN_ON_ONCE(tgt->percpu_size != c->percpu_size); in free_bulk()
374 inc_active(c, &flags); in free_bulk()
375 llnode = __llist_del_first(&c->free_llist); in free_bulk()
377 cnt = --c->free_cnt; in free_bulk()
380 dec_active(c, &flags); in free_bulk()
383 } while (cnt > (c->high_watermark + c->low_watermark) / 2); in free_bulk()
386 llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) in free_bulk()
393 struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu); in __free_by_rcu() local
394 struct bpf_mem_cache *tgt = c->tgt; in __free_by_rcu()
397 WARN_ON_ONCE(tgt->unit_size != c->unit_size); in __free_by_rcu()
398 WARN_ON_ONCE(tgt->percpu_size != c->percpu_size); in __free_by_rcu()
400 llnode = llist_del_all(&c->waiting_for_gp); in __free_by_rcu()
404 llist_add_batch(llnode, c->waiting_for_gp_tail, &tgt->free_by_rcu_ttrace); in __free_by_rcu()
409 atomic_set(&c->call_rcu_in_progress, 0); in __free_by_rcu()
412 static void check_free_by_rcu(struct bpf_mem_cache *c) in check_free_by_rcu() argument
418 if (unlikely(!llist_empty(&c->free_llist_extra_rcu))) { in check_free_by_rcu()
419 inc_active(c, &flags); in check_free_by_rcu()
420 llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra_rcu)) in check_free_by_rcu()
421 if (__llist_add(llnode, &c->free_by_rcu)) in check_free_by_rcu()
422 c->free_by_rcu_tail = llnode; in check_free_by_rcu()
423 dec_active(c, &flags); in check_free_by_rcu()
426 if (llist_empty(&c->free_by_rcu)) in check_free_by_rcu()
429 if (atomic_xchg(&c->call_rcu_in_progress, 1)) { in check_free_by_rcu()
442 WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); in check_free_by_rcu()
444 inc_active(c, &flags); in check_free_by_rcu()
445 WRITE_ONCE(c->waiting_for_gp.first, __llist_del_all(&c->free_by_rcu)); in check_free_by_rcu()
446 c->waiting_for_gp_tail = c->free_by_rcu_tail; in check_free_by_rcu()
447 dec_active(c, &flags); in check_free_by_rcu()
449 if (unlikely(READ_ONCE(c->draining))) { in check_free_by_rcu()
450 free_all(c, llist_del_all(&c->waiting_for_gp), !!c->percpu_size); in check_free_by_rcu()
451 atomic_set(&c->call_rcu_in_progress, 0); in check_free_by_rcu()
453 call_rcu_hurry(&c->rcu, __free_by_rcu); in check_free_by_rcu()
459 struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work); in bpf_mem_refill() local
463 cnt = c->free_cnt; in bpf_mem_refill()
464 if (cnt < c->low_watermark) in bpf_mem_refill()
468 alloc_bulk(c, c->batch, NUMA_NO_NODE, true); in bpf_mem_refill()
469 else if (cnt > c->high_watermark) in bpf_mem_refill()
470 free_bulk(c); in bpf_mem_refill()
472 check_free_by_rcu(c); in bpf_mem_refill()
475 static void notrace irq_work_raise(struct bpf_mem_cache *c) in irq_work_raise() argument
477 irq_work_queue(&c->refill_work); in irq_work_raise()
495 * memory consumption, set low_mark = 1 and high_mark = 3, resulting in c->batch = 1.
497 static void init_refill_work(struct bpf_mem_cache *c) in init_refill_work() argument
499 init_irq_work(&c->refill_work, bpf_mem_refill); in init_refill_work()
500 if (c->percpu_size) { in init_refill_work()
501 c->low_watermark = 1; in init_refill_work()
502 c->high_watermark = 3; in init_refill_work()
503 } else if (c->unit_size <= 256) { in init_refill_work()
504 c->low_watermark = 32; in init_refill_work()
505 c->high_watermark = 96; in init_refill_work()
512 c->low_watermark = max(32 * 256 / c->unit_size, 1); in init_refill_work()
513 c->high_watermark = max(96 * 256 / c->unit_size, 3); in init_refill_work()
515 c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1); in init_refill_work()
518 static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) in prefill_mem_cache() argument
527 if (!c->percpu_size && c->unit_size <= 256) in prefill_mem_cache()
529 alloc_bulk(c, cnt, cpu_to_node(cpu), false); in prefill_mem_cache()
542 struct bpf_mem_cache *c; struct bpf_mem_cache __percpu *pc; in bpf_mem_alloc_init() local
570 c = per_cpu_ptr(pc, cpu); in bpf_mem_alloc_init()
571 c->unit_size = unit_size; in bpf_mem_alloc_init()
572 c->objcg = objcg; in bpf_mem_alloc_init()
573 c->percpu_size = percpu_size; in bpf_mem_alloc_init()
574 c->tgt = c; in bpf_mem_alloc_init()
575 raw_spin_lock_init(&c->lock); in bpf_mem_alloc_init()
576 init_refill_work(c); in bpf_mem_alloc_init()
577 prefill_mem_cache(c, cpu); in bpf_mem_alloc_init()
593 c = &cc->cache[i]; in bpf_mem_alloc_init()
594 c->unit_size = sizes[i]; in bpf_mem_alloc_init()
595 c->objcg = objcg; in bpf_mem_alloc_init()
596 c->percpu_size = percpu_size; in bpf_mem_alloc_init()
597 c->tgt = c; in bpf_mem_alloc_init()
598 raw_spin_lock_init(&c->lock); in bpf_mem_alloc_init()
599 init_refill_work(c); in bpf_mem_alloc_init()
600 prefill_mem_cache(c, cpu); in bpf_mem_alloc_init()
627 struct bpf_mem_cache *c; in bpf_mem_alloc_percpu_unit_init() local
642 c = &cc->cache[i]; in bpf_mem_alloc_percpu_unit_init()
643 if (c->unit_size) in bpf_mem_alloc_percpu_unit_init()
646 c->unit_size = unit_size; in bpf_mem_alloc_percpu_unit_init()
647 c->objcg = objcg; in bpf_mem_alloc_percpu_unit_init()
648 c->percpu_size = percpu_size; in bpf_mem_alloc_percpu_unit_init()
649 c->tgt = c; in bpf_mem_alloc_percpu_unit_init()
650 raw_spin_lock_init(&c->lock); in bpf_mem_alloc_percpu_unit_init()
651 init_refill_work(c); in bpf_mem_alloc_percpu_unit_init()
652 prefill_mem_cache(c, cpu); in bpf_mem_alloc_percpu_unit_init()
658 static void drain_mem_cache(struct bpf_mem_cache *c) in drain_mem_cache() argument
660 bool percpu = !!c->percpu_size; in drain_mem_cache()
669 free_all(c, llist_del_all(&c->free_by_rcu_ttrace), percpu); in drain_mem_cache()
670 free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), percpu); in drain_mem_cache()
671 free_all(c, __llist_del_all(&c->free_llist), percpu); in drain_mem_cache()
672 free_all(c, __llist_del_all(&c->free_llist_extra), percpu); in drain_mem_cache()
673 free_all(c, __llist_del_all(&c->free_by_rcu), percpu); in drain_mem_cache()
674 free_all(c, __llist_del_all(&c->free_llist_extra_rcu), percpu); in drain_mem_cache()
675 free_all(c, llist_del_all(&c->waiting_for_gp), percpu); in drain_mem_cache()
678 static void check_mem_cache(struct bpf_mem_cache *c) in check_mem_cache() argument
680 WARN_ON_ONCE(!llist_empty(&c->free_by_rcu_ttrace)); in check_mem_cache()
681 WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); in check_mem_cache()
682 WARN_ON_ONCE(!llist_empty(&c->free_llist)); in check_mem_cache()
683 WARN_ON_ONCE(!llist_empty(&c->free_llist_extra)); in check_mem_cache()
684 WARN_ON_ONCE(!llist_empty(&c->free_by_rcu)); in check_mem_cache()
685 WARN_ON_ONCE(!llist_empty(&c->free_llist_extra_rcu)); in check_mem_cache()
686 WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); in check_mem_cache()
692 struct bpf_mem_cache *c; in check_leaked_objs() local
697 c = per_cpu_ptr(ma->cache, cpu); in check_leaked_objs()
698 check_mem_cache(c); in check_leaked_objs()
705 c = &cc->cache[i]; in check_leaked_objs()
706 check_mem_cache(c); in check_leaked_objs()
782 struct bpf_mem_cache *c; in bpf_mem_alloc_destroy() local
788 c = per_cpu_ptr(ma->cache, cpu); in bpf_mem_alloc_destroy()
789 WRITE_ONCE(c->draining, true); in bpf_mem_alloc_destroy()
790 irq_work_sync(&c->refill_work); in bpf_mem_alloc_destroy()
791 drain_mem_cache(c); in bpf_mem_alloc_destroy()
792 rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); in bpf_mem_alloc_destroy()
793 rcu_in_progress += atomic_read(&c->call_rcu_in_progress); in bpf_mem_alloc_destroy()
803 c = &cc->cache[i]; in bpf_mem_alloc_destroy()
804 WRITE_ONCE(c->draining, true); in bpf_mem_alloc_destroy()
805 irq_work_sync(&c->refill_work); in bpf_mem_alloc_destroy()
806 drain_mem_cache(c); in bpf_mem_alloc_destroy()
807 rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); in bpf_mem_alloc_destroy()
808 rcu_in_progress += atomic_read(&c->call_rcu_in_progress); in bpf_mem_alloc_destroy()
819 static void notrace *unit_alloc(struct bpf_mem_cache *c) in unit_alloc() argument
836 if (local_inc_return(&c->active) == 1) { in unit_alloc()
837 llnode = __llist_del_first(&c->free_llist); in unit_alloc()
839 cnt = --c->free_cnt; in unit_alloc()
840 *(struct bpf_mem_cache **)llnode = c; in unit_alloc()
843 local_dec(&c->active); in unit_alloc()
847 if (cnt < c->low_watermark) in unit_alloc()
848 irq_work_raise(c); in unit_alloc()
862 static void notrace unit_free(struct bpf_mem_cache *c, void *ptr) in unit_free() argument
874 c->tgt = *(struct bpf_mem_cache **)llnode; in unit_free()
877 if (local_inc_return(&c->active) == 1) { in unit_free()
878 __llist_add(llnode, &c->free_llist); in unit_free()
879 cnt = ++c->free_cnt; in unit_free()
887 llist_add(llnode, &c->free_llist_extra); in unit_free()
889 local_dec(&c->active); in unit_free()
891 if (cnt > c->high_watermark) in unit_free()
893 irq_work_raise(c); in unit_free()
902 static void notrace unit_free_rcu(struct bpf_mem_cache *c, void *ptr) in unit_free_rcu() argument
907 c->tgt = *(struct bpf_mem_cache **)llnode; in unit_free_rcu()
910 if (local_inc_return(&c->active) == 1) { in unit_free_rcu()
911 if (__llist_add(llnode, &c->free_by_rcu)) in unit_free_rcu()
912 c->free_by_rcu_tail = llnode; in unit_free_rcu()
914 llist_add(llnode, &c->free_llist_extra_rcu); in unit_free_rcu()
916 local_dec(&c->active); in unit_free_rcu()
918 if (!atomic_read(&c->call_rcu_in_progress)) in unit_free_rcu()
919 irq_work_raise(c); in unit_free_rcu()
946 struct bpf_mem_cache *c; in bpf_mem_free() local
952 c = *(void **)(ptr - LLIST_NODE_SZ); in bpf_mem_free()
953 idx = bpf_mem_cache_idx(c->unit_size); in bpf_mem_free()
962 struct bpf_mem_cache *c; in bpf_mem_free_rcu() local
968 c = *(void **)(ptr - LLIST_NODE_SZ); in bpf_mem_free_rcu()
969 idx = bpf_mem_cache_idx(c->unit_size); in bpf_mem_free_rcu()
1024 struct bpf_mem_cache *c; in bpf_mem_cache_alloc_flags() local
1027 c = this_cpu_ptr(ma->cache); in bpf_mem_cache_alloc_flags()
1029 ret = unit_alloc(c); in bpf_mem_cache_alloc_flags()
1033 memcg = get_memcg(c); in bpf_mem_cache_alloc_flags()
1035 ret = __alloc(c, NUMA_NO_NODE, GFP_KERNEL | __GFP_NOWARN | __GFP_ACCOUNT); in bpf_mem_cache_alloc_flags()
1037 *(struct bpf_mem_cache **)ret = c; in bpf_mem_cache_alloc_flags()
1059 struct bpf_mem_cache *c; in bpf_mem_alloc_set_dtor() local
1067 c = per_cpu_ptr(ma->cache, cpu); in bpf_mem_alloc_set_dtor()
1068 c->dtor = dtor; in bpf_mem_alloc_set_dtor()
1069 c->dtor_ctx = ctx; in bpf_mem_alloc_set_dtor()
1076 c = &cc->cache[i]; in bpf_mem_alloc_set_dtor()
1077 c->dtor = dtor; in bpf_mem_alloc_set_dtor()
1078 c->dtor_ctx = ctx; in bpf_mem_alloc_set_dtor()