| H A D | slub.c | 3 * SLUB: A slab allocator with low overhead percpu array caches and mostly 22 #include <linux/slab.h> 23 #include "slab.h" 65 * 3. slab_lock(slab) (Only on some arches) 71 * and to synchronize major metadata changes to slab cache structures. 82 * A. slab->freelist -> List of free objects in a slab 83 * B. slab->inuse -> Number of objects in use 84 * C. slab->objects -> Number of objects in slab 635 slab_test_pfmemalloc(const struct slab * slab) slab_test_pfmemalloc() argument 640 slab_set_pfmemalloc(struct slab * slab) slab_set_pfmemalloc() argument 645 __slab_clear_pfmemalloc(struct slab * slab) __slab_clear_pfmemalloc() argument 653 slab_lock(struct slab * slab) slab_lock() argument 658 slab_unlock(struct slab * slab) slab_unlock() argument 664 __update_freelist_fast(struct slab * slab,struct freelist_counters * old,struct freelist_counters * new) __update_freelist_fast() argument 677 __update_freelist_slow(struct slab * slab,struct freelist_counters * old,struct freelist_counters * new) __update_freelist_slow() argument 702 __slab_update_freelist(struct kmem_cache * s,struct slab * slab,struct freelist_counters * old,struct freelist_counters * new,const char * n) __slab_update_freelist() argument 728 slab_update_freelist(struct kmem_cache * s,struct slab * slab,struct freelist_counters * old,struct freelist_counters * new,const char * n) slab_update_freelist() argument 818 obj_exts_size_in_slab(struct slab * slab) obj_exts_size_in_slab() argument 824 obj_exts_offset_in_slab(struct kmem_cache * s,struct slab * slab) obj_exts_offset_in_slab() argument 834 obj_exts_fit_within_slab_leftover(struct kmem_cache * s,struct slab * slab) obj_exts_fit_within_slab_leftover() argument 842 obj_exts_in_slab(struct kmem_cache * s,struct slab * slab) obj_exts_in_slab() argument 862 obj_exts_size_in_slab(struct slab * slab) obj_exts_size_in_slab() argument 868 obj_exts_offset_in_slab(struct kmem_cache * s,struct slab * slab) obj_exts_offset_in_slab() argument 874 obj_exts_fit_within_slab_leftover(struct kmem_cache * s,struct slab * slab) obj_exts_fit_within_slab_leftover() argument 879 obj_exts_in_slab(struct kmem_cache * s,struct slab * slab) obj_exts_in_slab() argument 902 slab_set_obj_exts_in_object(struct slab * slab) slab_set_obj_exts_in_object() argument 912 slab_set_obj_exts_in_object(struct slab * slab) slab_set_obj_exts_in_object() argument 941 validate_slab_ptr(struct slab * slab) validate_slab_ptr() argument 950 __fill_map(unsigned long * obj_map,struct kmem_cache * s,struct slab * slab) __fill_map() argument 1030 check_valid_pointer(struct kmem_cache * s,struct slab * slab,void * object) check_valid_pointer() argument 1148 print_slab_info(const struct slab * slab) print_slab_info() argument 1199 print_trailer(struct kmem_cache * s,struct slab * slab,u8 * p) print_trailer() argument 1242 object_err(struct kmem_cache * s,struct slab * slab,u8 * object,const char * reason) object_err() argument 1260 __slab_err(struct slab * slab) __slab_err() argument 1271 slab_err(struct kmem_cache * s,struct slab * slab,const char * fmt,...) slab_err() argument 1334 check_bytes_and_report(struct kmem_cache * s,struct slab * slab,u8 * object,const char * what,u8 * start,unsigned int value,unsigned int bytes,bool slab_obj_print) check_bytes_and_report() argument 1424 check_pad_bytes(struct kmem_cache * s,struct slab * slab,u8 * p) check_pad_bytes() argument 1450 slab_pad_check(struct kmem_cache * s,struct slab * slab) slab_pad_check() argument 1494 check_object(struct kmem_cache * s,struct slab * slab,void * object,u8 val) check_object() argument 1579 check_slab(struct kmem_cache * s,struct slab * slab) check_slab() argument 1608 on_freelist(struct kmem_cache * s,struct slab * slab,void * search) on_freelist() argument 1665 trace(struct kmem_cache * s,struct slab * slab,void * object,int alloc) trace() argument 1687 add_full(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab) add_full() argument 1696 remove_full(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab) remove_full() argument 1736 setup_slab_debug(struct kmem_cache * s,struct slab * slab,void * addr) setup_slab_debug() argument 1747 alloc_consistency_checks(struct kmem_cache * s,struct slab * slab,void * object) alloc_consistency_checks() argument 1764 alloc_debug_processing(struct kmem_cache * s,struct slab * slab,void * object,int orig_size) alloc_debug_processing() argument 1791 free_consistency_checks(struct kmem_cache * s,struct slab * slab,void * object,unsigned long addr) free_consistency_checks() argument 2038 setup_slab_debug(struct kmem_cache * s,struct slab * slab,void * addr) setup_slab_debug() argument 2041 alloc_debug_processing(struct kmem_cache * s,struct slab * slab,void * object,int orig_size) alloc_debug_processing() argument 2044 free_debug_processing(struct kmem_cache * s,struct slab * slab,void * head,void * tail,int * bulk_cnt,unsigned long addr,depot_stack_handle_t handle) free_debug_processing() argument 2047 slab_pad_check(struct kmem_cache * s,struct slab * slab) slab_pad_check() argument 2048 check_object(struct kmem_cache * s,struct slab * slab,void * object,u8 val) check_object() argument 2054 add_full(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab) add_full() argument 2056 remove_full(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab) remove_full() argument 2119 mark_failed_objexts_alloc(struct slab * slab) mark_failed_objexts_alloc() argument 2124 handle_failed_objexts_alloc(struct slab * slab,unsigned long obj_exts,struct slabobj_ext * vec) handle_failed_objexts_alloc() argument 2153 mark_failed_objexts_alloc(struct slab * slab) mark_failed_objexts_alloc() argument 2154 handle_failed_objexts_alloc(struct slab * slab,unsigned long obj_exts,struct slabobj_ext * vec) handle_failed_objexts_alloc() argument 2159 init_slab_obj_exts(struct slab * slab) init_slab_obj_exts() argument 2164 alloc_slab_obj_exts(struct slab * slab,struct kmem_cache * s,gfp_t gfp,unsigned int alloc_flags) alloc_slab_obj_exts() argument 2253 free_slab_obj_exts(struct slab * slab,bool allow_spin) free_slab_obj_exts() argument 2285 alloc_slab_obj_exts_early(struct kmem_cache * s,struct slab * slab) alloc_slab_obj_exts_early() argument 2332 init_slab_obj_exts(struct slab * slab) init_slab_obj_exts() argument 2336 alloc_slab_obj_exts(struct slab * slab,struct kmem_cache * s,gfp_t gfp,unsigned int alloc_flags) alloc_slab_obj_exts() argument 2342 free_slab_obj_exts(struct slab * slab,bool allow_spin) free_slab_obj_exts() argument 2347 alloc_slab_obj_exts_early(struct kmem_cache * s,struct slab * slab) alloc_slab_obj_exts_early() argument 2356 prepare_slab_obj_exts_hook(struct kmem_cache * s,struct slab * slab,gfp_t flags,unsigned int alloc_flags,void * p) prepare_slab_obj_exts_hook() argument 2381 struct slab *slab; __alloc_tagging_slab_alloc_hook() local 2431 __alloc_tagging_slab_free_hook(struct kmem_cache * s,struct slab * slab,void ** p,int objects) __alloc_tagging_slab_free_hook() argument 2455 alloc_tagging_slab_free_hook(struct kmem_cache * s,struct slab * slab,void ** p,int objects) alloc_tagging_slab_free_hook() argument 2490 alloc_tagging_slab_free_hook(struct kmem_cache * s,struct slab * slab,void ** p,int objects) alloc_tagging_slab_free_hook() argument 2532 memcg_slab_free_hook(struct kmem_cache * s,struct slab * slab,void ** p,int objects) memcg_slab_free_hook() argument 2559 struct slab *slab; memcg_slab_post_charge() local 2620 memcg_slab_free_hook(struct kmem_cache * s,struct slab * slab,void ** p,int objects) memcg_slab_free_hook() argument 3004 struct slab *slab = virt_to_slab(p[i]); __rcu_free_sheaf_prepare() local 3338 struct slab *slab; alloc_slab_page() local 3412 account_slab(struct slab * slab,int order,struct kmem_cache * s,gfp_t gfp,unsigned int alloc_flags) account_slab() argument 3426 unaccount_slab(struct slab * slab,int order,struct kmem_cache * s,bool allow_spin) unaccount_slab() argument 3445 struct slab *slab; allocate_slab() local 3516 __free_slab(struct kmem_cache * s,struct slab * slab,bool allow_spin) __free_slab() argument 3533 free_new_slab_nolock(struct kmem_cache * s,struct slab * slab) free_new_slab_nolock() argument 3544 struct slab *slab = container_of(h, struct slab, rcu_head); rcu_free_slab() local 3549 free_slab(struct kmem_cache * s,struct slab * slab) free_slab() argument 3565 discard_slab(struct kmem_cache * s,struct slab * slab) discard_slab() argument 3571 slab_test_node_partial(const struct slab * slab) slab_test_node_partial() argument 3576 slab_set_node_partial(struct slab * slab) slab_set_node_partial() argument 3581 slab_clear_node_partial(struct slab * slab) slab_clear_node_partial() argument 3590 set_node_partial_state(struct kmem_cache_node * n,struct slab * slab) set_node_partial_state() argument 3597 __add_partial(struct kmem_cache_node * n,struct slab * slab,enum add_mode mode) __add_partial() argument 3607 add_partial(struct kmem_cache_node * n,struct slab * slab,enum add_mode mode) add_partial() argument 3614 clear_node_partial_state(struct kmem_cache_node * n,struct slab * slab) clear_node_partial_state() argument 3621 remove_partial(struct kmem_cache_node * n,struct slab * slab) remove_partial() argument 3635 alloc_single_from_partial(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab,int orig_size) alloc_single_from_partial() argument 3693 build_slab_freelist(struct kmem_cache * s,struct slab * slab,struct slab_obj_iter * iter) build_slab_freelist() argument 3718 init_slab_obj_iter(struct kmem_cache * s,struct slab * slab,struct slab_obj_iter * iter,bool allow_spin) init_slab_obj_iter() argument 3754 alloc_single_from_new_slab(struct kmem_cache * s,struct slab * slab,const struct slab_alloc_context * ac) alloc_single_from_new_slab() argument 3819 struct slab *slab, *slab2; get_partial_node_bulk() local 3890 struct slab *slab, *slab2; get_from_partial_node() local 4263 count_free(struct slab * slab) count_free() argument 4275 free_debug_processing(struct kmem_cache * s,struct slab * slab,void * head,void * tail,int * bulk_cnt,unsigned long addr,depot_stack_handle_t handle) free_debug_processing() argument 4334 count_partial(struct kmem_cache_node * n,int (* get_count)(struct slab *)) count_partial() argument 4338 struct slab *slab; count_partial() local 4355 struct slab *slab; count_partial_free_approx() local 4426 pfmemalloc_match(struct slab * slab,gfp_t gfpflags) pfmemalloc_match() argument 4442 get_freelist_nofreeze(struct kmem_cache * s,struct slab * slab,unsigned int * count) get_freelist_nofreeze() argument 4478 alloc_from_new_slab(struct kmem_cache * s,struct slab * slab,void ** p,unsigned int count,bool allow_spin) alloc_from_new_slab() argument 4539 struct slab *slab; ___slab_alloc() local 5612 free_to_partial_list(struct kmem_cache * s,struct slab * slab,void * head,void * tail,int bulk_cnt,unsigned long addr) free_to_partial_list() argument 5685 __slab_try_return_freelist(struct kmem_cache * s,struct kmem_cache_node * n,struct slab * slab,void * head,int cnt) __slab_try_return_freelist() argument 5720 __slab_free(struct kmem_cache * s,struct slab * slab,void * head,void * tail,int cnt,unsigned long addr) __slab_free() argument 6230 can_free_to_pcs(struct slab * slab) can_free_to_pcs() argument 6370 struct slab *slab = virt_to_slab(p[i]); free_to_pcs_bulk() local 6435 struct slab *slab; deferred_percpu_work_fn() local 6502 slab_free(struct kmem_cache * s,struct slab * slab,void * object,unsigned long addr) slab_free() argument 6523 struct slab *slab = virt_to_slab(object); memcg_alloc_abort_single() local 6533 slab_free_bulk(struct kmem_cache * s,struct slab * slab,void * head,void * tail,void ** p,int cnt,unsigned long addr) slab_free_bulk() argument 6554 struct slab *slab = virt_to_slab(object); slab_free_after_rcu_debug() local 6588 struct slab *slab; warn_free_bad_obj() local 6617 struct slab *slab; kmem_cache_free() local 6639 slab_ksize(struct slab * slab) slab_ksize() argument 6671 struct slab *slab; __ksize() local 6765 struct slab *slab = page_slab(page); kvfree_rcu_cb() local 6783 struct slab *slab; kfree() local 6816 struct slab *slab; kfree_nolock() local 6892 struct slab *slab = page_slab(page); __do_krealloc() local 7131 struct slab *slab; global() member 7157 struct slab *slab; build_detached_freelist() local 7269 struct slab *slab, *slab2; __refill_objects_node() local 7402 struct slab *slab; refill_objects() local 7762 struct slab *slab; early_kmem_cache_node_alloc() local 8078 list_slab_objects(struct kmem_cache * s,struct slab * slab) list_slab_objects() argument 8113 struct slab *slab, *h; free_partial() local 8173 __kmem_obj_info(struct kmem_obj_info * kpp,void * object,struct slab * slab) __kmem_obj_info() argument 8311 __check_heap_object(const void * ptr,unsigned long n,const struct slab * slab,bool to_user) __check_heap_object() argument 8367 struct slab *slab; __kmem_cache_do_shrink() local 8817 count_inuse(struct slab * slab) count_inuse() argument 8822 count_total(struct slab * slab) count_total() argument 8829 validate_slab(struct kmem_cache * s,struct slab * slab,unsigned long * obj_map) validate_slab() argument 8858 struct slab *slab; validate_slab_node() local 9061 process_slab(struct loc_track * t,struct kmem_cache * s,struct slab * slab,enum track_item alloc,unsigned long * obj_map) process_slab() argument 10045 struct slab *slab; slab_debug_trace_open() local [all...] |