4  * lockless freeing of objects to slabs in the slowpath.
81  *	A. slab->freelist	-> List of free objects in a slab
82  *	B. slab->inuse		-> Number of objects in use
83  *	C. slab->objects	-> Number of objects in slab
98  *   slab->objects and slab->freelist == NULL) are not placed on any list.
108  *   allocated objects will be simply leaked upon freeing instead of attempting
147  *   Objects may have to be freed to their slabs when they are from a remote
148  *   node (where we want to avoid filling local sheaves with remote objects)
172  * cannot scan all objects.
264  * sort the partial list by the number of objects in use.
292 #define MAX_OBJS_PER_PAGE	32767 /* since slab.objects is u15 */
352 	SHEAF_FLUSH,		/* Objects flushed from a sheaf */
353 	SHEAF_REFILL,		/* Objects refilled to a sheaf */
417 	void *objects[];  member
428  * The slab lists for all objects.
569 /* Loop over all objects in a slab */
765  * slab objects. If neither is enabled when this function is called,
789 	return sizeof(struct slabobj_ext) * slab->objects;  in obj_exts_size_in_slab()
797 	objext_offset = s->size * slab->objects;  in obj_exts_offset_in_slab()
914 	bitmap_zero(obj_map, slab->objects);  in __fill_map()
999 	if (object < base || object >= base + slab->objects * s->size ||  in check_valid_pointer()
1109 	pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",  in print_slab_info()
1110 	       slab, slab->objects, slab->inuse, slab->freelist,  in print_slab_info()
1331  *   - Filled with 0xbb (SLUB_RED_INACTIVE) for inactive objects and
1332  *     0xcc (SLUB_RED_ACTIVE) for objects in use when SLAB_RED_ZONE.
1347  *   - Filled with 0xbb (SLUB_RED_INACTIVE) for inactive objects and
1348  *     0xcc (SLUB_RED_ACTIVE) for objects in use when SLAB_RED_ZONE.
1375  *   (s->red_left_pad if redzoning); subsequent objects are reached by
1523 		 * of the free objects in this slab. May cause  in check_object()
1543 	if (slab->objects > maxobj) {  in check_slab()
1544 		slab_err(s, slab, "objects %u > max %u",  in check_slab()
1545 			slab->objects, maxobj);  in check_slab()
1548 	if (slab->inuse > slab->objects) {  in check_slab()
1550 			slab->inuse, slab->objects);  in check_slab()
1575 	while (fp && nr <= slab->objects) {  in on_freelist()
1587 				slab->inuse = slab->objects;  in on_freelist()
1597 	if (nr > slab->objects) {  in on_freelist()
1600 		slab->inuse = slab->objects;  in on_freelist()
1609 	if (slab->objects != max_objects) {  in on_freelist()
1610 		slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",  in on_freelist()
1611 			 slab->objects, max_objects);  in on_freelist()
1612 		slab->objects = max_objects;  in on_freelist()
1613 		slab_fix(s, "Number of objects adjusted");  in on_freelist()
1615 	if (slab->inuse != slab->objects - nr) {  in on_freelist()
1617 			 slab->inuse, slab->objects - nr);  in on_freelist()
1618 		slab->inuse = slab->objects - nr;  in on_freelist()
1669 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)  in inc_slabs_node()  argument
1674 	atomic_long_add(objects, &n->total_objects);  in inc_slabs_node()
1676 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)  in dec_slabs_node()  argument
1681 	atomic_long_sub(objects, &n->total_objects);  in dec_slabs_node()
1739 	 * objects as used avoids touching the remaining objects.  in alloc_debug_processing()
1741 	slab_fix(s, "Marking all objects used");  in alloc_debug_processing()
1742 	slab->inuse = slab->objects;  in alloc_debug_processing()
2027 							int objects) {}  in inc_slabs_node()  argument
2029 							int objects) {}  in dec_slabs_node()  argument
2076 			struct slabobj_ext *vec, unsigned int objects)  in handle_failed_objexts_alloc()  argument
2080 	 * objects with no tag reference. Mark all references in this  in handle_failed_objexts_alloc()
2086 		for (i = 0; i < objects; i++)  in handle_failed_objexts_alloc()
2096 			struct slabobj_ext *vec, unsigned int objects) {}  in handle_failed_objexts_alloc()  argument
2119 	size_t sz = sizeof(struct slabobj_ext) * slab->objects;  in obj_exts_alloc_size()
2145 	unsigned int objects = objs_per_slab(s, slab);  in alloc_slab_obj_exts()  local
2191 	handle_failed_objexts_alloc(old_exts, vec, objects);  in alloc_slab_obj_exts()
2294 		for_each_object(addr, s, slab_address(slab), slab->objects)  in alloc_slab_obj_exts_early()
2397 			       int objects)  in __alloc_tagging_slab_free_hook()  argument
2411 	for (i = 0; i < objects; i++) {  in __alloc_tagging_slab_free_hook()
2421 			     int objects)  in alloc_tagging_slab_free_hook()  argument
2424 		__alloc_tagging_slab_free_hook(s, slab, p, objects);  in alloc_tagging_slab_free_hook()
2436 			     int objects)  in alloc_tagging_slab_free_hook()  argument
2472 			  int objects)  in memcg_slab_free_hook()  argument
2484 	__memcg_slab_free_hook(s, slab, p, objects, obj_exts);  in memcg_slab_free_hook()
2532 	/* Ignore already charged objects. */  in memcg_slab_post_charge()
2558 					void **p, int objects)  in memcg_slab_free_hook()  argument
2585  * For objects allocated via kmalloc_nolock(), only a subset of alloc hooks
2763 	sheaf_size = struct_size(sheaf, objects, capacity);  in __alloc_empty_sheaf()
2812 	filled = refill_objects(s, &sheaf->objects[sheaf->size], gfp, to_fill,  in refill_sheaf()
2844  * Maximum number of objects freed during a single flush of main pcs sheaf.
2852  * Free all objects from the main sheaf. In order to perform
2860  * Returns how many objects are remaining to be flushed
2866 	void *objects[PCS_BATCH_MAX];  in __sheaf_flush_main_batch()  local
2877 	memcpy(objects, sheaf->objects + sheaf->size, batch * sizeof(void *));  in __sheaf_flush_main_batch()
2883 	__kmem_cache_free_bulk(s, batch, &objects[0]);  in __sheaf_flush_main_batch()
2923  * Free all objects from a sheaf that's unused, i.e. not linked to any
2935 	__kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]);  in sheaf_flush_unused()
2944 	void **p = &sheaf->objects[0];  in __rcu_free_sheaf_prepare()
3065 		 * was flushed and there should be no objects allocated from  in pcs_destroy()
3068 		 * warnings here trigger, and we should rather leave objects and  in pcs_destroy()
3355 	 * If the target page allocation failed, the number of objects on the  in next_freelist_entry()
3379 	if (slab->objects < 2 || !s->random_seq)  in shuffle_freelist()
3397 	page_limit = slab->objects * s->size;  in shuffle_freelist()
3405 	for (idx = 1; idx < slab->objects; idx++) {  in shuffle_freelist()
3495 	slab->objects = oo_objects(oo);  in allocate_slab()
3520 		for (idx = 0, p = start; idx < slab->objects - 1; idx++) {  in allocate_slab()
3582 		for_each_object(p, s, slab_address(slab), slab->objects)  in free_slab()
3594 	dec_slabs_node(s, slab_nid(slab), slab->objects);  in discard_slab()
3674 	if (slab->inuse == slab->objects) {  in alloc_single_from_partial()
3721 	if (slab->inuse == slab->objects)  in alloc_single_from_new_slab()
3726 	inc_slabs_node(s, nid, slab->objects);  in alloc_single_from_new_slab()
3762 		 * determine the number of free objects in the slab racily  in get_partial_node_bulk()
3765 		 * concurrent freeing, so the caller may get more objects than  in get_partial_node_bulk()
3769 		slab_free = flc.objects - flc.inuse;  in get_partial_node_bulk()
3873 	 * returns node local objects. If the ratio is higher then kmalloc()  in get_from_any_partial()
3874 	 * may return off node objects because partial slabs are obtained  in get_from_any_partial()
3882 	 * with available objects.  in get_from_any_partial()
4105 	return slab->objects - slab->inuse;  in count_free()
4128 		slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",  in free_debug_processing()
4158 		slab_err(s, slab, "Bulk free expected %d objects but found %d\n",  in free_debug_processing()
4200 			x += slab->objects - slab->inuse;  in count_partial_free_approx()
4203 		 * For a long list, approximate the total count of objects in  in count_partial_free_approx()
4210 			x += slab->objects - slab->inuse;  in count_partial_free_approx()
4215 			x += slab->objects - slab->inuse;  in count_partial_free_approx()
4294 		new.inuse = old.objects;  in get_freelist_nofreeze()
4329 	needs_add_partial = (slab->objects > count);  in alloc_from_new_slab()
4363 	inc_slabs_node(s, slab_nid(slab), slab->objects);  in alloc_from_new_slab()
4703 	 * We assume the percpu sheaves contain only local objects although it's  in alloc_from_pcs()
4722 	object = pcs->main->objects[pcs->main->size - 1];  in alloc_from_pcs()
4809 	memcpy(p, main->objects + main->size, batch * sizeof(void *));  in alloc_from_pcs_bulk()
4972 		sheaf = kzalloc_flex(*sheaf, objects, size, gfp);  in kmem_cache_prefill_sheaf()
4985 					     &sheaf->objects[0])) {  in kmem_cache_prefill_sheaf()
5095  * s->sheaf_capacity objects if such replacement is necessary, but the refill
5121 					     &sheaf->objects[sheaf->size])) {  in kmem_cache_refill_sheaf()
5168 		ret = sheaf->objects[--sheaf->size];  in kmem_cache_alloc_from_sheaf_noprof()
5451 		dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);  in free_to_partial_list()
5463  * Slow path handling. This may still be called frequently since objects
5551 	 * Objects left in the slab. If it was not on the partial list before  in __slab_free()
5780 	pcs->main->objects[pcs->main->size++] = object;  in free_to_pcs()
5801 	 * This may remove some objects due to slab_free_hook() returning false,  in rcu_free_sheaf()
5935 	rcu_sheaf->objects[rcu_sheaf->size++] = obj;  in __kfree_rcu_sheaf()
5964  * Bulk free objects to the percpu sheaves.
6047 	memcpy(main->objects + main->size, p, batch * sizeof(void *));  in free_to_pcs_bulk()
6088 	struct llist_head objects;  member
6095 	.objects = LLIST_HEAD_INIT(objects),
6107 	struct llist_head *objs = &df->objects;  in free_deferred_objects()
6126 		 * We used freepointer in 'x' to link 'x' into df->objects.  in free_deferred_objects()
6146 	if (llist_add(head + s->offset, &df->objects))  in defer_free()
6200 	 * to remove objects, whose reuse must be delayed.  in slab_free_bulk()
6489  * but ONLY for objects allocated by kmalloc_nolock().
6888  * This function progressively scans the array with free objects (with
6889  * a limited look ahead) and extract objects belonging to the same
6891  * slab/objects.  This can happen without any need for
6892  * synchronization, because the objects are owned by running process.
6893  * The freelist is build up as a single linked list in the objects.
6913 		/* Handle kalloc'ed objects */  in build_detached_freelist()
6961  * Internal bulk free of objects that were not initialised by the post alloc
7045 		 * Freelist had more objects than we can accommodate, we need to  in __refill_objects_node()
7326  * order 0 does not cause fragmentation in the page allocator. Larger objects
7332  * number of objects is in one slab. Otherwise we may generate too much
7337  * number of objects in a slab as critical. If we reach slab_max_order then
7341  * Higher order allocations also allow the placement of more objects in a
7480 		 * with zero-sized objects array as it's never modified.  in init_percpu_sheaves()
7539 	inc_slabs_node(kmem_cache_node, node, slab->objects);  in early_kmem_cache_node_alloc()
7645 	size = struct_size_t(struct slab_sheaf, objects, capacity);  in calculate_sheaf_capacity()
7647 	capacity = (size - struct_size_t(struct slab_sheaf, objects, 0)) / sizeof(void *);  in calculate_sheaf_capacity()
7714 		 * poisoning the objects, or are redzoning an object smaller  in calculate_sizes()
7757 		 * overwrites from earlier objects rather than let  in calculate_sizes()
7772 	 * offset 0. In order to align the objects we have to simply size  in calculate_sizes()
7809 	 * Determine the number of objects per slab  in calculate_sizes()
7824 		slab_bug(s, "Objects remaining on __kmem_cache_shutdown()");  in list_slab_objects()
7829 	for_each_object(p, s, addr, slab->objects) {  in list_slab_objects()
7895 	/* Attempt to free all objects */  in __kmem_cache_shutdown()
7931 	if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size  in __kmem_obj_info()
8037  * Rejects incorrectly sized objects and objects that are to be copied
8093  * being allocated from last increasing the chance that the last objects
8125 			int free = slab->objects - slab->inuse;  in __kmem_cache_do_shrink()
8133 			if (free == slab->objects) {  in __kmem_cache_do_shrink()
8137 				dec_slabs_node(s, node, slab->objects);  in __kmem_cache_do_shrink()
8536 	return slab->objects;  in count_total()
8557 	for_each_object(p, s, addr, slab->objects) {  in validate_slab()
8626  * Generate lists of code addresses where slabcache objects are allocated
8782 	for_each_object(p, s, addr, slab->objects)  in process_slab()
8796 	SL_OBJECTS,		/* Determine allocated objects not slabs */
8956 	unsigned int objects;  in cpu_partial_store()  local
8959 	err = kstrtouint(buf, 10, &objects);  in cpu_partial_store()
8962 	if (objects)  in cpu_partial_store()
9058 SLAB_ATTR_RO(objects);