166 void __kasan_unpoison_new_object(struct kmem_cache *cache, void *object)  in __kasan_unpoison_new_object()  argument
168 	kasan_unpoison(object, cache->object_size, false);  in __kasan_unpoison_new_object()
171 void __kasan_poison_new_object(struct kmem_cache *cache, void *object)  in __kasan_poison_new_object()  argument
173 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),  in __kasan_poison_new_object()
178  * This function assigns a tag to an object considering the following:
180  *    object somewhere (e.g. in the object itself). We preassign a tag for
181  *    each object in caches with constructors during slab creation and reuse
182  *    the same tag each time a particular object is allocated.
188 					const void *object, bool init)  in assign_tag()  argument
195 	 * set, assign a tag when the object is being allocated (init == false).  in assign_tag()
205 	return init ? kasan_random_tag() : get_tag(object);  in assign_tag()
209 						const void *object)  in __kasan_init_slab_obj()  argument
211 	/* Initialize per-object metadata if it is present. */  in __kasan_init_slab_obj()
213 		kasan_init_object_meta(cache, object);  in __kasan_init_slab_obj()
216 	object = set_tag(object, assign_tag(cache, object, true));  in __kasan_init_slab_obj()
218 	return (void *)object;  in __kasan_init_slab_obj()
221 /* Returns true when freeing the object is not safe. */
222 static bool check_slab_allocation(struct kmem_cache *cache, void *object,  in check_slab_allocation()  argument
225 	void *tagged_object = object;  in check_slab_allocation()
227 	object = kasan_reset_tag(object);  in check_slab_allocation()
229 	if (unlikely(nearest_obj(cache, virt_to_slab(object), object) != object)) {  in check_slab_allocation()
242 static inline void poison_slab_object(struct kmem_cache *cache, void *object,  in poison_slab_object()  argument
245 	void *tagged_object = object;  in poison_slab_object()
247 	object = kasan_reset_tag(object);  in poison_slab_object()
249 	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),  in poison_slab_object()
256 bool __kasan_slab_pre_free(struct kmem_cache *cache, void *object,  in __kasan_slab_pre_free()  argument
259 	if (is_kfence_address(object))  in __kasan_slab_pre_free()
261 	return check_slab_allocation(cache, object, ip);  in __kasan_slab_pre_free()
264 bool __kasan_slab_free(struct kmem_cache *cache, void *object, bool init,  in __kasan_slab_free()  argument
267 	if (is_kfence_address(object))  in __kasan_slab_free()
271 	 * If this point is reached with an object that must still be  in __kasan_slab_free()
276 	 * Putting the object on the quarantine wouldn't help catch UAFs (since  in __kasan_slab_free()
278 	 * SLAB_TYPESAFE_BY_RCU users not being careful enough about object  in __kasan_slab_free()
279 	 * reuse; so overall, putting the object into the quarantine here would  in __kasan_slab_free()
285 	poison_slab_object(cache, object, init);  in __kasan_slab_free()
291 	 * If the object is put into quarantine, do not let slab put the object  in __kasan_slab_free()
292 	 * onto the freelist for now. The object's metadata is kept until the  in __kasan_slab_free()
293 	 * object gets evicted from quarantine.  in __kasan_slab_free()
295 	if (kasan_quarantine_put(cache, object))  in __kasan_slab_free()
299 	 * Note: Keep per-object metadata to allow KASAN print stack traces for  in __kasan_slab_free()
303 	/* Let slab put the object onto the freelist. */  in __kasan_slab_free()
326 	/* The object will be poisoned by kasan_poison_pages(). */  in __kasan_kfree_large()
329 static inline void unpoison_slab_object(struct kmem_cache *cache, void *object,  in unpoison_slab_object()  argument
333 	 * Unpoison the whole object. For kmalloc() allocations,  in unpoison_slab_object()
336 	kasan_unpoison(object, cache->object_size, init);  in unpoison_slab_object()
340 		kasan_save_alloc_info(cache, object, flags);  in unpoison_slab_object()
344 					void *object, gfp_t flags, bool init)  in __kasan_slab_alloc()  argument
352 	if (unlikely(object == NULL))  in __kasan_slab_alloc()
355 	if (is_kfence_address(object))  in __kasan_slab_alloc()
356 		return (void *)object;  in __kasan_slab_alloc()
362 	tag = assign_tag(cache, object, false);  in __kasan_slab_alloc()
363 	tagged_object = set_tag(object, tag);  in __kasan_slab_alloc()
365 	/* Unpoison the object and save alloc info for non-kmalloc() allocations. */  in __kasan_slab_alloc()
372 				const void *object, size_t size, gfp_t flags)  in poison_kmalloc_redzone()  argument
379 	 * Partially poison the last object granule to cover the unaligned  in poison_kmalloc_redzone()
383 		kasan_poison_last_granule((void *)object, size);  in poison_kmalloc_redzone()
386 	redzone_start = round_up((unsigned long)(object + size),  in poison_kmalloc_redzone()
388 	redzone_end = round_up((unsigned long)(object + cache->object_size),  in poison_kmalloc_redzone()
398 		kasan_save_alloc_info(cache, (void *)object, flags);  in poison_kmalloc_redzone()
402 void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object,  in __kasan_kmalloc()  argument
408 	if (unlikely(object == NULL))  in __kasan_kmalloc()
411 	if (is_kfence_address(object))  in __kasan_kmalloc()
412 		return (void *)object;  in __kasan_kmalloc()
414 	/* The object has already been unpoisoned by kasan_slab_alloc(). */  in __kasan_kmalloc()
415 	poison_kmalloc_redzone(cache, object, size, flags);  in __kasan_kmalloc()
418 	return (void *)object;  in __kasan_kmalloc()
430 	 * Partially poison the last object granule to cover the unaligned  in poison_kmalloc_large_redzone()
452 	/* The object has already been unpoisoned by kasan_unpoison_pages(). */  in __kasan_kmalloc_large()
459 void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)  in __kasan_krealloc()  argument
466 	if (unlikely(object == ZERO_SIZE_PTR))  in __kasan_krealloc()
467 		return (void *)object;  in __kasan_krealloc()
469 	if (is_kfence_address(object))  in __kasan_krealloc()
470 		return (void *)object;  in __kasan_krealloc()
473 	 * Unpoison the object's data.  in __kasan_krealloc()
477 	kasan_unpoison(object, size, false);  in __kasan_krealloc()
479 	slab = virt_to_slab(object);  in __kasan_krealloc()
483 		poison_kmalloc_large_redzone(object, size, flags);  in __kasan_krealloc()
485 		poison_kmalloc_redzone(slab->slab_cache, object, size, flags);  in __kasan_krealloc()
487 	return (void *)object;  in __kasan_krealloc()
563 	/* Unpoison the object and save alloc info for non-kmalloc() allocations. */  in __kasan_mempool_unpoison_object()