1 // SPDX-License-Identifier: GPL-2.0-only
13 #include "memory-alloc.h"
38  * @flag_ptr: Location of the allocation-allowed flag
71  * performance-critical stage for us, so a linked list should be fine.
75 	size_t size;  member
97 static void add_kmalloc_block(size_t size)  in add_kmalloc_block()  argument
103 	memory_stats.kmalloc_bytes += size;  in add_kmalloc_block()
108 static void remove_kmalloc_block(size_t size)  in remove_kmalloc_block()  argument
113 	memory_stats.kmalloc_blocks--;  in remove_kmalloc_block()
114 	memory_stats.kmalloc_bytes -= size;  in remove_kmalloc_block()
118 static void add_vmalloc_block(struct vmalloc_block_info *block)  in add_vmalloc_block()  argument
123 	block->next = memory_stats.vmalloc_list;  in add_vmalloc_block()
124 	memory_stats.vmalloc_list = block;  in add_vmalloc_block()
126 	memory_stats.vmalloc_bytes += block->size;  in add_vmalloc_block()
133 	struct vmalloc_block_info *block;  in remove_vmalloc_block()  local
139 	     (block = *block_ptr) != NULL;  in remove_vmalloc_block()
140 	     block_ptr = &block->next) {  in remove_vmalloc_block()
141 		if (block->ptr == ptr) {  in remove_vmalloc_block()
142 			*block_ptr = block->next;  in remove_vmalloc_block()
143 			memory_stats.vmalloc_blocks--;  in remove_vmalloc_block()
144 			memory_stats.vmalloc_bytes -= block->size;  in remove_vmalloc_block()
150 	if (block != NULL)  in remove_vmalloc_block()
151 		vdo_free(block);  in remove_vmalloc_block()
157  * Determine whether allocating a memory block should use kmalloc or __vmalloc.
165  * kmalloc usually rounds the size of the block up to the next power of two, so when the requested
166  * block is bigger than PAGE_SIZE / 2 bytes, kmalloc will never give you less space than the
172  * have used vmalloc as a backup to kmalloc in the past, and a follow-up vmalloc of 32KB will work.
173  * But there is no strong case to be made for using kmalloc over vmalloc for these size chunks.
181  * @size: How many bytes to allocate
183 static inline bool use_kmalloc(size_t size)  in use_kmalloc()  argument
185 	return size <= PAGE_SIZE;  in use_kmalloc()
189  * Allocate storage based on memory size and alignment, logging an error if the allocation fails.
192  * @size: The size of an object
199 int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr)  in vdo_allocate_memory()  argument
205 	 * freeing memory such as compaction (which removes fragmentation) and page-out. There is  in vdo_allocate_memory()
210 	 * The caller must handle failure, but can reasonably do so by failing a higher-level  in vdo_allocate_memory()
220 		return -EINVAL;  in vdo_allocate_memory()
222 	if (size == 0) {  in vdo_allocate_memory()
231 	if (use_kmalloc(size) && (align < PAGE_SIZE)) {  in vdo_allocate_memory()
232 		p = kmalloc(size, gfp_flags | __GFP_NOWARN);  in vdo_allocate_memory()
240 			p = kmalloc(size, gfp_flags);  in vdo_allocate_memory()
246 		struct vmalloc_block_info *block;  in vdo_allocate_memory()  local
248 		if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == VDO_SUCCESS) {  in vdo_allocate_memory()
260 				p = __vmalloc(size, gfp_flags | __GFP_NOWARN);  in vdo_allocate_memory()
264 				if (jiffies_to_msecs(jiffies - start_time) > 1000) {  in vdo_allocate_memory()
266 					p = __vmalloc(size, gfp_flags);  in vdo_allocate_memory()
274 				vdo_free(block);  in vdo_allocate_memory()
276 				block->ptr = p;  in vdo_allocate_memory()
277 				block->size = PAGE_ALIGN(size);  in vdo_allocate_memory()
278 				add_vmalloc_block(block);  in vdo_allocate_memory()
288 			      size, what, jiffies_to_msecs(jiffies - start_time));  in vdo_allocate_memory()
289 		return -ENOMEM;  in vdo_allocate_memory()
297  * Allocate storage based on memory size, failing immediately if the required memory is not
300  * @size: The size of an object.
305 void *vdo_allocate_memory_nowait(size_t size, const char *what __maybe_unused)  in vdo_allocate_memory_nowait()  argument
307 	void *p = kmalloc(size, GFP_NOWAIT | __GFP_ZERO);  in vdo_allocate_memory_nowait()
333  * @old_size: The old size of the memory
334  * @size: The new size to allocate
340 int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what,  in vdo_reallocate_memory()  argument
345 	if (size == 0) {  in vdo_reallocate_memory()
351 	result = vdo_allocate(size, char, what, new_ptr);  in vdo_reallocate_memory()
356 		if (old_size < size)  in vdo_reallocate_memory()
357 			size = old_size;  in vdo_reallocate_memory()
359 		memcpy(*((void **) new_ptr), ptr, size);  in vdo_reallocate_memory()