1fbf59bc9STejun Heo /* 2fbf59bc9STejun Heo * linux/mm/percpu.c - percpu memory allocator 3fbf59bc9STejun Heo * 4fbf59bc9STejun Heo * Copyright (C) 2009 SUSE Linux Products GmbH 5fbf59bc9STejun Heo * Copyright (C) 2009 Tejun Heo <tj@kernel.org> 6fbf59bc9STejun Heo * 7fbf59bc9STejun Heo * This file is released under the GPLv2. 8fbf59bc9STejun Heo * 9fbf59bc9STejun Heo * This is percpu allocator which can handle both static and dynamic 10fbf59bc9STejun Heo * areas. Percpu areas are allocated in chunks in vmalloc area. Each 11fbf59bc9STejun Heo * chunk is consisted of num_possible_cpus() units and the first chunk 12fbf59bc9STejun Heo * is used for static percpu variables in the kernel image (special 13fbf59bc9STejun Heo * boot time alloc/init handling necessary as these areas need to be 14fbf59bc9STejun Heo * brought up before allocation services are running). Unit grows as 15fbf59bc9STejun Heo * necessary and all units grow or shrink in unison. When a chunk is 16fbf59bc9STejun Heo * filled up, another chunk is allocated. ie. in vmalloc area 17fbf59bc9STejun Heo * 18fbf59bc9STejun Heo * c0 c1 c2 19fbf59bc9STejun Heo * ------------------- ------------------- ------------ 20fbf59bc9STejun Heo * | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u 21fbf59bc9STejun Heo * ------------------- ...... ------------------- .... ------------ 22fbf59bc9STejun Heo * 23fbf59bc9STejun Heo * Allocation is done in offset-size areas of single unit space. Ie, 24fbf59bc9STejun Heo * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0, 25fbf59bc9STejun Heo * c1:u1, c1:u2 and c1:u3. Percpu access can be done by configuring 26e1b9aa3fSChristoph Lameter * percpu base registers pcpu_unit_size apart. 27fbf59bc9STejun Heo * 28fbf59bc9STejun Heo * There are usually many small percpu allocations many of them as 29fbf59bc9STejun Heo * small as 4 bytes. The allocator organizes chunks into lists 30fbf59bc9STejun Heo * according to free size and tries to allocate from the fullest one. 31fbf59bc9STejun Heo * Each chunk keeps the maximum contiguous area size hint which is 32fbf59bc9STejun Heo * guaranteed to be eqaul to or larger than the maximum contiguous 33fbf59bc9STejun Heo * area in the chunk. This helps the allocator not to iterate the 34fbf59bc9STejun Heo * chunk maps unnecessarily. 35fbf59bc9STejun Heo * 36fbf59bc9STejun Heo * Allocation state in each chunk is kept using an array of integers 37fbf59bc9STejun Heo * on chunk->map. A positive value in the map represents a free 38fbf59bc9STejun Heo * region and negative allocated. Allocation inside a chunk is done 39fbf59bc9STejun Heo * by scanning this map sequentially and serving the first matching 40fbf59bc9STejun Heo * entry. This is mostly copied from the percpu_modalloc() allocator. 41e1b9aa3fSChristoph Lameter * Chunks can be determined from the address using the index field 42e1b9aa3fSChristoph Lameter * in the page struct. The index field contains a pointer to the chunk. 43fbf59bc9STejun Heo * 44fbf59bc9STejun Heo * To use this allocator, arch code should do the followings. 45fbf59bc9STejun Heo * 46e74e3962STejun Heo * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA 47fbf59bc9STejun Heo * 48fbf59bc9STejun Heo * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate 49e0100983STejun Heo * regular address to percpu pointer and back if they need to be 50e0100983STejun Heo * different from the default 51fbf59bc9STejun Heo * 528d408b4bSTejun Heo * - use pcpu_setup_first_chunk() during percpu area initialization to 538d408b4bSTejun Heo * setup the first chunk containing the kernel static percpu area 54fbf59bc9STejun Heo */ 55fbf59bc9STejun Heo 56fbf59bc9STejun Heo #include <linux/bitmap.h> 57fbf59bc9STejun Heo #include <linux/bootmem.h> 58fbf59bc9STejun Heo #include <linux/list.h> 59fbf59bc9STejun Heo #include <linux/mm.h> 60fbf59bc9STejun Heo #include <linux/module.h> 61fbf59bc9STejun Heo #include <linux/mutex.h> 62fbf59bc9STejun Heo #include <linux/percpu.h> 63fbf59bc9STejun Heo #include <linux/pfn.h> 64fbf59bc9STejun Heo #include <linux/slab.h> 65ccea34b5STejun Heo #include <linux/spinlock.h> 66fbf59bc9STejun Heo #include <linux/vmalloc.h> 67a56dbddfSTejun Heo #include <linux/workqueue.h> 68fbf59bc9STejun Heo 69fbf59bc9STejun Heo #include <asm/cacheflush.h> 70e0100983STejun Heo #include <asm/sections.h> 71fbf59bc9STejun Heo #include <asm/tlbflush.h> 72fbf59bc9STejun Heo 73fbf59bc9STejun Heo #define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ 74fbf59bc9STejun Heo #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ 75fbf59bc9STejun Heo 76e0100983STejun Heo /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */ 77e0100983STejun Heo #ifndef __addr_to_pcpu_ptr 78e0100983STejun Heo #define __addr_to_pcpu_ptr(addr) \ 79e0100983STejun Heo (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr \ 80e0100983STejun Heo + (unsigned long)__per_cpu_start) 81e0100983STejun Heo #endif 82e0100983STejun Heo #ifndef __pcpu_ptr_to_addr 83e0100983STejun Heo #define __pcpu_ptr_to_addr(ptr) \ 84e0100983STejun Heo (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr \ 85e0100983STejun Heo - (unsigned long)__per_cpu_start) 86e0100983STejun Heo #endif 87e0100983STejun Heo 88fbf59bc9STejun Heo struct pcpu_chunk { 89fbf59bc9STejun Heo struct list_head list; /* linked to pcpu_slot lists */ 90fbf59bc9STejun Heo int free_size; /* free bytes in the chunk */ 91fbf59bc9STejun Heo int contig_hint; /* max contiguous size hint */ 92fbf59bc9STejun Heo struct vm_struct *vm; /* mapped vmalloc region */ 93fbf59bc9STejun Heo int map_used; /* # of map entries used */ 94fbf59bc9STejun Heo int map_alloc; /* # of map entries allocated */ 95fbf59bc9STejun Heo int *map; /* allocation map */ 968d408b4bSTejun Heo bool immutable; /* no [de]population allowed */ 973e24aa58STejun Heo struct page **page; /* points to page array */ 983e24aa58STejun Heo struct page *page_ar[]; /* #cpus * UNIT_PAGES */ 99fbf59bc9STejun Heo }; 100fbf59bc9STejun Heo 10140150d37STejun Heo static int pcpu_unit_pages __read_mostly; 10240150d37STejun Heo static int pcpu_unit_size __read_mostly; 10340150d37STejun Heo static int pcpu_chunk_size __read_mostly; 10440150d37STejun Heo static int pcpu_nr_slots __read_mostly; 10540150d37STejun Heo static size_t pcpu_chunk_struct_size __read_mostly; 106fbf59bc9STejun Heo 107fbf59bc9STejun Heo /* the address of the first chunk which starts with the kernel static area */ 10840150d37STejun Heo void *pcpu_base_addr __read_mostly; 109fbf59bc9STejun Heo EXPORT_SYMBOL_GPL(pcpu_base_addr); 110fbf59bc9STejun Heo 111ae9e6bc9STejun Heo /* 112ae9e6bc9STejun Heo * The first chunk which always exists. Note that unlike other 113ae9e6bc9STejun Heo * chunks, this one can be allocated and mapped in several different 114ae9e6bc9STejun Heo * ways and thus often doesn't live in the vmalloc area. 115ae9e6bc9STejun Heo */ 116ae9e6bc9STejun Heo static struct pcpu_chunk *pcpu_first_chunk; 117ae9e6bc9STejun Heo 118ae9e6bc9STejun Heo /* 119ae9e6bc9STejun Heo * Optional reserved chunk. This chunk reserves part of the first 120ae9e6bc9STejun Heo * chunk and serves it for reserved allocations. The amount of 121ae9e6bc9STejun Heo * reserved offset is in pcpu_reserved_chunk_limit. When reserved 122ae9e6bc9STejun Heo * area doesn't exist, the following variables contain NULL and 0 123ae9e6bc9STejun Heo * respectively. 124ae9e6bc9STejun Heo */ 125edcb4639STejun Heo static struct pcpu_chunk *pcpu_reserved_chunk; 126edcb4639STejun Heo static int pcpu_reserved_chunk_limit; 127edcb4639STejun Heo 128fbf59bc9STejun Heo /* 129ccea34b5STejun Heo * Synchronization rules. 130fbf59bc9STejun Heo * 131ccea34b5STejun Heo * There are two locks - pcpu_alloc_mutex and pcpu_lock. The former 132ccea34b5STejun Heo * protects allocation/reclaim paths, chunks and chunk->page arrays. 133ccea34b5STejun Heo * The latter is a spinlock and protects the index data structures - 134e1b9aa3fSChristoph Lameter * chunk slots, chunks and area maps in chunks. 135fbf59bc9STejun Heo * 136ccea34b5STejun Heo * During allocation, pcpu_alloc_mutex is kept locked all the time and 137ccea34b5STejun Heo * pcpu_lock is grabbed and released as necessary. All actual memory 138ccea34b5STejun Heo * allocations are done using GFP_KERNEL with pcpu_lock released. 139ccea34b5STejun Heo * 140ccea34b5STejun Heo * Free path accesses and alters only the index data structures, so it 141ccea34b5STejun Heo * can be safely called from atomic context. When memory needs to be 142ccea34b5STejun Heo * returned to the system, free path schedules reclaim_work which 143ccea34b5STejun Heo * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be 144ccea34b5STejun Heo * reclaimed, release both locks and frees the chunks. Note that it's 145ccea34b5STejun Heo * necessary to grab both locks to remove a chunk from circulation as 146ccea34b5STejun Heo * allocation path might be referencing the chunk with only 147ccea34b5STejun Heo * pcpu_alloc_mutex locked. 148fbf59bc9STejun Heo */ 149ccea34b5STejun Heo static DEFINE_MUTEX(pcpu_alloc_mutex); /* protects whole alloc and reclaim */ 150ccea34b5STejun Heo static DEFINE_SPINLOCK(pcpu_lock); /* protects index data structures */ 151fbf59bc9STejun Heo 15240150d37STejun Heo static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */ 153fbf59bc9STejun Heo 154a56dbddfSTejun Heo /* reclaim work to release fully free chunks, scheduled from free path */ 155a56dbddfSTejun Heo static void pcpu_reclaim(struct work_struct *work); 156a56dbddfSTejun Heo static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim); 157a56dbddfSTejun Heo 158d9b55eebSTejun Heo static int __pcpu_size_to_slot(int size) 159fbf59bc9STejun Heo { 160cae3aeb8STejun Heo int highbit = fls(size); /* size is in bytes */ 161fbf59bc9STejun Heo return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1); 162fbf59bc9STejun Heo } 163fbf59bc9STejun Heo 164d9b55eebSTejun Heo static int pcpu_size_to_slot(int size) 165d9b55eebSTejun Heo { 166d9b55eebSTejun Heo if (size == pcpu_unit_size) 167d9b55eebSTejun Heo return pcpu_nr_slots - 1; 168d9b55eebSTejun Heo return __pcpu_size_to_slot(size); 169d9b55eebSTejun Heo } 170d9b55eebSTejun Heo 171fbf59bc9STejun Heo static int pcpu_chunk_slot(const struct pcpu_chunk *chunk) 172fbf59bc9STejun Heo { 173fbf59bc9STejun Heo if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int)) 174fbf59bc9STejun Heo return 0; 175fbf59bc9STejun Heo 176fbf59bc9STejun Heo return pcpu_size_to_slot(chunk->free_size); 177fbf59bc9STejun Heo } 178fbf59bc9STejun Heo 179fbf59bc9STejun Heo static int pcpu_page_idx(unsigned int cpu, int page_idx) 180fbf59bc9STejun Heo { 181d9b55eebSTejun Heo return cpu * pcpu_unit_pages + page_idx; 182fbf59bc9STejun Heo } 183fbf59bc9STejun Heo 184fbf59bc9STejun Heo static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk, 185fbf59bc9STejun Heo unsigned int cpu, int page_idx) 186fbf59bc9STejun Heo { 187fbf59bc9STejun Heo return &chunk->page[pcpu_page_idx(cpu, page_idx)]; 188fbf59bc9STejun Heo } 189fbf59bc9STejun Heo 190fbf59bc9STejun Heo static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, 191fbf59bc9STejun Heo unsigned int cpu, int page_idx) 192fbf59bc9STejun Heo { 193fbf59bc9STejun Heo return (unsigned long)chunk->vm->addr + 194fbf59bc9STejun Heo (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT); 195fbf59bc9STejun Heo } 196fbf59bc9STejun Heo 197fbf59bc9STejun Heo static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk, 198fbf59bc9STejun Heo int page_idx) 199fbf59bc9STejun Heo { 200fbf59bc9STejun Heo return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL; 201fbf59bc9STejun Heo } 202fbf59bc9STejun Heo 203e1b9aa3fSChristoph Lameter /* set the pointer to a chunk in a page struct */ 204e1b9aa3fSChristoph Lameter static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu) 205e1b9aa3fSChristoph Lameter { 206e1b9aa3fSChristoph Lameter page->index = (unsigned long)pcpu; 207e1b9aa3fSChristoph Lameter } 208e1b9aa3fSChristoph Lameter 209e1b9aa3fSChristoph Lameter /* obtain pointer to a chunk from a page struct */ 210e1b9aa3fSChristoph Lameter static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page) 211e1b9aa3fSChristoph Lameter { 212e1b9aa3fSChristoph Lameter return (struct pcpu_chunk *)page->index; 213e1b9aa3fSChristoph Lameter } 214e1b9aa3fSChristoph Lameter 215fbf59bc9STejun Heo /** 2161880d93bSTejun Heo * pcpu_mem_alloc - allocate memory 2171880d93bSTejun Heo * @size: bytes to allocate 218fbf59bc9STejun Heo * 2191880d93bSTejun Heo * Allocate @size bytes. If @size is smaller than PAGE_SIZE, 2201880d93bSTejun Heo * kzalloc() is used; otherwise, vmalloc() is used. The returned 2211880d93bSTejun Heo * memory is always zeroed. 222fbf59bc9STejun Heo * 223ccea34b5STejun Heo * CONTEXT: 224ccea34b5STejun Heo * Does GFP_KERNEL allocation. 225ccea34b5STejun Heo * 226fbf59bc9STejun Heo * RETURNS: 2271880d93bSTejun Heo * Pointer to the allocated area on success, NULL on failure. 228fbf59bc9STejun Heo */ 2291880d93bSTejun Heo static void *pcpu_mem_alloc(size_t size) 230fbf59bc9STejun Heo { 231fbf59bc9STejun Heo if (size <= PAGE_SIZE) 2321880d93bSTejun Heo return kzalloc(size, GFP_KERNEL); 2331880d93bSTejun Heo else { 2341880d93bSTejun Heo void *ptr = vmalloc(size); 2351880d93bSTejun Heo if (ptr) 2361880d93bSTejun Heo memset(ptr, 0, size); 2371880d93bSTejun Heo return ptr; 2381880d93bSTejun Heo } 2391880d93bSTejun Heo } 240fbf59bc9STejun Heo 2411880d93bSTejun Heo /** 2421880d93bSTejun Heo * pcpu_mem_free - free memory 2431880d93bSTejun Heo * @ptr: memory to free 2441880d93bSTejun Heo * @size: size of the area 2451880d93bSTejun Heo * 2461880d93bSTejun Heo * Free @ptr. @ptr should have been allocated using pcpu_mem_alloc(). 2471880d93bSTejun Heo */ 2481880d93bSTejun Heo static void pcpu_mem_free(void *ptr, size_t size) 2491880d93bSTejun Heo { 2501880d93bSTejun Heo if (size <= PAGE_SIZE) 2511880d93bSTejun Heo kfree(ptr); 2521880d93bSTejun Heo else 2531880d93bSTejun Heo vfree(ptr); 254fbf59bc9STejun Heo } 255fbf59bc9STejun Heo 256fbf59bc9STejun Heo /** 257fbf59bc9STejun Heo * pcpu_chunk_relocate - put chunk in the appropriate chunk slot 258fbf59bc9STejun Heo * @chunk: chunk of interest 259fbf59bc9STejun Heo * @oslot: the previous slot it was on 260fbf59bc9STejun Heo * 261fbf59bc9STejun Heo * This function is called after an allocation or free changed @chunk. 262fbf59bc9STejun Heo * New slot according to the changed state is determined and @chunk is 263edcb4639STejun Heo * moved to the slot. Note that the reserved chunk is never put on 264edcb4639STejun Heo * chunk slots. 265ccea34b5STejun Heo * 266ccea34b5STejun Heo * CONTEXT: 267ccea34b5STejun Heo * pcpu_lock. 268fbf59bc9STejun Heo */ 269fbf59bc9STejun Heo static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) 270fbf59bc9STejun Heo { 271fbf59bc9STejun Heo int nslot = pcpu_chunk_slot(chunk); 272fbf59bc9STejun Heo 273edcb4639STejun Heo if (chunk != pcpu_reserved_chunk && oslot != nslot) { 274fbf59bc9STejun Heo if (oslot < nslot) 275fbf59bc9STejun Heo list_move(&chunk->list, &pcpu_slot[nslot]); 276fbf59bc9STejun Heo else 277fbf59bc9STejun Heo list_move_tail(&chunk->list, &pcpu_slot[nslot]); 278fbf59bc9STejun Heo } 279fbf59bc9STejun Heo } 280fbf59bc9STejun Heo 281fbf59bc9STejun Heo /** 282e1b9aa3fSChristoph Lameter * pcpu_chunk_addr_search - determine chunk containing specified address 283e1b9aa3fSChristoph Lameter * @addr: address for which the chunk needs to be determined. 284ccea34b5STejun Heo * 285fbf59bc9STejun Heo * RETURNS: 286fbf59bc9STejun Heo * The address of the found chunk. 287fbf59bc9STejun Heo */ 288fbf59bc9STejun Heo static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr) 289fbf59bc9STejun Heo { 290ae9e6bc9STejun Heo void *first_start = pcpu_first_chunk->vm->addr; 291fbf59bc9STejun Heo 292ae9e6bc9STejun Heo /* is it in the first chunk? */ 29379ba6ac8STejun Heo if (addr >= first_start && addr < first_start + pcpu_unit_size) { 294ae9e6bc9STejun Heo /* is it in the reserved area? */ 295ae9e6bc9STejun Heo if (addr < first_start + pcpu_reserved_chunk_limit) 296edcb4639STejun Heo return pcpu_reserved_chunk; 297ae9e6bc9STejun Heo return pcpu_first_chunk; 298edcb4639STejun Heo } 299edcb4639STejun Heo 300e1b9aa3fSChristoph Lameter return pcpu_get_page_chunk(vmalloc_to_page(addr)); 301fbf59bc9STejun Heo } 302fbf59bc9STejun Heo 303fbf59bc9STejun Heo /** 3049f7dcf22STejun Heo * pcpu_extend_area_map - extend area map for allocation 3059f7dcf22STejun Heo * @chunk: target chunk 3069f7dcf22STejun Heo * 3079f7dcf22STejun Heo * Extend area map of @chunk so that it can accomodate an allocation. 3089f7dcf22STejun Heo * A single allocation can split an area into three areas, so this 3099f7dcf22STejun Heo * function makes sure that @chunk->map has at least two extra slots. 3109f7dcf22STejun Heo * 311ccea34b5STejun Heo * CONTEXT: 312ccea34b5STejun Heo * pcpu_alloc_mutex, pcpu_lock. pcpu_lock is released and reacquired 313ccea34b5STejun Heo * if area map is extended. 314ccea34b5STejun Heo * 3159f7dcf22STejun Heo * RETURNS: 3169f7dcf22STejun Heo * 0 if noop, 1 if successfully extended, -errno on failure. 3179f7dcf22STejun Heo */ 3189f7dcf22STejun Heo static int pcpu_extend_area_map(struct pcpu_chunk *chunk) 3199f7dcf22STejun Heo { 3209f7dcf22STejun Heo int new_alloc; 3219f7dcf22STejun Heo int *new; 3229f7dcf22STejun Heo size_t size; 3239f7dcf22STejun Heo 3249f7dcf22STejun Heo /* has enough? */ 3259f7dcf22STejun Heo if (chunk->map_alloc >= chunk->map_used + 2) 3269f7dcf22STejun Heo return 0; 3279f7dcf22STejun Heo 328ccea34b5STejun Heo spin_unlock_irq(&pcpu_lock); 329ccea34b5STejun Heo 3309f7dcf22STejun Heo new_alloc = PCPU_DFL_MAP_ALLOC; 3319f7dcf22STejun Heo while (new_alloc < chunk->map_used + 2) 3329f7dcf22STejun Heo new_alloc *= 2; 3339f7dcf22STejun Heo 3349f7dcf22STejun Heo new = pcpu_mem_alloc(new_alloc * sizeof(new[0])); 335ccea34b5STejun Heo if (!new) { 336ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 3379f7dcf22STejun Heo return -ENOMEM; 338ccea34b5STejun Heo } 339ccea34b5STejun Heo 340ccea34b5STejun Heo /* 341ccea34b5STejun Heo * Acquire pcpu_lock and switch to new area map. Only free 342ccea34b5STejun Heo * could have happened inbetween, so map_used couldn't have 343ccea34b5STejun Heo * grown. 344ccea34b5STejun Heo */ 345ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 346ccea34b5STejun Heo BUG_ON(new_alloc < chunk->map_used + 2); 3479f7dcf22STejun Heo 3489f7dcf22STejun Heo size = chunk->map_alloc * sizeof(chunk->map[0]); 3499f7dcf22STejun Heo memcpy(new, chunk->map, size); 3509f7dcf22STejun Heo 3519f7dcf22STejun Heo /* 3529f7dcf22STejun Heo * map_alloc < PCPU_DFL_MAP_ALLOC indicates that the chunk is 3539f7dcf22STejun Heo * one of the first chunks and still using static map. 3549f7dcf22STejun Heo */ 3559f7dcf22STejun Heo if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC) 3569f7dcf22STejun Heo pcpu_mem_free(chunk->map, size); 3579f7dcf22STejun Heo 3589f7dcf22STejun Heo chunk->map_alloc = new_alloc; 3599f7dcf22STejun Heo chunk->map = new; 3609f7dcf22STejun Heo return 0; 3619f7dcf22STejun Heo } 3629f7dcf22STejun Heo 3639f7dcf22STejun Heo /** 364fbf59bc9STejun Heo * pcpu_split_block - split a map block 365fbf59bc9STejun Heo * @chunk: chunk of interest 366fbf59bc9STejun Heo * @i: index of map block to split 367cae3aeb8STejun Heo * @head: head size in bytes (can be 0) 368cae3aeb8STejun Heo * @tail: tail size in bytes (can be 0) 369fbf59bc9STejun Heo * 370fbf59bc9STejun Heo * Split the @i'th map block into two or three blocks. If @head is 371fbf59bc9STejun Heo * non-zero, @head bytes block is inserted before block @i moving it 372fbf59bc9STejun Heo * to @i+1 and reducing its size by @head bytes. 373fbf59bc9STejun Heo * 374fbf59bc9STejun Heo * If @tail is non-zero, the target block, which can be @i or @i+1 375fbf59bc9STejun Heo * depending on @head, is reduced by @tail bytes and @tail byte block 376fbf59bc9STejun Heo * is inserted after the target block. 377fbf59bc9STejun Heo * 3789f7dcf22STejun Heo * @chunk->map must have enough free slots to accomodate the split. 379ccea34b5STejun Heo * 380ccea34b5STejun Heo * CONTEXT: 381ccea34b5STejun Heo * pcpu_lock. 382fbf59bc9STejun Heo */ 3839f7dcf22STejun Heo static void pcpu_split_block(struct pcpu_chunk *chunk, int i, 3849f7dcf22STejun Heo int head, int tail) 385fbf59bc9STejun Heo { 386fbf59bc9STejun Heo int nr_extra = !!head + !!tail; 387fbf59bc9STejun Heo 3889f7dcf22STejun Heo BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra); 389fbf59bc9STejun Heo 3909f7dcf22STejun Heo /* insert new subblocks */ 391fbf59bc9STejun Heo memmove(&chunk->map[i + nr_extra], &chunk->map[i], 392fbf59bc9STejun Heo sizeof(chunk->map[0]) * (chunk->map_used - i)); 393fbf59bc9STejun Heo chunk->map_used += nr_extra; 394fbf59bc9STejun Heo 395fbf59bc9STejun Heo if (head) { 396fbf59bc9STejun Heo chunk->map[i + 1] = chunk->map[i] - head; 397fbf59bc9STejun Heo chunk->map[i++] = head; 398fbf59bc9STejun Heo } 399fbf59bc9STejun Heo if (tail) { 400fbf59bc9STejun Heo chunk->map[i++] -= tail; 401fbf59bc9STejun Heo chunk->map[i] = tail; 402fbf59bc9STejun Heo } 403fbf59bc9STejun Heo } 404fbf59bc9STejun Heo 405fbf59bc9STejun Heo /** 406fbf59bc9STejun Heo * pcpu_alloc_area - allocate area from a pcpu_chunk 407fbf59bc9STejun Heo * @chunk: chunk of interest 408cae3aeb8STejun Heo * @size: wanted size in bytes 409fbf59bc9STejun Heo * @align: wanted align 410fbf59bc9STejun Heo * 411fbf59bc9STejun Heo * Try to allocate @size bytes area aligned at @align from @chunk. 412fbf59bc9STejun Heo * Note that this function only allocates the offset. It doesn't 413fbf59bc9STejun Heo * populate or map the area. 414fbf59bc9STejun Heo * 4159f7dcf22STejun Heo * @chunk->map must have at least two free slots. 4169f7dcf22STejun Heo * 417ccea34b5STejun Heo * CONTEXT: 418ccea34b5STejun Heo * pcpu_lock. 419ccea34b5STejun Heo * 420fbf59bc9STejun Heo * RETURNS: 4219f7dcf22STejun Heo * Allocated offset in @chunk on success, -1 if no matching area is 4229f7dcf22STejun Heo * found. 423fbf59bc9STejun Heo */ 424fbf59bc9STejun Heo static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align) 425fbf59bc9STejun Heo { 426fbf59bc9STejun Heo int oslot = pcpu_chunk_slot(chunk); 427fbf59bc9STejun Heo int max_contig = 0; 428fbf59bc9STejun Heo int i, off; 429fbf59bc9STejun Heo 430fbf59bc9STejun Heo for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) { 431fbf59bc9STejun Heo bool is_last = i + 1 == chunk->map_used; 432fbf59bc9STejun Heo int head, tail; 433fbf59bc9STejun Heo 434fbf59bc9STejun Heo /* extra for alignment requirement */ 435fbf59bc9STejun Heo head = ALIGN(off, align) - off; 436fbf59bc9STejun Heo BUG_ON(i == 0 && head != 0); 437fbf59bc9STejun Heo 438fbf59bc9STejun Heo if (chunk->map[i] < 0) 439fbf59bc9STejun Heo continue; 440fbf59bc9STejun Heo if (chunk->map[i] < head + size) { 441fbf59bc9STejun Heo max_contig = max(chunk->map[i], max_contig); 442fbf59bc9STejun Heo continue; 443fbf59bc9STejun Heo } 444fbf59bc9STejun Heo 445fbf59bc9STejun Heo /* 446fbf59bc9STejun Heo * If head is small or the previous block is free, 447fbf59bc9STejun Heo * merge'em. Note that 'small' is defined as smaller 448fbf59bc9STejun Heo * than sizeof(int), which is very small but isn't too 449fbf59bc9STejun Heo * uncommon for percpu allocations. 450fbf59bc9STejun Heo */ 451fbf59bc9STejun Heo if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) { 452fbf59bc9STejun Heo if (chunk->map[i - 1] > 0) 453fbf59bc9STejun Heo chunk->map[i - 1] += head; 454fbf59bc9STejun Heo else { 455fbf59bc9STejun Heo chunk->map[i - 1] -= head; 456fbf59bc9STejun Heo chunk->free_size -= head; 457fbf59bc9STejun Heo } 458fbf59bc9STejun Heo chunk->map[i] -= head; 459fbf59bc9STejun Heo off += head; 460fbf59bc9STejun Heo head = 0; 461fbf59bc9STejun Heo } 462fbf59bc9STejun Heo 463fbf59bc9STejun Heo /* if tail is small, just keep it around */ 464fbf59bc9STejun Heo tail = chunk->map[i] - head - size; 465fbf59bc9STejun Heo if (tail < sizeof(int)) 466fbf59bc9STejun Heo tail = 0; 467fbf59bc9STejun Heo 468fbf59bc9STejun Heo /* split if warranted */ 469fbf59bc9STejun Heo if (head || tail) { 4709f7dcf22STejun Heo pcpu_split_block(chunk, i, head, tail); 471fbf59bc9STejun Heo if (head) { 472fbf59bc9STejun Heo i++; 473fbf59bc9STejun Heo off += head; 474fbf59bc9STejun Heo max_contig = max(chunk->map[i - 1], max_contig); 475fbf59bc9STejun Heo } 476fbf59bc9STejun Heo if (tail) 477fbf59bc9STejun Heo max_contig = max(chunk->map[i + 1], max_contig); 478fbf59bc9STejun Heo } 479fbf59bc9STejun Heo 480fbf59bc9STejun Heo /* update hint and mark allocated */ 481fbf59bc9STejun Heo if (is_last) 482fbf59bc9STejun Heo chunk->contig_hint = max_contig; /* fully scanned */ 483fbf59bc9STejun Heo else 484fbf59bc9STejun Heo chunk->contig_hint = max(chunk->contig_hint, 485fbf59bc9STejun Heo max_contig); 486fbf59bc9STejun Heo 487fbf59bc9STejun Heo chunk->free_size -= chunk->map[i]; 488fbf59bc9STejun Heo chunk->map[i] = -chunk->map[i]; 489fbf59bc9STejun Heo 490fbf59bc9STejun Heo pcpu_chunk_relocate(chunk, oslot); 491fbf59bc9STejun Heo return off; 492fbf59bc9STejun Heo } 493fbf59bc9STejun Heo 494fbf59bc9STejun Heo chunk->contig_hint = max_contig; /* fully scanned */ 495fbf59bc9STejun Heo pcpu_chunk_relocate(chunk, oslot); 496fbf59bc9STejun Heo 4979f7dcf22STejun Heo /* tell the upper layer that this chunk has no matching area */ 4989f7dcf22STejun Heo return -1; 499fbf59bc9STejun Heo } 500fbf59bc9STejun Heo 501fbf59bc9STejun Heo /** 502fbf59bc9STejun Heo * pcpu_free_area - free area to a pcpu_chunk 503fbf59bc9STejun Heo * @chunk: chunk of interest 504fbf59bc9STejun Heo * @freeme: offset of area to free 505fbf59bc9STejun Heo * 506fbf59bc9STejun Heo * Free area starting from @freeme to @chunk. Note that this function 507fbf59bc9STejun Heo * only modifies the allocation map. It doesn't depopulate or unmap 508fbf59bc9STejun Heo * the area. 509ccea34b5STejun Heo * 510ccea34b5STejun Heo * CONTEXT: 511ccea34b5STejun Heo * pcpu_lock. 512fbf59bc9STejun Heo */ 513fbf59bc9STejun Heo static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme) 514fbf59bc9STejun Heo { 515fbf59bc9STejun Heo int oslot = pcpu_chunk_slot(chunk); 516fbf59bc9STejun Heo int i, off; 517fbf59bc9STejun Heo 518fbf59bc9STejun Heo for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) 519fbf59bc9STejun Heo if (off == freeme) 520fbf59bc9STejun Heo break; 521fbf59bc9STejun Heo BUG_ON(off != freeme); 522fbf59bc9STejun Heo BUG_ON(chunk->map[i] > 0); 523fbf59bc9STejun Heo 524fbf59bc9STejun Heo chunk->map[i] = -chunk->map[i]; 525fbf59bc9STejun Heo chunk->free_size += chunk->map[i]; 526fbf59bc9STejun Heo 527fbf59bc9STejun Heo /* merge with previous? */ 528fbf59bc9STejun Heo if (i > 0 && chunk->map[i - 1] >= 0) { 529fbf59bc9STejun Heo chunk->map[i - 1] += chunk->map[i]; 530fbf59bc9STejun Heo chunk->map_used--; 531fbf59bc9STejun Heo memmove(&chunk->map[i], &chunk->map[i + 1], 532fbf59bc9STejun Heo (chunk->map_used - i) * sizeof(chunk->map[0])); 533fbf59bc9STejun Heo i--; 534fbf59bc9STejun Heo } 535fbf59bc9STejun Heo /* merge with next? */ 536fbf59bc9STejun Heo if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) { 537fbf59bc9STejun Heo chunk->map[i] += chunk->map[i + 1]; 538fbf59bc9STejun Heo chunk->map_used--; 539fbf59bc9STejun Heo memmove(&chunk->map[i + 1], &chunk->map[i + 2], 540fbf59bc9STejun Heo (chunk->map_used - (i + 1)) * sizeof(chunk->map[0])); 541fbf59bc9STejun Heo } 542fbf59bc9STejun Heo 543fbf59bc9STejun Heo chunk->contig_hint = max(chunk->map[i], chunk->contig_hint); 544fbf59bc9STejun Heo pcpu_chunk_relocate(chunk, oslot); 545fbf59bc9STejun Heo } 546fbf59bc9STejun Heo 547fbf59bc9STejun Heo /** 548fbf59bc9STejun Heo * pcpu_unmap - unmap pages out of a pcpu_chunk 549fbf59bc9STejun Heo * @chunk: chunk of interest 550fbf59bc9STejun Heo * @page_start: page index of the first page to unmap 551fbf59bc9STejun Heo * @page_end: page index of the last page to unmap + 1 55285ae87c1STejun Heo * @flush_tlb: whether to flush tlb or not 553fbf59bc9STejun Heo * 554fbf59bc9STejun Heo * For each cpu, unmap pages [@page_start,@page_end) out of @chunk. 555fbf59bc9STejun Heo * If @flush is true, vcache is flushed before unmapping and tlb 556fbf59bc9STejun Heo * after. 557fbf59bc9STejun Heo */ 558fbf59bc9STejun Heo static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end, 55985ae87c1STejun Heo bool flush_tlb) 560fbf59bc9STejun Heo { 561fbf59bc9STejun Heo unsigned int last = num_possible_cpus() - 1; 562fbf59bc9STejun Heo unsigned int cpu; 563fbf59bc9STejun Heo 5648d408b4bSTejun Heo /* unmap must not be done on immutable chunk */ 5658d408b4bSTejun Heo WARN_ON(chunk->immutable); 5668d408b4bSTejun Heo 567fbf59bc9STejun Heo /* 568fbf59bc9STejun Heo * Each flushing trial can be very expensive, issue flush on 569fbf59bc9STejun Heo * the whole region at once rather than doing it for each cpu. 570fbf59bc9STejun Heo * This could be an overkill but is more scalable. 571fbf59bc9STejun Heo */ 572fbf59bc9STejun Heo flush_cache_vunmap(pcpu_chunk_addr(chunk, 0, page_start), 573fbf59bc9STejun Heo pcpu_chunk_addr(chunk, last, page_end)); 574fbf59bc9STejun Heo 575fbf59bc9STejun Heo for_each_possible_cpu(cpu) 576fbf59bc9STejun Heo unmap_kernel_range_noflush( 577fbf59bc9STejun Heo pcpu_chunk_addr(chunk, cpu, page_start), 578fbf59bc9STejun Heo (page_end - page_start) << PAGE_SHIFT); 579fbf59bc9STejun Heo 580fbf59bc9STejun Heo /* ditto as flush_cache_vunmap() */ 58185ae87c1STejun Heo if (flush_tlb) 582fbf59bc9STejun Heo flush_tlb_kernel_range(pcpu_chunk_addr(chunk, 0, page_start), 583fbf59bc9STejun Heo pcpu_chunk_addr(chunk, last, page_end)); 584fbf59bc9STejun Heo } 585fbf59bc9STejun Heo 586fbf59bc9STejun Heo /** 587fbf59bc9STejun Heo * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk 588fbf59bc9STejun Heo * @chunk: chunk to depopulate 589fbf59bc9STejun Heo * @off: offset to the area to depopulate 590cae3aeb8STejun Heo * @size: size of the area to depopulate in bytes 591fbf59bc9STejun Heo * @flush: whether to flush cache and tlb or not 592fbf59bc9STejun Heo * 593fbf59bc9STejun Heo * For each cpu, depopulate and unmap pages [@page_start,@page_end) 594fbf59bc9STejun Heo * from @chunk. If @flush is true, vcache is flushed before unmapping 595fbf59bc9STejun Heo * and tlb after. 596ccea34b5STejun Heo * 597ccea34b5STejun Heo * CONTEXT: 598ccea34b5STejun Heo * pcpu_alloc_mutex. 599fbf59bc9STejun Heo */ 600cae3aeb8STejun Heo static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size, 601cae3aeb8STejun Heo bool flush) 602fbf59bc9STejun Heo { 603fbf59bc9STejun Heo int page_start = PFN_DOWN(off); 604fbf59bc9STejun Heo int page_end = PFN_UP(off + size); 605fbf59bc9STejun Heo int unmap_start = -1; 606fbf59bc9STejun Heo int uninitialized_var(unmap_end); 607fbf59bc9STejun Heo unsigned int cpu; 608fbf59bc9STejun Heo int i; 609fbf59bc9STejun Heo 610fbf59bc9STejun Heo for (i = page_start; i < page_end; i++) { 611fbf59bc9STejun Heo for_each_possible_cpu(cpu) { 612fbf59bc9STejun Heo struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i); 613fbf59bc9STejun Heo 614fbf59bc9STejun Heo if (!*pagep) 615fbf59bc9STejun Heo continue; 616fbf59bc9STejun Heo 617fbf59bc9STejun Heo __free_page(*pagep); 618fbf59bc9STejun Heo 619fbf59bc9STejun Heo /* 620fbf59bc9STejun Heo * If it's partial depopulation, it might get 621fbf59bc9STejun Heo * populated or depopulated again. Mark the 622fbf59bc9STejun Heo * page gone. 623fbf59bc9STejun Heo */ 624fbf59bc9STejun Heo *pagep = NULL; 625fbf59bc9STejun Heo 626fbf59bc9STejun Heo unmap_start = unmap_start < 0 ? i : unmap_start; 627fbf59bc9STejun Heo unmap_end = i + 1; 628fbf59bc9STejun Heo } 629fbf59bc9STejun Heo } 630fbf59bc9STejun Heo 631fbf59bc9STejun Heo if (unmap_start >= 0) 632fbf59bc9STejun Heo pcpu_unmap(chunk, unmap_start, unmap_end, flush); 633fbf59bc9STejun Heo } 634fbf59bc9STejun Heo 635fbf59bc9STejun Heo /** 636fbf59bc9STejun Heo * pcpu_map - map pages into a pcpu_chunk 637fbf59bc9STejun Heo * @chunk: chunk of interest 638fbf59bc9STejun Heo * @page_start: page index of the first page to map 639fbf59bc9STejun Heo * @page_end: page index of the last page to map + 1 640fbf59bc9STejun Heo * 641fbf59bc9STejun Heo * For each cpu, map pages [@page_start,@page_end) into @chunk. 642fbf59bc9STejun Heo * vcache is flushed afterwards. 643fbf59bc9STejun Heo */ 644fbf59bc9STejun Heo static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end) 645fbf59bc9STejun Heo { 646fbf59bc9STejun Heo unsigned int last = num_possible_cpus() - 1; 647fbf59bc9STejun Heo unsigned int cpu; 648fbf59bc9STejun Heo int err; 649fbf59bc9STejun Heo 6508d408b4bSTejun Heo /* map must not be done on immutable chunk */ 6518d408b4bSTejun Heo WARN_ON(chunk->immutable); 6528d408b4bSTejun Heo 653fbf59bc9STejun Heo for_each_possible_cpu(cpu) { 654fbf59bc9STejun Heo err = map_kernel_range_noflush( 655fbf59bc9STejun Heo pcpu_chunk_addr(chunk, cpu, page_start), 656fbf59bc9STejun Heo (page_end - page_start) << PAGE_SHIFT, 657fbf59bc9STejun Heo PAGE_KERNEL, 658fbf59bc9STejun Heo pcpu_chunk_pagep(chunk, cpu, page_start)); 659fbf59bc9STejun Heo if (err < 0) 660fbf59bc9STejun Heo return err; 661fbf59bc9STejun Heo } 662fbf59bc9STejun Heo 663fbf59bc9STejun Heo /* flush at once, please read comments in pcpu_unmap() */ 664fbf59bc9STejun Heo flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start), 665fbf59bc9STejun Heo pcpu_chunk_addr(chunk, last, page_end)); 666fbf59bc9STejun Heo return 0; 667fbf59bc9STejun Heo } 668fbf59bc9STejun Heo 669fbf59bc9STejun Heo /** 670fbf59bc9STejun Heo * pcpu_populate_chunk - populate and map an area of a pcpu_chunk 671fbf59bc9STejun Heo * @chunk: chunk of interest 672fbf59bc9STejun Heo * @off: offset to the area to populate 673cae3aeb8STejun Heo * @size: size of the area to populate in bytes 674fbf59bc9STejun Heo * 675fbf59bc9STejun Heo * For each cpu, populate and map pages [@page_start,@page_end) into 676fbf59bc9STejun Heo * @chunk. The area is cleared on return. 677ccea34b5STejun Heo * 678ccea34b5STejun Heo * CONTEXT: 679ccea34b5STejun Heo * pcpu_alloc_mutex, does GFP_KERNEL allocation. 680fbf59bc9STejun Heo */ 681fbf59bc9STejun Heo static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size) 682fbf59bc9STejun Heo { 683fbf59bc9STejun Heo const gfp_t alloc_mask = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD; 684fbf59bc9STejun Heo int page_start = PFN_DOWN(off); 685fbf59bc9STejun Heo int page_end = PFN_UP(off + size); 686fbf59bc9STejun Heo int map_start = -1; 68702d51fdfSTejun Heo int uninitialized_var(map_end); 688fbf59bc9STejun Heo unsigned int cpu; 689fbf59bc9STejun Heo int i; 690fbf59bc9STejun Heo 691fbf59bc9STejun Heo for (i = page_start; i < page_end; i++) { 692fbf59bc9STejun Heo if (pcpu_chunk_page_occupied(chunk, i)) { 693fbf59bc9STejun Heo if (map_start >= 0) { 694fbf59bc9STejun Heo if (pcpu_map(chunk, map_start, map_end)) 695fbf59bc9STejun Heo goto err; 696fbf59bc9STejun Heo map_start = -1; 697fbf59bc9STejun Heo } 698fbf59bc9STejun Heo continue; 699fbf59bc9STejun Heo } 700fbf59bc9STejun Heo 701fbf59bc9STejun Heo map_start = map_start < 0 ? i : map_start; 702fbf59bc9STejun Heo map_end = i + 1; 703fbf59bc9STejun Heo 704fbf59bc9STejun Heo for_each_possible_cpu(cpu) { 705fbf59bc9STejun Heo struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i); 706fbf59bc9STejun Heo 707fbf59bc9STejun Heo *pagep = alloc_pages_node(cpu_to_node(cpu), 708fbf59bc9STejun Heo alloc_mask, 0); 709fbf59bc9STejun Heo if (!*pagep) 710fbf59bc9STejun Heo goto err; 711e1b9aa3fSChristoph Lameter pcpu_set_page_chunk(*pagep, chunk); 712fbf59bc9STejun Heo } 713fbf59bc9STejun Heo } 714fbf59bc9STejun Heo 715fbf59bc9STejun Heo if (map_start >= 0 && pcpu_map(chunk, map_start, map_end)) 716fbf59bc9STejun Heo goto err; 717fbf59bc9STejun Heo 718fbf59bc9STejun Heo for_each_possible_cpu(cpu) 719d9b55eebSTejun Heo memset(chunk->vm->addr + cpu * pcpu_unit_size + off, 0, 720fbf59bc9STejun Heo size); 721fbf59bc9STejun Heo 722fbf59bc9STejun Heo return 0; 723fbf59bc9STejun Heo err: 724fbf59bc9STejun Heo /* likely under heavy memory pressure, give memory back */ 725fbf59bc9STejun Heo pcpu_depopulate_chunk(chunk, off, size, true); 726fbf59bc9STejun Heo return -ENOMEM; 727fbf59bc9STejun Heo } 728fbf59bc9STejun Heo 729fbf59bc9STejun Heo static void free_pcpu_chunk(struct pcpu_chunk *chunk) 730fbf59bc9STejun Heo { 731fbf59bc9STejun Heo if (!chunk) 732fbf59bc9STejun Heo return; 733fbf59bc9STejun Heo if (chunk->vm) 734fbf59bc9STejun Heo free_vm_area(chunk->vm); 7351880d93bSTejun Heo pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0])); 736fbf59bc9STejun Heo kfree(chunk); 737fbf59bc9STejun Heo } 738fbf59bc9STejun Heo 739fbf59bc9STejun Heo static struct pcpu_chunk *alloc_pcpu_chunk(void) 740fbf59bc9STejun Heo { 741fbf59bc9STejun Heo struct pcpu_chunk *chunk; 742fbf59bc9STejun Heo 743fbf59bc9STejun Heo chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL); 744fbf59bc9STejun Heo if (!chunk) 745fbf59bc9STejun Heo return NULL; 746fbf59bc9STejun Heo 7471880d93bSTejun Heo chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0])); 748fbf59bc9STejun Heo chunk->map_alloc = PCPU_DFL_MAP_ALLOC; 749fbf59bc9STejun Heo chunk->map[chunk->map_used++] = pcpu_unit_size; 7503e24aa58STejun Heo chunk->page = chunk->page_ar; 751fbf59bc9STejun Heo 752fbf59bc9STejun Heo chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL); 753fbf59bc9STejun Heo if (!chunk->vm) { 754fbf59bc9STejun Heo free_pcpu_chunk(chunk); 755fbf59bc9STejun Heo return NULL; 756fbf59bc9STejun Heo } 757fbf59bc9STejun Heo 758fbf59bc9STejun Heo INIT_LIST_HEAD(&chunk->list); 759fbf59bc9STejun Heo chunk->free_size = pcpu_unit_size; 760fbf59bc9STejun Heo chunk->contig_hint = pcpu_unit_size; 761fbf59bc9STejun Heo 762fbf59bc9STejun Heo return chunk; 763fbf59bc9STejun Heo } 764fbf59bc9STejun Heo 765fbf59bc9STejun Heo /** 766edcb4639STejun Heo * pcpu_alloc - the percpu allocator 767cae3aeb8STejun Heo * @size: size of area to allocate in bytes 768fbf59bc9STejun Heo * @align: alignment of area (max PAGE_SIZE) 769edcb4639STejun Heo * @reserved: allocate from the reserved chunk if available 770fbf59bc9STejun Heo * 771ccea34b5STejun Heo * Allocate percpu area of @size bytes aligned at @align. 772ccea34b5STejun Heo * 773ccea34b5STejun Heo * CONTEXT: 774ccea34b5STejun Heo * Does GFP_KERNEL allocation. 775fbf59bc9STejun Heo * 776fbf59bc9STejun Heo * RETURNS: 777fbf59bc9STejun Heo * Percpu pointer to the allocated area on success, NULL on failure. 778fbf59bc9STejun Heo */ 779edcb4639STejun Heo static void *pcpu_alloc(size_t size, size_t align, bool reserved) 780fbf59bc9STejun Heo { 781fbf59bc9STejun Heo struct pcpu_chunk *chunk; 782fbf59bc9STejun Heo int slot, off; 783fbf59bc9STejun Heo 7848d408b4bSTejun Heo if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { 785fbf59bc9STejun Heo WARN(true, "illegal size (%zu) or align (%zu) for " 786fbf59bc9STejun Heo "percpu allocation\n", size, align); 787fbf59bc9STejun Heo return NULL; 788fbf59bc9STejun Heo } 789fbf59bc9STejun Heo 790ccea34b5STejun Heo mutex_lock(&pcpu_alloc_mutex); 791ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 792fbf59bc9STejun Heo 793edcb4639STejun Heo /* serve reserved allocations from the reserved chunk if available */ 794edcb4639STejun Heo if (reserved && pcpu_reserved_chunk) { 795edcb4639STejun Heo chunk = pcpu_reserved_chunk; 7969f7dcf22STejun Heo if (size > chunk->contig_hint || 7979f7dcf22STejun Heo pcpu_extend_area_map(chunk) < 0) 798ccea34b5STejun Heo goto fail_unlock; 799edcb4639STejun Heo off = pcpu_alloc_area(chunk, size, align); 800edcb4639STejun Heo if (off >= 0) 801edcb4639STejun Heo goto area_found; 802ccea34b5STejun Heo goto fail_unlock; 803edcb4639STejun Heo } 804edcb4639STejun Heo 805ccea34b5STejun Heo restart: 806edcb4639STejun Heo /* search through normal chunks */ 807fbf59bc9STejun Heo for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { 808fbf59bc9STejun Heo list_for_each_entry(chunk, &pcpu_slot[slot], list) { 809fbf59bc9STejun Heo if (size > chunk->contig_hint) 810fbf59bc9STejun Heo continue; 811ccea34b5STejun Heo 812ccea34b5STejun Heo switch (pcpu_extend_area_map(chunk)) { 813ccea34b5STejun Heo case 0: 814ccea34b5STejun Heo break; 815ccea34b5STejun Heo case 1: 816ccea34b5STejun Heo goto restart; /* pcpu_lock dropped, restart */ 817ccea34b5STejun Heo default: 818ccea34b5STejun Heo goto fail_unlock; 819ccea34b5STejun Heo } 820ccea34b5STejun Heo 821fbf59bc9STejun Heo off = pcpu_alloc_area(chunk, size, align); 822fbf59bc9STejun Heo if (off >= 0) 823fbf59bc9STejun Heo goto area_found; 824fbf59bc9STejun Heo } 825fbf59bc9STejun Heo } 826fbf59bc9STejun Heo 827fbf59bc9STejun Heo /* hmmm... no space left, create a new chunk */ 828ccea34b5STejun Heo spin_unlock_irq(&pcpu_lock); 829ccea34b5STejun Heo 830fbf59bc9STejun Heo chunk = alloc_pcpu_chunk(); 831fbf59bc9STejun Heo if (!chunk) 832ccea34b5STejun Heo goto fail_unlock_mutex; 833ccea34b5STejun Heo 834ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 835fbf59bc9STejun Heo pcpu_chunk_relocate(chunk, -1); 836ccea34b5STejun Heo goto restart; 837fbf59bc9STejun Heo 838fbf59bc9STejun Heo area_found: 839ccea34b5STejun Heo spin_unlock_irq(&pcpu_lock); 840ccea34b5STejun Heo 841fbf59bc9STejun Heo /* populate, map and clear the area */ 842fbf59bc9STejun Heo if (pcpu_populate_chunk(chunk, off, size)) { 843ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 844fbf59bc9STejun Heo pcpu_free_area(chunk, off); 845ccea34b5STejun Heo goto fail_unlock; 846fbf59bc9STejun Heo } 847fbf59bc9STejun Heo 848ccea34b5STejun Heo mutex_unlock(&pcpu_alloc_mutex); 849ccea34b5STejun Heo 850ccea34b5STejun Heo return __addr_to_pcpu_ptr(chunk->vm->addr + off); 851ccea34b5STejun Heo 852ccea34b5STejun Heo fail_unlock: 853ccea34b5STejun Heo spin_unlock_irq(&pcpu_lock); 854ccea34b5STejun Heo fail_unlock_mutex: 855ccea34b5STejun Heo mutex_unlock(&pcpu_alloc_mutex); 856ccea34b5STejun Heo return NULL; 857fbf59bc9STejun Heo } 858edcb4639STejun Heo 859edcb4639STejun Heo /** 860edcb4639STejun Heo * __alloc_percpu - allocate dynamic percpu area 861edcb4639STejun Heo * @size: size of area to allocate in bytes 862edcb4639STejun Heo * @align: alignment of area (max PAGE_SIZE) 863edcb4639STejun Heo * 864edcb4639STejun Heo * Allocate percpu area of @size bytes aligned at @align. Might 865edcb4639STejun Heo * sleep. Might trigger writeouts. 866edcb4639STejun Heo * 867ccea34b5STejun Heo * CONTEXT: 868ccea34b5STejun Heo * Does GFP_KERNEL allocation. 869ccea34b5STejun Heo * 870edcb4639STejun Heo * RETURNS: 871edcb4639STejun Heo * Percpu pointer to the allocated area on success, NULL on failure. 872edcb4639STejun Heo */ 873edcb4639STejun Heo void *__alloc_percpu(size_t size, size_t align) 874edcb4639STejun Heo { 875edcb4639STejun Heo return pcpu_alloc(size, align, false); 876edcb4639STejun Heo } 877fbf59bc9STejun Heo EXPORT_SYMBOL_GPL(__alloc_percpu); 878fbf59bc9STejun Heo 879edcb4639STejun Heo /** 880edcb4639STejun Heo * __alloc_reserved_percpu - allocate reserved percpu area 881edcb4639STejun Heo * @size: size of area to allocate in bytes 882edcb4639STejun Heo * @align: alignment of area (max PAGE_SIZE) 883edcb4639STejun Heo * 884edcb4639STejun Heo * Allocate percpu area of @size bytes aligned at @align from reserved 885edcb4639STejun Heo * percpu area if arch has set it up; otherwise, allocation is served 886edcb4639STejun Heo * from the same dynamic area. Might sleep. Might trigger writeouts. 887edcb4639STejun Heo * 888ccea34b5STejun Heo * CONTEXT: 889ccea34b5STejun Heo * Does GFP_KERNEL allocation. 890ccea34b5STejun Heo * 891edcb4639STejun Heo * RETURNS: 892edcb4639STejun Heo * Percpu pointer to the allocated area on success, NULL on failure. 893edcb4639STejun Heo */ 894edcb4639STejun Heo void *__alloc_reserved_percpu(size_t size, size_t align) 895edcb4639STejun Heo { 896edcb4639STejun Heo return pcpu_alloc(size, align, true); 897edcb4639STejun Heo } 898edcb4639STejun Heo 899a56dbddfSTejun Heo /** 900a56dbddfSTejun Heo * pcpu_reclaim - reclaim fully free chunks, workqueue function 901a56dbddfSTejun Heo * @work: unused 902a56dbddfSTejun Heo * 903a56dbddfSTejun Heo * Reclaim all fully free chunks except for the first one. 904ccea34b5STejun Heo * 905ccea34b5STejun Heo * CONTEXT: 906ccea34b5STejun Heo * workqueue context. 907a56dbddfSTejun Heo */ 908a56dbddfSTejun Heo static void pcpu_reclaim(struct work_struct *work) 909fbf59bc9STejun Heo { 910a56dbddfSTejun Heo LIST_HEAD(todo); 911a56dbddfSTejun Heo struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1]; 912a56dbddfSTejun Heo struct pcpu_chunk *chunk, *next; 913a56dbddfSTejun Heo 914ccea34b5STejun Heo mutex_lock(&pcpu_alloc_mutex); 915ccea34b5STejun Heo spin_lock_irq(&pcpu_lock); 916a56dbddfSTejun Heo 917a56dbddfSTejun Heo list_for_each_entry_safe(chunk, next, head, list) { 9188d408b4bSTejun Heo WARN_ON(chunk->immutable); 919a56dbddfSTejun Heo 920a56dbddfSTejun Heo /* spare the first one */ 921a56dbddfSTejun Heo if (chunk == list_first_entry(head, struct pcpu_chunk, list)) 922a56dbddfSTejun Heo continue; 923a56dbddfSTejun Heo 924a56dbddfSTejun Heo list_move(&chunk->list, &todo); 925a56dbddfSTejun Heo } 926a56dbddfSTejun Heo 927ccea34b5STejun Heo spin_unlock_irq(&pcpu_lock); 928ccea34b5STejun Heo mutex_unlock(&pcpu_alloc_mutex); 929a56dbddfSTejun Heo 930a56dbddfSTejun Heo list_for_each_entry_safe(chunk, next, &todo, list) { 931a56dbddfSTejun Heo pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false); 932fbf59bc9STejun Heo free_pcpu_chunk(chunk); 933fbf59bc9STejun Heo } 934a56dbddfSTejun Heo } 935fbf59bc9STejun Heo 936fbf59bc9STejun Heo /** 937fbf59bc9STejun Heo * free_percpu - free percpu area 938fbf59bc9STejun Heo * @ptr: pointer to area to free 939fbf59bc9STejun Heo * 940ccea34b5STejun Heo * Free percpu area @ptr. 941ccea34b5STejun Heo * 942ccea34b5STejun Heo * CONTEXT: 943ccea34b5STejun Heo * Can be called from atomic context. 944fbf59bc9STejun Heo */ 945fbf59bc9STejun Heo void free_percpu(void *ptr) 946fbf59bc9STejun Heo { 947fbf59bc9STejun Heo void *addr = __pcpu_ptr_to_addr(ptr); 948fbf59bc9STejun Heo struct pcpu_chunk *chunk; 949ccea34b5STejun Heo unsigned long flags; 950fbf59bc9STejun Heo int off; 951fbf59bc9STejun Heo 952fbf59bc9STejun Heo if (!ptr) 953fbf59bc9STejun Heo return; 954fbf59bc9STejun Heo 955ccea34b5STejun Heo spin_lock_irqsave(&pcpu_lock, flags); 956fbf59bc9STejun Heo 957fbf59bc9STejun Heo chunk = pcpu_chunk_addr_search(addr); 958fbf59bc9STejun Heo off = addr - chunk->vm->addr; 959fbf59bc9STejun Heo 960fbf59bc9STejun Heo pcpu_free_area(chunk, off); 961fbf59bc9STejun Heo 962a56dbddfSTejun Heo /* if there are more than one fully free chunks, wake up grim reaper */ 963fbf59bc9STejun Heo if (chunk->free_size == pcpu_unit_size) { 964fbf59bc9STejun Heo struct pcpu_chunk *pos; 965fbf59bc9STejun Heo 966a56dbddfSTejun Heo list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list) 967fbf59bc9STejun Heo if (pos != chunk) { 968a56dbddfSTejun Heo schedule_work(&pcpu_reclaim_work); 969fbf59bc9STejun Heo break; 970fbf59bc9STejun Heo } 971fbf59bc9STejun Heo } 972fbf59bc9STejun Heo 973ccea34b5STejun Heo spin_unlock_irqrestore(&pcpu_lock, flags); 974fbf59bc9STejun Heo } 975fbf59bc9STejun Heo EXPORT_SYMBOL_GPL(free_percpu); 976fbf59bc9STejun Heo 977fbf59bc9STejun Heo /** 9788d408b4bSTejun Heo * pcpu_setup_first_chunk - initialize the first percpu chunk 9798d408b4bSTejun Heo * @get_page_fn: callback to fetch page pointer 9808d408b4bSTejun Heo * @static_size: the size of static percpu area in bytes 981edcb4639STejun Heo * @reserved_size: the size of reserved percpu area in bytes 982cafe8816STejun Heo * @dyn_size: free size for dynamic allocation in bytes, -1 for auto 9836074d5b0STejun Heo * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto 9848d408b4bSTejun Heo * @base_addr: mapped address, NULL for auto 9858d408b4bSTejun Heo * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary 986fbf59bc9STejun Heo * 9878d408b4bSTejun Heo * Initialize the first percpu chunk which contains the kernel static 9888d408b4bSTejun Heo * perpcu area. This function is to be called from arch percpu area 9898d408b4bSTejun Heo * setup path. The first two parameters are mandatory. The rest are 9908d408b4bSTejun Heo * optional. 9918d408b4bSTejun Heo * 9928d408b4bSTejun Heo * @get_page_fn() should return pointer to percpu page given cpu 9938d408b4bSTejun Heo * number and page number. It should at least return enough pages to 9948d408b4bSTejun Heo * cover the static area. The returned pages for static area should 9958d408b4bSTejun Heo * have been initialized with valid data. If @unit_size is specified, 9968d408b4bSTejun Heo * it can also return pages after the static area. NULL return 9978d408b4bSTejun Heo * indicates end of pages for the cpu. Note that @get_page_fn() must 9988d408b4bSTejun Heo * return the same number of pages for all cpus. 9998d408b4bSTejun Heo * 1000edcb4639STejun Heo * @reserved_size, if non-zero, specifies the amount of bytes to 1001edcb4639STejun Heo * reserve after the static area in the first chunk. This reserves 1002edcb4639STejun Heo * the first chunk such that it's available only through reserved 1003edcb4639STejun Heo * percpu allocation. This is primarily used to serve module percpu 1004edcb4639STejun Heo * static areas on architectures where the addressing model has 1005edcb4639STejun Heo * limited offset range for symbol relocations to guarantee module 1006edcb4639STejun Heo * percpu symbols fall inside the relocatable range. 1007edcb4639STejun Heo * 10086074d5b0STejun Heo * @dyn_size, if non-negative, determines the number of bytes 10096074d5b0STejun Heo * available for dynamic allocation in the first chunk. Specifying 10106074d5b0STejun Heo * non-negative value makes percpu leave alone the area beyond 10116074d5b0STejun Heo * @static_size + @reserved_size + @dyn_size. 10126074d5b0STejun Heo * 1013cafe8816STejun Heo * @unit_size, if non-negative, specifies unit size and must be 1014cafe8816STejun Heo * aligned to PAGE_SIZE and equal to or larger than @static_size + 10156074d5b0STejun Heo * @reserved_size + if non-negative, @dyn_size. 10168d408b4bSTejun Heo * 10178d408b4bSTejun Heo * Non-null @base_addr means that the caller already allocated virtual 10188d408b4bSTejun Heo * region for the first chunk and mapped it. percpu must not mess 10198d408b4bSTejun Heo * with the chunk. Note that @base_addr with 0 @unit_size or non-NULL 10208d408b4bSTejun Heo * @populate_pte_fn doesn't make any sense. 10218d408b4bSTejun Heo * 10228d408b4bSTejun Heo * @populate_pte_fn is used to populate the pagetable. NULL means the 10238d408b4bSTejun Heo * caller already populated the pagetable. 1024fbf59bc9STejun Heo * 1025edcb4639STejun Heo * If the first chunk ends up with both reserved and dynamic areas, it 1026edcb4639STejun Heo * is served by two chunks - one to serve the core static and reserved 1027edcb4639STejun Heo * areas and the other for the dynamic area. They share the same vm 1028edcb4639STejun Heo * and page map but uses different area allocation map to stay away 1029edcb4639STejun Heo * from each other. The latter chunk is circulated in the chunk slots 1030edcb4639STejun Heo * and available for dynamic allocation like any other chunks. 1031edcb4639STejun Heo * 1032fbf59bc9STejun Heo * RETURNS: 1033fbf59bc9STejun Heo * The determined pcpu_unit_size which can be used to initialize 1034fbf59bc9STejun Heo * percpu access. 1035fbf59bc9STejun Heo */ 10368d408b4bSTejun Heo size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 1037edcb4639STejun Heo size_t static_size, size_t reserved_size, 10386074d5b0STejun Heo ssize_t dyn_size, ssize_t unit_size, 1039cafe8816STejun Heo void *base_addr, 1040*d4b95f80STejun Heo pcpu_fc_populate_pte_fn_t populate_pte_fn) 1041fbf59bc9STejun Heo { 10422441d15cSTejun Heo static struct vm_struct first_vm; 1043edcb4639STejun Heo static int smap[2], dmap[2]; 10446074d5b0STejun Heo size_t size_sum = static_size + reserved_size + 10456074d5b0STejun Heo (dyn_size >= 0 ? dyn_size : 0); 1046edcb4639STejun Heo struct pcpu_chunk *schunk, *dchunk = NULL; 1047fbf59bc9STejun Heo unsigned int cpu; 10488d408b4bSTejun Heo int nr_pages; 1049fbf59bc9STejun Heo int err, i; 1050fbf59bc9STejun Heo 10518d408b4bSTejun Heo /* santiy checks */ 1052edcb4639STejun Heo BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC || 1053edcb4639STejun Heo ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); 10548d408b4bSTejun Heo BUG_ON(!static_size); 1055cafe8816STejun Heo if (unit_size >= 0) { 10566074d5b0STejun Heo BUG_ON(unit_size < size_sum); 10578d408b4bSTejun Heo BUG_ON(unit_size & ~PAGE_MASK); 10586074d5b0STejun Heo BUG_ON(unit_size < PCPU_MIN_UNIT_SIZE); 10596074d5b0STejun Heo } else 1060cafe8816STejun Heo BUG_ON(base_addr); 10618d408b4bSTejun Heo BUG_ON(base_addr && populate_pte_fn); 1062fbf59bc9STejun Heo 1063cafe8816STejun Heo if (unit_size >= 0) 10648d408b4bSTejun Heo pcpu_unit_pages = unit_size >> PAGE_SHIFT; 10658d408b4bSTejun Heo else 10668d408b4bSTejun Heo pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, 10676074d5b0STejun Heo PFN_UP(size_sum)); 10688d408b4bSTejun Heo 1069d9b55eebSTejun Heo pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; 1070fbf59bc9STejun Heo pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; 1071fbf59bc9STejun Heo pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) 1072cb83b42eSTejun Heo + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *); 1073fbf59bc9STejun Heo 1074cafe8816STejun Heo if (dyn_size < 0) 1075edcb4639STejun Heo dyn_size = pcpu_unit_size - static_size - reserved_size; 1076cafe8816STejun Heo 1077d9b55eebSTejun Heo /* 1078d9b55eebSTejun Heo * Allocate chunk slots. The additional last slot is for 1079d9b55eebSTejun Heo * empty chunks. 1080d9b55eebSTejun Heo */ 1081d9b55eebSTejun Heo pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2; 1082fbf59bc9STejun Heo pcpu_slot = alloc_bootmem(pcpu_nr_slots * sizeof(pcpu_slot[0])); 1083fbf59bc9STejun Heo for (i = 0; i < pcpu_nr_slots; i++) 1084fbf59bc9STejun Heo INIT_LIST_HEAD(&pcpu_slot[i]); 1085fbf59bc9STejun Heo 1086edcb4639STejun Heo /* 1087edcb4639STejun Heo * Initialize static chunk. If reserved_size is zero, the 1088edcb4639STejun Heo * static chunk covers static area + dynamic allocation area 1089edcb4639STejun Heo * in the first chunk. If reserved_size is not zero, it 1090edcb4639STejun Heo * covers static area + reserved area (mostly used for module 1091edcb4639STejun Heo * static percpu allocation). 1092edcb4639STejun Heo */ 10932441d15cSTejun Heo schunk = alloc_bootmem(pcpu_chunk_struct_size); 10942441d15cSTejun Heo INIT_LIST_HEAD(&schunk->list); 10952441d15cSTejun Heo schunk->vm = &first_vm; 109661ace7faSTejun Heo schunk->map = smap; 109761ace7faSTejun Heo schunk->map_alloc = ARRAY_SIZE(smap); 10983e24aa58STejun Heo schunk->page = schunk->page_ar; 1099edcb4639STejun Heo 1100edcb4639STejun Heo if (reserved_size) { 1101edcb4639STejun Heo schunk->free_size = reserved_size; 1102ae9e6bc9STejun Heo pcpu_reserved_chunk = schunk; 1103ae9e6bc9STejun Heo pcpu_reserved_chunk_limit = static_size + reserved_size; 1104edcb4639STejun Heo } else { 11052441d15cSTejun Heo schunk->free_size = dyn_size; 1106edcb4639STejun Heo dyn_size = 0; /* dynamic area covered */ 1107edcb4639STejun Heo } 11082441d15cSTejun Heo schunk->contig_hint = schunk->free_size; 1109fbf59bc9STejun Heo 111061ace7faSTejun Heo schunk->map[schunk->map_used++] = -static_size; 111161ace7faSTejun Heo if (schunk->free_size) 111261ace7faSTejun Heo schunk->map[schunk->map_used++] = schunk->free_size; 111361ace7faSTejun Heo 1114edcb4639STejun Heo /* init dynamic chunk if necessary */ 1115edcb4639STejun Heo if (dyn_size) { 1116edcb4639STejun Heo dchunk = alloc_bootmem(sizeof(struct pcpu_chunk)); 1117edcb4639STejun Heo INIT_LIST_HEAD(&dchunk->list); 1118edcb4639STejun Heo dchunk->vm = &first_vm; 1119edcb4639STejun Heo dchunk->map = dmap; 1120edcb4639STejun Heo dchunk->map_alloc = ARRAY_SIZE(dmap); 1121edcb4639STejun Heo dchunk->page = schunk->page_ar; /* share page map with schunk */ 1122edcb4639STejun Heo 1123edcb4639STejun Heo dchunk->contig_hint = dchunk->free_size = dyn_size; 1124edcb4639STejun Heo dchunk->map[dchunk->map_used++] = -pcpu_reserved_chunk_limit; 1125edcb4639STejun Heo dchunk->map[dchunk->map_used++] = dchunk->free_size; 1126edcb4639STejun Heo } 1127edcb4639STejun Heo 11288d408b4bSTejun Heo /* allocate vm address */ 11292441d15cSTejun Heo first_vm.flags = VM_ALLOC; 11302441d15cSTejun Heo first_vm.size = pcpu_chunk_size; 11318d408b4bSTejun Heo 11328d408b4bSTejun Heo if (!base_addr) 11332441d15cSTejun Heo vm_area_register_early(&first_vm, PAGE_SIZE); 11348d408b4bSTejun Heo else { 11358d408b4bSTejun Heo /* 11368d408b4bSTejun Heo * Pages already mapped. No need to remap into 1137edcb4639STejun Heo * vmalloc area. In this case the first chunks can't 1138edcb4639STejun Heo * be mapped or unmapped by percpu and are marked 11398d408b4bSTejun Heo * immutable. 11408d408b4bSTejun Heo */ 11412441d15cSTejun Heo first_vm.addr = base_addr; 11422441d15cSTejun Heo schunk->immutable = true; 1143edcb4639STejun Heo if (dchunk) 1144edcb4639STejun Heo dchunk->immutable = true; 1145fbf59bc9STejun Heo } 1146fbf59bc9STejun Heo 11478d408b4bSTejun Heo /* assign pages */ 11488d408b4bSTejun Heo nr_pages = -1; 11498d408b4bSTejun Heo for_each_possible_cpu(cpu) { 11508d408b4bSTejun Heo for (i = 0; i < pcpu_unit_pages; i++) { 11518d408b4bSTejun Heo struct page *page = get_page_fn(cpu, i); 11528d408b4bSTejun Heo 11538d408b4bSTejun Heo if (!page) 11548d408b4bSTejun Heo break; 11552441d15cSTejun Heo *pcpu_chunk_pagep(schunk, cpu, i) = page; 11568d408b4bSTejun Heo } 11578d408b4bSTejun Heo 115861ace7faSTejun Heo BUG_ON(i < PFN_UP(static_size)); 11598d408b4bSTejun Heo 11608d408b4bSTejun Heo if (nr_pages < 0) 11618d408b4bSTejun Heo nr_pages = i; 11628d408b4bSTejun Heo else 11638d408b4bSTejun Heo BUG_ON(nr_pages != i); 11648d408b4bSTejun Heo } 11658d408b4bSTejun Heo 11668d408b4bSTejun Heo /* map them */ 11678d408b4bSTejun Heo if (populate_pte_fn) { 11688d408b4bSTejun Heo for_each_possible_cpu(cpu) 11698d408b4bSTejun Heo for (i = 0; i < nr_pages; i++) 11702441d15cSTejun Heo populate_pte_fn(pcpu_chunk_addr(schunk, 11718d408b4bSTejun Heo cpu, i)); 11728d408b4bSTejun Heo 11732441d15cSTejun Heo err = pcpu_map(schunk, 0, nr_pages); 1174fbf59bc9STejun Heo if (err) 11758d408b4bSTejun Heo panic("failed to setup static percpu area, err=%d\n", 11768d408b4bSTejun Heo err); 11778d408b4bSTejun Heo } 1178fbf59bc9STejun Heo 11792441d15cSTejun Heo /* link the first chunk in */ 1180ae9e6bc9STejun Heo pcpu_first_chunk = dchunk ?: schunk; 1181ae9e6bc9STejun Heo pcpu_chunk_relocate(pcpu_first_chunk, -1); 1182fbf59bc9STejun Heo 1183fbf59bc9STejun Heo /* we're done */ 11842441d15cSTejun Heo pcpu_base_addr = (void *)pcpu_chunk_addr(schunk, 0, 0); 1185fbf59bc9STejun Heo return pcpu_unit_size; 1186fbf59bc9STejun Heo } 118766c3a757STejun Heo 118866c3a757STejun Heo /* 118966c3a757STejun Heo * Embedding first chunk setup helper. 119066c3a757STejun Heo */ 119166c3a757STejun Heo static void *pcpue_ptr __initdata; 119266c3a757STejun Heo static size_t pcpue_size __initdata; 119366c3a757STejun Heo static size_t pcpue_unit_size __initdata; 119466c3a757STejun Heo 119566c3a757STejun Heo static struct page * __init pcpue_get_page(unsigned int cpu, int pageno) 119666c3a757STejun Heo { 119766c3a757STejun Heo size_t off = (size_t)pageno << PAGE_SHIFT; 119866c3a757STejun Heo 119966c3a757STejun Heo if (off >= pcpue_size) 120066c3a757STejun Heo return NULL; 120166c3a757STejun Heo 120266c3a757STejun Heo return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off); 120366c3a757STejun Heo } 120466c3a757STejun Heo 120566c3a757STejun Heo /** 120666c3a757STejun Heo * pcpu_embed_first_chunk - embed the first percpu chunk into bootmem 120766c3a757STejun Heo * @static_size: the size of static percpu area in bytes 120866c3a757STejun Heo * @reserved_size: the size of reserved percpu area in bytes 120966c3a757STejun Heo * @dyn_size: free size for dynamic allocation in bytes, -1 for auto 121066c3a757STejun Heo * 121166c3a757STejun Heo * This is a helper to ease setting up embedded first percpu chunk and 121266c3a757STejun Heo * can be called where pcpu_setup_first_chunk() is expected. 121366c3a757STejun Heo * 121466c3a757STejun Heo * If this function is used to setup the first chunk, it is allocated 121566c3a757STejun Heo * as a contiguous area using bootmem allocator and used as-is without 121666c3a757STejun Heo * being mapped into vmalloc area. This enables the first chunk to 121766c3a757STejun Heo * piggy back on the linear physical mapping which often uses larger 121866c3a757STejun Heo * page size. 121966c3a757STejun Heo * 122066c3a757STejun Heo * When @dyn_size is positive, dynamic area might be larger than 1221788e5abcSTejun Heo * specified to fill page alignment. When @dyn_size is auto, 1222788e5abcSTejun Heo * @dyn_size is just big enough to fill page alignment after static 1223788e5abcSTejun Heo * and reserved areas. 122466c3a757STejun Heo * 122566c3a757STejun Heo * If the needed size is smaller than the minimum or specified unit 122666c3a757STejun Heo * size, the leftover is returned to the bootmem allocator. 122766c3a757STejun Heo * 122866c3a757STejun Heo * RETURNS: 122966c3a757STejun Heo * The determined pcpu_unit_size which can be used to initialize 123066c3a757STejun Heo * percpu access on success, -errno on failure. 123166c3a757STejun Heo */ 123266c3a757STejun Heo ssize_t __init pcpu_embed_first_chunk(size_t static_size, size_t reserved_size, 1233788e5abcSTejun Heo ssize_t dyn_size) 123466c3a757STejun Heo { 1235fa8a7094STejun Heo size_t chunk_size; 123666c3a757STejun Heo unsigned int cpu; 123766c3a757STejun Heo 123866c3a757STejun Heo /* determine parameters and allocate */ 123966c3a757STejun Heo pcpue_size = PFN_ALIGN(static_size + reserved_size + 124066c3a757STejun Heo (dyn_size >= 0 ? dyn_size : 0)); 124166c3a757STejun Heo if (dyn_size != 0) 124266c3a757STejun Heo dyn_size = pcpue_size - static_size - reserved_size; 124366c3a757STejun Heo 124466c3a757STejun Heo pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE); 1245fa8a7094STejun Heo chunk_size = pcpue_unit_size * num_possible_cpus(); 1246fa8a7094STejun Heo 1247fa8a7094STejun Heo pcpue_ptr = __alloc_bootmem_nopanic(chunk_size, PAGE_SIZE, 1248fa8a7094STejun Heo __pa(MAX_DMA_ADDRESS)); 1249fa8a7094STejun Heo if (!pcpue_ptr) { 1250fa8a7094STejun Heo pr_warning("PERCPU: failed to allocate %zu bytes for " 1251fa8a7094STejun Heo "embedding\n", chunk_size); 125266c3a757STejun Heo return -ENOMEM; 1253fa8a7094STejun Heo } 125466c3a757STejun Heo 125566c3a757STejun Heo /* return the leftover and copy */ 125666c3a757STejun Heo for_each_possible_cpu(cpu) { 125766c3a757STejun Heo void *ptr = pcpue_ptr + cpu * pcpue_unit_size; 125866c3a757STejun Heo 125966c3a757STejun Heo free_bootmem(__pa(ptr + pcpue_size), 126066c3a757STejun Heo pcpue_unit_size - pcpue_size); 126166c3a757STejun Heo memcpy(ptr, __per_cpu_load, static_size); 126266c3a757STejun Heo } 126366c3a757STejun Heo 126466c3a757STejun Heo /* we're ready, commit */ 126566c3a757STejun Heo pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n", 126666c3a757STejun Heo pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); 126766c3a757STejun Heo 126866c3a757STejun Heo return pcpu_setup_first_chunk(pcpue_get_page, static_size, 126966c3a757STejun Heo reserved_size, dyn_size, 127066c3a757STejun Heo pcpue_unit_size, pcpue_ptr, NULL); 127166c3a757STejun Heo } 1272e74e3962STejun Heo 1273e74e3962STejun Heo /* 1274*d4b95f80STejun Heo * 4k page first chunk setup helper. 1275*d4b95f80STejun Heo */ 1276*d4b95f80STejun Heo static struct page **pcpu4k_pages __initdata; 1277*d4b95f80STejun Heo static int pcpu4k_nr_static_pages __initdata; 1278*d4b95f80STejun Heo 1279*d4b95f80STejun Heo static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno) 1280*d4b95f80STejun Heo { 1281*d4b95f80STejun Heo if (pageno < pcpu4k_nr_static_pages) 1282*d4b95f80STejun Heo return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno]; 1283*d4b95f80STejun Heo return NULL; 1284*d4b95f80STejun Heo } 1285*d4b95f80STejun Heo 1286*d4b95f80STejun Heo /** 1287*d4b95f80STejun Heo * pcpu_4k_first_chunk - map the first chunk using PAGE_SIZE pages 1288*d4b95f80STejun Heo * @static_size: the size of static percpu area in bytes 1289*d4b95f80STejun Heo * @reserved_size: the size of reserved percpu area in bytes 1290*d4b95f80STejun Heo * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE 1291*d4b95f80STejun Heo * @free_fn: funtion to free percpu page, always called with PAGE_SIZE 1292*d4b95f80STejun Heo * @populate_pte_fn: function to populate pte 1293*d4b95f80STejun Heo * 1294*d4b95f80STejun Heo * This is a helper to ease setting up embedded first percpu chunk and 1295*d4b95f80STejun Heo * can be called where pcpu_setup_first_chunk() is expected. 1296*d4b95f80STejun Heo * 1297*d4b95f80STejun Heo * This is the basic allocator. Static percpu area is allocated 1298*d4b95f80STejun Heo * page-by-page into vmalloc area. 1299*d4b95f80STejun Heo * 1300*d4b95f80STejun Heo * RETURNS: 1301*d4b95f80STejun Heo * The determined pcpu_unit_size which can be used to initialize 1302*d4b95f80STejun Heo * percpu access on success, -errno on failure. 1303*d4b95f80STejun Heo */ 1304*d4b95f80STejun Heo ssize_t __init pcpu_4k_first_chunk(size_t static_size, size_t reserved_size, 1305*d4b95f80STejun Heo pcpu_fc_alloc_fn_t alloc_fn, 1306*d4b95f80STejun Heo pcpu_fc_free_fn_t free_fn, 1307*d4b95f80STejun Heo pcpu_fc_populate_pte_fn_t populate_pte_fn) 1308*d4b95f80STejun Heo { 1309*d4b95f80STejun Heo size_t pages_size; 1310*d4b95f80STejun Heo unsigned int cpu; 1311*d4b95f80STejun Heo int i, j; 1312*d4b95f80STejun Heo ssize_t ret; 1313*d4b95f80STejun Heo 1314*d4b95f80STejun Heo pcpu4k_nr_static_pages = PFN_UP(static_size); 1315*d4b95f80STejun Heo 1316*d4b95f80STejun Heo /* unaligned allocations can't be freed, round up to page size */ 1317*d4b95f80STejun Heo pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * num_possible_cpus() * 1318*d4b95f80STejun Heo sizeof(pcpu4k_pages[0])); 1319*d4b95f80STejun Heo pcpu4k_pages = alloc_bootmem(pages_size); 1320*d4b95f80STejun Heo 1321*d4b95f80STejun Heo /* allocate and copy */ 1322*d4b95f80STejun Heo j = 0; 1323*d4b95f80STejun Heo for_each_possible_cpu(cpu) 1324*d4b95f80STejun Heo for (i = 0; i < pcpu4k_nr_static_pages; i++) { 1325*d4b95f80STejun Heo void *ptr; 1326*d4b95f80STejun Heo 1327*d4b95f80STejun Heo ptr = alloc_fn(cpu, PAGE_SIZE); 1328*d4b95f80STejun Heo if (!ptr) { 1329*d4b95f80STejun Heo pr_warning("PERCPU: failed to allocate " 1330*d4b95f80STejun Heo "4k page for cpu%u\n", cpu); 1331*d4b95f80STejun Heo goto enomem; 1332*d4b95f80STejun Heo } 1333*d4b95f80STejun Heo 1334*d4b95f80STejun Heo memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE); 1335*d4b95f80STejun Heo pcpu4k_pages[j++] = virt_to_page(ptr); 1336*d4b95f80STejun Heo } 1337*d4b95f80STejun Heo 1338*d4b95f80STejun Heo /* we're ready, commit */ 1339*d4b95f80STejun Heo pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n", 1340*d4b95f80STejun Heo pcpu4k_nr_static_pages, static_size); 1341*d4b95f80STejun Heo 1342*d4b95f80STejun Heo ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, 1343*d4b95f80STejun Heo reserved_size, -1, 1344*d4b95f80STejun Heo -1, NULL, populate_pte_fn); 1345*d4b95f80STejun Heo goto out_free_ar; 1346*d4b95f80STejun Heo 1347*d4b95f80STejun Heo enomem: 1348*d4b95f80STejun Heo while (--j >= 0) 1349*d4b95f80STejun Heo free_fn(page_address(pcpu4k_pages[j]), PAGE_SIZE); 1350*d4b95f80STejun Heo ret = -ENOMEM; 1351*d4b95f80STejun Heo out_free_ar: 1352*d4b95f80STejun Heo free_bootmem(__pa(pcpu4k_pages), pages_size); 1353*d4b95f80STejun Heo return ret; 1354*d4b95f80STejun Heo } 1355*d4b95f80STejun Heo 1356*d4b95f80STejun Heo /* 1357e74e3962STejun Heo * Generic percpu area setup. 1358e74e3962STejun Heo * 1359e74e3962STejun Heo * The embedding helper is used because its behavior closely resembles 1360e74e3962STejun Heo * the original non-dynamic generic percpu area setup. This is 1361e74e3962STejun Heo * important because many archs have addressing restrictions and might 1362e74e3962STejun Heo * fail if the percpu area is located far away from the previous 1363e74e3962STejun Heo * location. As an added bonus, in non-NUMA cases, embedding is 1364e74e3962STejun Heo * generally a good idea TLB-wise because percpu area can piggy back 1365e74e3962STejun Heo * on the physical linear memory mapping which uses large page 1366e74e3962STejun Heo * mappings on applicable archs. 1367e74e3962STejun Heo */ 1368e74e3962STejun Heo #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA 1369e74e3962STejun Heo unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 1370e74e3962STejun Heo EXPORT_SYMBOL(__per_cpu_offset); 1371e74e3962STejun Heo 1372e74e3962STejun Heo void __init setup_per_cpu_areas(void) 1373e74e3962STejun Heo { 1374e74e3962STejun Heo size_t static_size = __per_cpu_end - __per_cpu_start; 1375e74e3962STejun Heo ssize_t unit_size; 1376e74e3962STejun Heo unsigned long delta; 1377e74e3962STejun Heo unsigned int cpu; 1378e74e3962STejun Heo 1379e74e3962STejun Heo /* 1380e74e3962STejun Heo * Always reserve area for module percpu variables. That's 1381e74e3962STejun Heo * what the legacy allocator did. 1382e74e3962STejun Heo */ 1383e74e3962STejun Heo unit_size = pcpu_embed_first_chunk(static_size, PERCPU_MODULE_RESERVE, 1384788e5abcSTejun Heo PERCPU_DYNAMIC_RESERVE); 1385e74e3962STejun Heo if (unit_size < 0) 1386e74e3962STejun Heo panic("Failed to initialized percpu areas."); 1387e74e3962STejun Heo 1388e74e3962STejun Heo delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 1389e74e3962STejun Heo for_each_possible_cpu(cpu) 1390e74e3962STejun Heo __per_cpu_offset[cpu] = delta + cpu * unit_size; 1391e74e3962STejun Heo } 1392e74e3962STejun Heo #endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */ 1393