19a001fc1SVitaly Wool /* 29a001fc1SVitaly Wool * z3fold.c 39a001fc1SVitaly Wool * 49a001fc1SVitaly Wool * Author: Vitaly Wool <vitaly.wool@konsulko.com> 59a001fc1SVitaly Wool * Copyright (C) 2016, Sony Mobile Communications Inc. 69a001fc1SVitaly Wool * 79a001fc1SVitaly Wool * This implementation is based on zbud written by Seth Jennings. 89a001fc1SVitaly Wool * 99a001fc1SVitaly Wool * z3fold is an special purpose allocator for storing compressed pages. It 109a001fc1SVitaly Wool * can store up to three compressed pages per page which improves the 119a001fc1SVitaly Wool * compression ratio of zbud while retaining its main concepts (e. g. always 129a001fc1SVitaly Wool * storing an integral number of objects per page) and simplicity. 139a001fc1SVitaly Wool * It still has simple and deterministic reclaim properties that make it 149a001fc1SVitaly Wool * preferable to a higher density approach (with no requirement on integral 159a001fc1SVitaly Wool * number of object per page) when reclaim is used. 169a001fc1SVitaly Wool * 179a001fc1SVitaly Wool * As in zbud, pages are divided into "chunks". The size of the chunks is 189a001fc1SVitaly Wool * fixed at compile time and is determined by NCHUNKS_ORDER below. 199a001fc1SVitaly Wool * 209a001fc1SVitaly Wool * z3fold doesn't export any API and is meant to be used via zpool API. 219a001fc1SVitaly Wool */ 229a001fc1SVitaly Wool 239a001fc1SVitaly Wool #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 249a001fc1SVitaly Wool 259a001fc1SVitaly Wool #include <linux/atomic.h> 269a001fc1SVitaly Wool #include <linux/list.h> 279a001fc1SVitaly Wool #include <linux/mm.h> 289a001fc1SVitaly Wool #include <linux/module.h> 299a001fc1SVitaly Wool #include <linux/preempt.h> 309a001fc1SVitaly Wool #include <linux/slab.h> 319a001fc1SVitaly Wool #include <linux/spinlock.h> 329a001fc1SVitaly Wool #include <linux/zpool.h> 339a001fc1SVitaly Wool 349a001fc1SVitaly Wool /***************** 359a001fc1SVitaly Wool * Structures 369a001fc1SVitaly Wool *****************/ 37*ede93213SVitaly Wool struct z3fold_pool; 38*ede93213SVitaly Wool struct z3fold_ops { 39*ede93213SVitaly Wool int (*evict)(struct z3fold_pool *pool, unsigned long handle); 40*ede93213SVitaly Wool }; 41*ede93213SVitaly Wool 42*ede93213SVitaly Wool enum buddy { 43*ede93213SVitaly Wool HEADLESS = 0, 44*ede93213SVitaly Wool FIRST, 45*ede93213SVitaly Wool MIDDLE, 46*ede93213SVitaly Wool LAST, 47*ede93213SVitaly Wool BUDDIES_MAX 48*ede93213SVitaly Wool }; 49*ede93213SVitaly Wool 50*ede93213SVitaly Wool /* 51*ede93213SVitaly Wool * struct z3fold_header - z3fold page metadata occupying the first chunk of each 52*ede93213SVitaly Wool * z3fold page, except for HEADLESS pages 53*ede93213SVitaly Wool * @buddy: links the z3fold page into the relevant list in the pool 54*ede93213SVitaly Wool * @first_chunks: the size of the first buddy in chunks, 0 if free 55*ede93213SVitaly Wool * @middle_chunks: the size of the middle buddy in chunks, 0 if free 56*ede93213SVitaly Wool * @last_chunks: the size of the last buddy in chunks, 0 if free 57*ede93213SVitaly Wool * @first_num: the starting number (for the first handle) 58*ede93213SVitaly Wool */ 59*ede93213SVitaly Wool struct z3fold_header { 60*ede93213SVitaly Wool struct list_head buddy; 61*ede93213SVitaly Wool unsigned short first_chunks; 62*ede93213SVitaly Wool unsigned short middle_chunks; 63*ede93213SVitaly Wool unsigned short last_chunks; 64*ede93213SVitaly Wool unsigned short start_middle; 65*ede93213SVitaly Wool unsigned short first_num:2; 66*ede93213SVitaly Wool }; 67*ede93213SVitaly Wool 689a001fc1SVitaly Wool /* 699a001fc1SVitaly Wool * NCHUNKS_ORDER determines the internal allocation granularity, effectively 709a001fc1SVitaly Wool * adjusting internal fragmentation. It also determines the number of 719a001fc1SVitaly Wool * freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the 72*ede93213SVitaly Wool * allocation granularity will be in chunks of size PAGE_SIZE/64. Some chunks 73*ede93213SVitaly Wool * in the beginning of an allocated page are occupied by z3fold header, so 74*ede93213SVitaly Wool * NCHUNKS will be calculated to 63 (or 62 in case CONFIG_DEBUG_SPINLOCK=y), 75*ede93213SVitaly Wool * which shows the max number of free chunks in z3fold page, also there will 76*ede93213SVitaly Wool * be 63, or 62, respectively, freelists per pool. 779a001fc1SVitaly Wool */ 789a001fc1SVitaly Wool #define NCHUNKS_ORDER 6 799a001fc1SVitaly Wool 809a001fc1SVitaly Wool #define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER) 819a001fc1SVitaly Wool #define CHUNK_SIZE (1 << CHUNK_SHIFT) 82*ede93213SVitaly Wool #define ZHDR_SIZE_ALIGNED round_up(sizeof(struct z3fold_header), CHUNK_SIZE) 83*ede93213SVitaly Wool #define ZHDR_CHUNKS (ZHDR_SIZE_ALIGNED >> CHUNK_SHIFT) 84*ede93213SVitaly Wool #define TOTAL_CHUNKS (PAGE_SIZE >> CHUNK_SHIFT) 859a001fc1SVitaly Wool #define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT) 869a001fc1SVitaly Wool 87f201ebd8Szhong jiang #define BUDDY_MASK (0x3) 889a001fc1SVitaly Wool 899a001fc1SVitaly Wool /** 909a001fc1SVitaly Wool * struct z3fold_pool - stores metadata for each z3fold pool 919a001fc1SVitaly Wool * @lock: protects all pool fields and first|last_chunk fields of any 929a001fc1SVitaly Wool * z3fold page in the pool 939a001fc1SVitaly Wool * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies; 949a001fc1SVitaly Wool * the lists each z3fold page is added to depends on the size of 959a001fc1SVitaly Wool * its free region. 969a001fc1SVitaly Wool * @buddied: list tracking the z3fold pages that contain 3 buddies; 979a001fc1SVitaly Wool * these z3fold pages are full 989a001fc1SVitaly Wool * @lru: list tracking the z3fold pages in LRU order by most recently 999a001fc1SVitaly Wool * added buddy. 1009a001fc1SVitaly Wool * @pages_nr: number of z3fold pages in the pool. 1019a001fc1SVitaly Wool * @ops: pointer to a structure of user defined operations specified at 1029a001fc1SVitaly Wool * pool creation time. 1039a001fc1SVitaly Wool * 1049a001fc1SVitaly Wool * This structure is allocated at pool creation time and maintains metadata 1059a001fc1SVitaly Wool * pertaining to a particular z3fold pool. 1069a001fc1SVitaly Wool */ 1079a001fc1SVitaly Wool struct z3fold_pool { 1089a001fc1SVitaly Wool spinlock_t lock; 1099a001fc1SVitaly Wool struct list_head unbuddied[NCHUNKS]; 1109a001fc1SVitaly Wool struct list_head buddied; 1119a001fc1SVitaly Wool struct list_head lru; 11212d59ae6SVitaly Wool atomic64_t pages_nr; 1139a001fc1SVitaly Wool const struct z3fold_ops *ops; 1149a001fc1SVitaly Wool struct zpool *zpool; 1159a001fc1SVitaly Wool const struct zpool_ops *zpool_ops; 1169a001fc1SVitaly Wool }; 1179a001fc1SVitaly Wool 1189a001fc1SVitaly Wool /* 1199a001fc1SVitaly Wool * Internal z3fold page flags 1209a001fc1SVitaly Wool */ 1219a001fc1SVitaly Wool enum z3fold_page_flags { 1229a001fc1SVitaly Wool UNDER_RECLAIM = 0, 1239a001fc1SVitaly Wool PAGE_HEADLESS, 1249a001fc1SVitaly Wool MIDDLE_CHUNK_MAPPED, 1259a001fc1SVitaly Wool }; 1269a001fc1SVitaly Wool 127*ede93213SVitaly Wool 1289a001fc1SVitaly Wool /***************** 1299a001fc1SVitaly Wool * Helpers 1309a001fc1SVitaly Wool *****************/ 1319a001fc1SVitaly Wool 1329a001fc1SVitaly Wool /* Converts an allocation size in bytes to size in z3fold chunks */ 1339a001fc1SVitaly Wool static int size_to_chunks(size_t size) 1349a001fc1SVitaly Wool { 1359a001fc1SVitaly Wool return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; 1369a001fc1SVitaly Wool } 1379a001fc1SVitaly Wool 1389a001fc1SVitaly Wool #define for_each_unbuddied_list(_iter, _begin) \ 1399a001fc1SVitaly Wool for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++) 1409a001fc1SVitaly Wool 1419a001fc1SVitaly Wool /* Initializes the z3fold header of a newly allocated z3fold page */ 1429a001fc1SVitaly Wool static struct z3fold_header *init_z3fold_page(struct page *page) 1439a001fc1SVitaly Wool { 1449a001fc1SVitaly Wool struct z3fold_header *zhdr = page_address(page); 1459a001fc1SVitaly Wool 1469a001fc1SVitaly Wool INIT_LIST_HEAD(&page->lru); 1479a001fc1SVitaly Wool clear_bit(UNDER_RECLAIM, &page->private); 1489a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 1499a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 1509a001fc1SVitaly Wool 1519a001fc1SVitaly Wool zhdr->first_chunks = 0; 1529a001fc1SVitaly Wool zhdr->middle_chunks = 0; 1539a001fc1SVitaly Wool zhdr->last_chunks = 0; 1549a001fc1SVitaly Wool zhdr->first_num = 0; 1559a001fc1SVitaly Wool zhdr->start_middle = 0; 1569a001fc1SVitaly Wool INIT_LIST_HEAD(&zhdr->buddy); 1579a001fc1SVitaly Wool return zhdr; 1589a001fc1SVitaly Wool } 1599a001fc1SVitaly Wool 1609a001fc1SVitaly Wool /* Resets the struct page fields and frees the page */ 1619a001fc1SVitaly Wool static void free_z3fold_page(struct z3fold_header *zhdr) 1629a001fc1SVitaly Wool { 1639a001fc1SVitaly Wool __free_page(virt_to_page(zhdr)); 1649a001fc1SVitaly Wool } 1659a001fc1SVitaly Wool 1669a001fc1SVitaly Wool /* 1679a001fc1SVitaly Wool * Encodes the handle of a particular buddy within a z3fold page 1689a001fc1SVitaly Wool * Pool lock should be held as this function accesses first_num 1699a001fc1SVitaly Wool */ 1709a001fc1SVitaly Wool static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) 1719a001fc1SVitaly Wool { 1729a001fc1SVitaly Wool unsigned long handle; 1739a001fc1SVitaly Wool 1749a001fc1SVitaly Wool handle = (unsigned long)zhdr; 1759a001fc1SVitaly Wool if (bud != HEADLESS) 1769a001fc1SVitaly Wool handle += (bud + zhdr->first_num) & BUDDY_MASK; 1779a001fc1SVitaly Wool return handle; 1789a001fc1SVitaly Wool } 1799a001fc1SVitaly Wool 1809a001fc1SVitaly Wool /* Returns the z3fold page where a given handle is stored */ 1819a001fc1SVitaly Wool static struct z3fold_header *handle_to_z3fold_header(unsigned long handle) 1829a001fc1SVitaly Wool { 1839a001fc1SVitaly Wool return (struct z3fold_header *)(handle & PAGE_MASK); 1849a001fc1SVitaly Wool } 1859a001fc1SVitaly Wool 186f201ebd8Szhong jiang /* 187f201ebd8Szhong jiang * (handle & BUDDY_MASK) < zhdr->first_num is possible in encode_handle 188f201ebd8Szhong jiang * but that doesn't matter. because the masking will result in the 189f201ebd8Szhong jiang * correct buddy number. 190f201ebd8Szhong jiang */ 1919a001fc1SVitaly Wool static enum buddy handle_to_buddy(unsigned long handle) 1929a001fc1SVitaly Wool { 1939a001fc1SVitaly Wool struct z3fold_header *zhdr = handle_to_z3fold_header(handle); 1949a001fc1SVitaly Wool return (handle - zhdr->first_num) & BUDDY_MASK; 1959a001fc1SVitaly Wool } 1969a001fc1SVitaly Wool 1979a001fc1SVitaly Wool /* 1989a001fc1SVitaly Wool * Returns the number of free chunks in a z3fold page. 1999a001fc1SVitaly Wool * NB: can't be used with HEADLESS pages. 2009a001fc1SVitaly Wool */ 2019a001fc1SVitaly Wool static int num_free_chunks(struct z3fold_header *zhdr) 2029a001fc1SVitaly Wool { 2039a001fc1SVitaly Wool int nfree; 2049a001fc1SVitaly Wool /* 2059a001fc1SVitaly Wool * If there is a middle object, pick up the bigger free space 2069a001fc1SVitaly Wool * either before or after it. Otherwise just subtract the number 2079a001fc1SVitaly Wool * of chunks occupied by the first and the last objects. 2089a001fc1SVitaly Wool */ 2099a001fc1SVitaly Wool if (zhdr->middle_chunks != 0) { 2109a001fc1SVitaly Wool int nfree_before = zhdr->first_chunks ? 211*ede93213SVitaly Wool 0 : zhdr->start_middle - ZHDR_CHUNKS; 2129a001fc1SVitaly Wool int nfree_after = zhdr->last_chunks ? 213*ede93213SVitaly Wool 0 : TOTAL_CHUNKS - 214*ede93213SVitaly Wool (zhdr->start_middle + zhdr->middle_chunks); 2159a001fc1SVitaly Wool nfree = max(nfree_before, nfree_after); 2169a001fc1SVitaly Wool } else 2179a001fc1SVitaly Wool nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; 2189a001fc1SVitaly Wool return nfree; 2199a001fc1SVitaly Wool } 2209a001fc1SVitaly Wool 2219a001fc1SVitaly Wool /***************** 2229a001fc1SVitaly Wool * API Functions 2239a001fc1SVitaly Wool *****************/ 2249a001fc1SVitaly Wool /** 2259a001fc1SVitaly Wool * z3fold_create_pool() - create a new z3fold pool 2269a001fc1SVitaly Wool * @gfp: gfp flags when allocating the z3fold pool structure 2279a001fc1SVitaly Wool * @ops: user-defined operations for the z3fold pool 2289a001fc1SVitaly Wool * 2299a001fc1SVitaly Wool * Return: pointer to the new z3fold pool or NULL if the metadata allocation 2309a001fc1SVitaly Wool * failed. 2319a001fc1SVitaly Wool */ 2329a001fc1SVitaly Wool static struct z3fold_pool *z3fold_create_pool(gfp_t gfp, 2339a001fc1SVitaly Wool const struct z3fold_ops *ops) 2349a001fc1SVitaly Wool { 2359a001fc1SVitaly Wool struct z3fold_pool *pool; 2369a001fc1SVitaly Wool int i; 2379a001fc1SVitaly Wool 2389a001fc1SVitaly Wool pool = kzalloc(sizeof(struct z3fold_pool), gfp); 2399a001fc1SVitaly Wool if (!pool) 2409a001fc1SVitaly Wool return NULL; 2419a001fc1SVitaly Wool spin_lock_init(&pool->lock); 2429a001fc1SVitaly Wool for_each_unbuddied_list(i, 0) 2439a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->unbuddied[i]); 2449a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->buddied); 2459a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->lru); 24612d59ae6SVitaly Wool atomic64_set(&pool->pages_nr, 0); 2479a001fc1SVitaly Wool pool->ops = ops; 2489a001fc1SVitaly Wool return pool; 2499a001fc1SVitaly Wool } 2509a001fc1SVitaly Wool 2519a001fc1SVitaly Wool /** 2529a001fc1SVitaly Wool * z3fold_destroy_pool() - destroys an existing z3fold pool 2539a001fc1SVitaly Wool * @pool: the z3fold pool to be destroyed 2549a001fc1SVitaly Wool * 2559a001fc1SVitaly Wool * The pool should be emptied before this function is called. 2569a001fc1SVitaly Wool */ 2579a001fc1SVitaly Wool static void z3fold_destroy_pool(struct z3fold_pool *pool) 2589a001fc1SVitaly Wool { 2599a001fc1SVitaly Wool kfree(pool); 2609a001fc1SVitaly Wool } 2619a001fc1SVitaly Wool 262*ede93213SVitaly Wool static inline void *mchunk_memmove(struct z3fold_header *zhdr, 263*ede93213SVitaly Wool unsigned short dst_chunk) 264*ede93213SVitaly Wool { 265*ede93213SVitaly Wool void *beg = zhdr; 266*ede93213SVitaly Wool return memmove(beg + (dst_chunk << CHUNK_SHIFT), 267*ede93213SVitaly Wool beg + (zhdr->start_middle << CHUNK_SHIFT), 268*ede93213SVitaly Wool zhdr->middle_chunks << CHUNK_SHIFT); 269*ede93213SVitaly Wool } 270*ede93213SVitaly Wool 2719a001fc1SVitaly Wool /* Has to be called with lock held */ 2729a001fc1SVitaly Wool static int z3fold_compact_page(struct z3fold_header *zhdr) 2739a001fc1SVitaly Wool { 2749a001fc1SVitaly Wool struct page *page = virt_to_page(zhdr); 2759a001fc1SVitaly Wool 276*ede93213SVitaly Wool if (test_bit(MIDDLE_CHUNK_MAPPED, &page->private)) 277*ede93213SVitaly Wool return 0; /* can't move middle chunk, it's used */ 2789a001fc1SVitaly Wool 279*ede93213SVitaly Wool if (zhdr->middle_chunks == 0) 280*ede93213SVitaly Wool return 0; /* nothing to compact */ 281*ede93213SVitaly Wool 282*ede93213SVitaly Wool if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) { 283*ede93213SVitaly Wool /* move to the beginning */ 284*ede93213SVitaly Wool mchunk_memmove(zhdr, ZHDR_CHUNKS); 2859a001fc1SVitaly Wool zhdr->first_chunks = zhdr->middle_chunks; 2869a001fc1SVitaly Wool zhdr->middle_chunks = 0; 2879a001fc1SVitaly Wool zhdr->start_middle = 0; 2889a001fc1SVitaly Wool zhdr->first_num++; 2899a001fc1SVitaly Wool } 290*ede93213SVitaly Wool return 1; 2919a001fc1SVitaly Wool } 2929a001fc1SVitaly Wool 2939a001fc1SVitaly Wool /** 2949a001fc1SVitaly Wool * z3fold_alloc() - allocates a region of a given size 2959a001fc1SVitaly Wool * @pool: z3fold pool from which to allocate 2969a001fc1SVitaly Wool * @size: size in bytes of the desired allocation 2979a001fc1SVitaly Wool * @gfp: gfp flags used if the pool needs to grow 2989a001fc1SVitaly Wool * @handle: handle of the new allocation 2999a001fc1SVitaly Wool * 3009a001fc1SVitaly Wool * This function will attempt to find a free region in the pool large enough to 3019a001fc1SVitaly Wool * satisfy the allocation request. A search of the unbuddied lists is 3029a001fc1SVitaly Wool * performed first. If no suitable free region is found, then a new page is 3039a001fc1SVitaly Wool * allocated and added to the pool to satisfy the request. 3049a001fc1SVitaly Wool * 3059a001fc1SVitaly Wool * gfp should not set __GFP_HIGHMEM as highmem pages cannot be used 3069a001fc1SVitaly Wool * as z3fold pool pages. 3079a001fc1SVitaly Wool * 3089a001fc1SVitaly Wool * Return: 0 if success and handle is set, otherwise -EINVAL if the size or 3099a001fc1SVitaly Wool * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate 3109a001fc1SVitaly Wool * a new page. 3119a001fc1SVitaly Wool */ 3129a001fc1SVitaly Wool static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, 3139a001fc1SVitaly Wool unsigned long *handle) 3149a001fc1SVitaly Wool { 3159a001fc1SVitaly Wool int chunks = 0, i, freechunks; 3169a001fc1SVitaly Wool struct z3fold_header *zhdr = NULL; 3179a001fc1SVitaly Wool enum buddy bud; 3189a001fc1SVitaly Wool struct page *page; 3199a001fc1SVitaly Wool 3209a001fc1SVitaly Wool if (!size || (gfp & __GFP_HIGHMEM)) 3219a001fc1SVitaly Wool return -EINVAL; 3229a001fc1SVitaly Wool 3239a001fc1SVitaly Wool if (size > PAGE_SIZE) 3249a001fc1SVitaly Wool return -ENOSPC; 3259a001fc1SVitaly Wool 3269a001fc1SVitaly Wool if (size > PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE) 3279a001fc1SVitaly Wool bud = HEADLESS; 3289a001fc1SVitaly Wool else { 3299a001fc1SVitaly Wool chunks = size_to_chunks(size); 3309a001fc1SVitaly Wool spin_lock(&pool->lock); 3319a001fc1SVitaly Wool 3329a001fc1SVitaly Wool /* First, try to find an unbuddied z3fold page. */ 3339a001fc1SVitaly Wool zhdr = NULL; 3349a001fc1SVitaly Wool for_each_unbuddied_list(i, chunks) { 3359a001fc1SVitaly Wool if (!list_empty(&pool->unbuddied[i])) { 3369a001fc1SVitaly Wool zhdr = list_first_entry(&pool->unbuddied[i], 3379a001fc1SVitaly Wool struct z3fold_header, buddy); 3389a001fc1SVitaly Wool page = virt_to_page(zhdr); 3399a001fc1SVitaly Wool if (zhdr->first_chunks == 0) { 3409a001fc1SVitaly Wool if (zhdr->middle_chunks != 0 && 3419a001fc1SVitaly Wool chunks >= zhdr->start_middle) 3429a001fc1SVitaly Wool bud = LAST; 3439a001fc1SVitaly Wool else 3449a001fc1SVitaly Wool bud = FIRST; 3459a001fc1SVitaly Wool } else if (zhdr->last_chunks == 0) 3469a001fc1SVitaly Wool bud = LAST; 3479a001fc1SVitaly Wool else if (zhdr->middle_chunks == 0) 3489a001fc1SVitaly Wool bud = MIDDLE; 3499a001fc1SVitaly Wool else { 3509a001fc1SVitaly Wool pr_err("No free chunks in unbuddied\n"); 3519a001fc1SVitaly Wool WARN_ON(1); 3529a001fc1SVitaly Wool continue; 3539a001fc1SVitaly Wool } 3549a001fc1SVitaly Wool list_del(&zhdr->buddy); 3559a001fc1SVitaly Wool goto found; 3569a001fc1SVitaly Wool } 3579a001fc1SVitaly Wool } 3589a001fc1SVitaly Wool bud = FIRST; 3599a001fc1SVitaly Wool spin_unlock(&pool->lock); 3609a001fc1SVitaly Wool } 3619a001fc1SVitaly Wool 3629a001fc1SVitaly Wool /* Couldn't find unbuddied z3fold page, create new one */ 3639a001fc1SVitaly Wool page = alloc_page(gfp); 3649a001fc1SVitaly Wool if (!page) 3659a001fc1SVitaly Wool return -ENOMEM; 3669a001fc1SVitaly Wool spin_lock(&pool->lock); 36712d59ae6SVitaly Wool atomic64_inc(&pool->pages_nr); 3689a001fc1SVitaly Wool zhdr = init_z3fold_page(page); 3699a001fc1SVitaly Wool 3709a001fc1SVitaly Wool if (bud == HEADLESS) { 3719a001fc1SVitaly Wool set_bit(PAGE_HEADLESS, &page->private); 3729a001fc1SVitaly Wool goto headless; 3739a001fc1SVitaly Wool } 3749a001fc1SVitaly Wool 3759a001fc1SVitaly Wool found: 3769a001fc1SVitaly Wool if (bud == FIRST) 3779a001fc1SVitaly Wool zhdr->first_chunks = chunks; 3789a001fc1SVitaly Wool else if (bud == LAST) 3799a001fc1SVitaly Wool zhdr->last_chunks = chunks; 3809a001fc1SVitaly Wool else { 3819a001fc1SVitaly Wool zhdr->middle_chunks = chunks; 382*ede93213SVitaly Wool zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; 3839a001fc1SVitaly Wool } 3849a001fc1SVitaly Wool 3859a001fc1SVitaly Wool if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || 3869a001fc1SVitaly Wool zhdr->middle_chunks == 0) { 3879a001fc1SVitaly Wool /* Add to unbuddied list */ 3889a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 3899a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->unbuddied[freechunks]); 3909a001fc1SVitaly Wool } else { 3919a001fc1SVitaly Wool /* Add to buddied list */ 3929a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->buddied); 3939a001fc1SVitaly Wool } 3949a001fc1SVitaly Wool 3959a001fc1SVitaly Wool headless: 3969a001fc1SVitaly Wool /* Add/move z3fold page to beginning of LRU */ 3979a001fc1SVitaly Wool if (!list_empty(&page->lru)) 3989a001fc1SVitaly Wool list_del(&page->lru); 3999a001fc1SVitaly Wool 4009a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 4019a001fc1SVitaly Wool 4029a001fc1SVitaly Wool *handle = encode_handle(zhdr, bud); 4039a001fc1SVitaly Wool spin_unlock(&pool->lock); 4049a001fc1SVitaly Wool 4059a001fc1SVitaly Wool return 0; 4069a001fc1SVitaly Wool } 4079a001fc1SVitaly Wool 4089a001fc1SVitaly Wool /** 4099a001fc1SVitaly Wool * z3fold_free() - frees the allocation associated with the given handle 4109a001fc1SVitaly Wool * @pool: pool in which the allocation resided 4119a001fc1SVitaly Wool * @handle: handle associated with the allocation returned by z3fold_alloc() 4129a001fc1SVitaly Wool * 4139a001fc1SVitaly Wool * In the case that the z3fold page in which the allocation resides is under 4149a001fc1SVitaly Wool * reclaim, as indicated by the PG_reclaim flag being set, this function 4159a001fc1SVitaly Wool * only sets the first|last_chunks to 0. The page is actually freed 4169a001fc1SVitaly Wool * once both buddies are evicted (see z3fold_reclaim_page() below). 4179a001fc1SVitaly Wool */ 4189a001fc1SVitaly Wool static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) 4199a001fc1SVitaly Wool { 4209a001fc1SVitaly Wool struct z3fold_header *zhdr; 4219a001fc1SVitaly Wool int freechunks; 4229a001fc1SVitaly Wool struct page *page; 4239a001fc1SVitaly Wool enum buddy bud; 4249a001fc1SVitaly Wool 4259a001fc1SVitaly Wool spin_lock(&pool->lock); 4269a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 4279a001fc1SVitaly Wool page = virt_to_page(zhdr); 4289a001fc1SVitaly Wool 4299a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 4309a001fc1SVitaly Wool /* HEADLESS page stored */ 4319a001fc1SVitaly Wool bud = HEADLESS; 4329a001fc1SVitaly Wool } else { 43343afc194SVitaly Wool bud = handle_to_buddy(handle); 4349a001fc1SVitaly Wool 4359a001fc1SVitaly Wool switch (bud) { 4369a001fc1SVitaly Wool case FIRST: 4379a001fc1SVitaly Wool zhdr->first_chunks = 0; 4389a001fc1SVitaly Wool break; 4399a001fc1SVitaly Wool case MIDDLE: 4409a001fc1SVitaly Wool zhdr->middle_chunks = 0; 4419a001fc1SVitaly Wool zhdr->start_middle = 0; 4429a001fc1SVitaly Wool break; 4439a001fc1SVitaly Wool case LAST: 4449a001fc1SVitaly Wool zhdr->last_chunks = 0; 4459a001fc1SVitaly Wool break; 4469a001fc1SVitaly Wool default: 4479a001fc1SVitaly Wool pr_err("%s: unknown bud %d\n", __func__, bud); 4489a001fc1SVitaly Wool WARN_ON(1); 4499a001fc1SVitaly Wool spin_unlock(&pool->lock); 4509a001fc1SVitaly Wool return; 4519a001fc1SVitaly Wool } 4529a001fc1SVitaly Wool } 4539a001fc1SVitaly Wool 4549a001fc1SVitaly Wool if (test_bit(UNDER_RECLAIM, &page->private)) { 4559a001fc1SVitaly Wool /* z3fold page is under reclaim, reclaim will free */ 4569a001fc1SVitaly Wool spin_unlock(&pool->lock); 4579a001fc1SVitaly Wool return; 4589a001fc1SVitaly Wool } 4599a001fc1SVitaly Wool 4609a001fc1SVitaly Wool /* Remove from existing buddy list */ 46112d59ae6SVitaly Wool if (bud != HEADLESS) 4629a001fc1SVitaly Wool list_del(&zhdr->buddy); 4639a001fc1SVitaly Wool 4649a001fc1SVitaly Wool if (bud == HEADLESS || 4659a001fc1SVitaly Wool (zhdr->first_chunks == 0 && zhdr->middle_chunks == 0 && 4669a001fc1SVitaly Wool zhdr->last_chunks == 0)) { 4679a001fc1SVitaly Wool /* z3fold page is empty, free */ 4689a001fc1SVitaly Wool list_del(&page->lru); 4699a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 4709a001fc1SVitaly Wool free_z3fold_page(zhdr); 47112d59ae6SVitaly Wool atomic64_dec(&pool->pages_nr); 4729a001fc1SVitaly Wool } else { 4739a001fc1SVitaly Wool z3fold_compact_page(zhdr); 4749a001fc1SVitaly Wool /* Add to the unbuddied list */ 4759a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 4769a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->unbuddied[freechunks]); 4779a001fc1SVitaly Wool } 4789a001fc1SVitaly Wool 4799a001fc1SVitaly Wool spin_unlock(&pool->lock); 4809a001fc1SVitaly Wool } 4819a001fc1SVitaly Wool 4829a001fc1SVitaly Wool /** 4839a001fc1SVitaly Wool * z3fold_reclaim_page() - evicts allocations from a pool page and frees it 4849a001fc1SVitaly Wool * @pool: pool from which a page will attempt to be evicted 4859a001fc1SVitaly Wool * @retires: number of pages on the LRU list for which eviction will 4869a001fc1SVitaly Wool * be attempted before failing 4879a001fc1SVitaly Wool * 4889a001fc1SVitaly Wool * z3fold reclaim is different from normal system reclaim in that it is done 4899a001fc1SVitaly Wool * from the bottom, up. This is because only the bottom layer, z3fold, has 4909a001fc1SVitaly Wool * information on how the allocations are organized within each z3fold page. 4919a001fc1SVitaly Wool * This has the potential to create interesting locking situations between 4929a001fc1SVitaly Wool * z3fold and the user, however. 4939a001fc1SVitaly Wool * 4949a001fc1SVitaly Wool * To avoid these, this is how z3fold_reclaim_page() should be called: 4959a001fc1SVitaly Wool 4969a001fc1SVitaly Wool * The user detects a page should be reclaimed and calls z3fold_reclaim_page(). 4979a001fc1SVitaly Wool * z3fold_reclaim_page() will remove a z3fold page from the pool LRU list and 4989a001fc1SVitaly Wool * call the user-defined eviction handler with the pool and handle as 4999a001fc1SVitaly Wool * arguments. 5009a001fc1SVitaly Wool * 5019a001fc1SVitaly Wool * If the handle can not be evicted, the eviction handler should return 5029a001fc1SVitaly Wool * non-zero. z3fold_reclaim_page() will add the z3fold page back to the 5039a001fc1SVitaly Wool * appropriate list and try the next z3fold page on the LRU up to 5049a001fc1SVitaly Wool * a user defined number of retries. 5059a001fc1SVitaly Wool * 5069a001fc1SVitaly Wool * If the handle is successfully evicted, the eviction handler should 5079a001fc1SVitaly Wool * return 0 _and_ should have called z3fold_free() on the handle. z3fold_free() 5089a001fc1SVitaly Wool * contains logic to delay freeing the page if the page is under reclaim, 5099a001fc1SVitaly Wool * as indicated by the setting of the PG_reclaim flag on the underlying page. 5109a001fc1SVitaly Wool * 5119a001fc1SVitaly Wool * If all buddies in the z3fold page are successfully evicted, then the 5129a001fc1SVitaly Wool * z3fold page can be freed. 5139a001fc1SVitaly Wool * 5149a001fc1SVitaly Wool * Returns: 0 if page is successfully freed, otherwise -EINVAL if there are 5159a001fc1SVitaly Wool * no pages to evict or an eviction handler is not registered, -EAGAIN if 5169a001fc1SVitaly Wool * the retry limit was hit. 5179a001fc1SVitaly Wool */ 5189a001fc1SVitaly Wool static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) 5199a001fc1SVitaly Wool { 5209a001fc1SVitaly Wool int i, ret = 0, freechunks; 5219a001fc1SVitaly Wool struct z3fold_header *zhdr; 5229a001fc1SVitaly Wool struct page *page; 5239a001fc1SVitaly Wool unsigned long first_handle = 0, middle_handle = 0, last_handle = 0; 5249a001fc1SVitaly Wool 5259a001fc1SVitaly Wool spin_lock(&pool->lock); 5269a001fc1SVitaly Wool if (!pool->ops || !pool->ops->evict || list_empty(&pool->lru) || 5279a001fc1SVitaly Wool retries == 0) { 5289a001fc1SVitaly Wool spin_unlock(&pool->lock); 5299a001fc1SVitaly Wool return -EINVAL; 5309a001fc1SVitaly Wool } 5319a001fc1SVitaly Wool for (i = 0; i < retries; i++) { 5329a001fc1SVitaly Wool page = list_last_entry(&pool->lru, struct page, lru); 5339a001fc1SVitaly Wool list_del(&page->lru); 5349a001fc1SVitaly Wool 5359a001fc1SVitaly Wool /* Protect z3fold page against free */ 5369a001fc1SVitaly Wool set_bit(UNDER_RECLAIM, &page->private); 5379a001fc1SVitaly Wool zhdr = page_address(page); 5389a001fc1SVitaly Wool if (!test_bit(PAGE_HEADLESS, &page->private)) { 5399a001fc1SVitaly Wool list_del(&zhdr->buddy); 5409a001fc1SVitaly Wool /* 5419a001fc1SVitaly Wool * We need encode the handles before unlocking, since 5429a001fc1SVitaly Wool * we can race with free that will set 5439a001fc1SVitaly Wool * (first|last)_chunks to 0 5449a001fc1SVitaly Wool */ 5459a001fc1SVitaly Wool first_handle = 0; 5469a001fc1SVitaly Wool last_handle = 0; 5479a001fc1SVitaly Wool middle_handle = 0; 5489a001fc1SVitaly Wool if (zhdr->first_chunks) 5499a001fc1SVitaly Wool first_handle = encode_handle(zhdr, FIRST); 5509a001fc1SVitaly Wool if (zhdr->middle_chunks) 5519a001fc1SVitaly Wool middle_handle = encode_handle(zhdr, MIDDLE); 5529a001fc1SVitaly Wool if (zhdr->last_chunks) 5539a001fc1SVitaly Wool last_handle = encode_handle(zhdr, LAST); 5549a001fc1SVitaly Wool } else { 5559a001fc1SVitaly Wool first_handle = encode_handle(zhdr, HEADLESS); 5569a001fc1SVitaly Wool last_handle = middle_handle = 0; 5579a001fc1SVitaly Wool } 5589a001fc1SVitaly Wool 5599a001fc1SVitaly Wool spin_unlock(&pool->lock); 5609a001fc1SVitaly Wool 5619a001fc1SVitaly Wool /* Issue the eviction callback(s) */ 5629a001fc1SVitaly Wool if (middle_handle) { 5639a001fc1SVitaly Wool ret = pool->ops->evict(pool, middle_handle); 5649a001fc1SVitaly Wool if (ret) 5659a001fc1SVitaly Wool goto next; 5669a001fc1SVitaly Wool } 5679a001fc1SVitaly Wool if (first_handle) { 5689a001fc1SVitaly Wool ret = pool->ops->evict(pool, first_handle); 5699a001fc1SVitaly Wool if (ret) 5709a001fc1SVitaly Wool goto next; 5719a001fc1SVitaly Wool } 5729a001fc1SVitaly Wool if (last_handle) { 5739a001fc1SVitaly Wool ret = pool->ops->evict(pool, last_handle); 5749a001fc1SVitaly Wool if (ret) 5759a001fc1SVitaly Wool goto next; 5769a001fc1SVitaly Wool } 5779a001fc1SVitaly Wool next: 5789a001fc1SVitaly Wool spin_lock(&pool->lock); 5799a001fc1SVitaly Wool clear_bit(UNDER_RECLAIM, &page->private); 5809a001fc1SVitaly Wool if ((test_bit(PAGE_HEADLESS, &page->private) && ret == 0) || 5819a001fc1SVitaly Wool (zhdr->first_chunks == 0 && zhdr->last_chunks == 0 && 5829a001fc1SVitaly Wool zhdr->middle_chunks == 0)) { 5839a001fc1SVitaly Wool /* 5849a001fc1SVitaly Wool * All buddies are now free, free the z3fold page and 5859a001fc1SVitaly Wool * return success. 5869a001fc1SVitaly Wool */ 5879a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 5889a001fc1SVitaly Wool free_z3fold_page(zhdr); 58912d59ae6SVitaly Wool atomic64_dec(&pool->pages_nr); 5909a001fc1SVitaly Wool spin_unlock(&pool->lock); 5919a001fc1SVitaly Wool return 0; 59243afc194SVitaly Wool } else if (!test_bit(PAGE_HEADLESS, &page->private)) { 59343afc194SVitaly Wool if (zhdr->first_chunks != 0 && 59443afc194SVitaly Wool zhdr->last_chunks != 0 && 59543afc194SVitaly Wool zhdr->middle_chunks != 0) { 5969a001fc1SVitaly Wool /* Full, add to buddied list */ 5979a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->buddied); 59843afc194SVitaly Wool } else { 5999a001fc1SVitaly Wool z3fold_compact_page(zhdr); 6009a001fc1SVitaly Wool /* add to unbuddied list */ 6019a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 60243afc194SVitaly Wool list_add(&zhdr->buddy, 60343afc194SVitaly Wool &pool->unbuddied[freechunks]); 60443afc194SVitaly Wool } 6059a001fc1SVitaly Wool } 6069a001fc1SVitaly Wool 6079a001fc1SVitaly Wool /* add to beginning of LRU */ 6089a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 6099a001fc1SVitaly Wool } 6109a001fc1SVitaly Wool spin_unlock(&pool->lock); 6119a001fc1SVitaly Wool return -EAGAIN; 6129a001fc1SVitaly Wool } 6139a001fc1SVitaly Wool 6149a001fc1SVitaly Wool /** 6159a001fc1SVitaly Wool * z3fold_map() - maps the allocation associated with the given handle 6169a001fc1SVitaly Wool * @pool: pool in which the allocation resides 6179a001fc1SVitaly Wool * @handle: handle associated with the allocation to be mapped 6189a001fc1SVitaly Wool * 6199a001fc1SVitaly Wool * Extracts the buddy number from handle and constructs the pointer to the 6209a001fc1SVitaly Wool * correct starting chunk within the page. 6219a001fc1SVitaly Wool * 6229a001fc1SVitaly Wool * Returns: a pointer to the mapped allocation 6239a001fc1SVitaly Wool */ 6249a001fc1SVitaly Wool static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) 6259a001fc1SVitaly Wool { 6269a001fc1SVitaly Wool struct z3fold_header *zhdr; 6279a001fc1SVitaly Wool struct page *page; 6289a001fc1SVitaly Wool void *addr; 6299a001fc1SVitaly Wool enum buddy buddy; 6309a001fc1SVitaly Wool 6319a001fc1SVitaly Wool spin_lock(&pool->lock); 6329a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 6339a001fc1SVitaly Wool addr = zhdr; 6349a001fc1SVitaly Wool page = virt_to_page(zhdr); 6359a001fc1SVitaly Wool 6369a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 6379a001fc1SVitaly Wool goto out; 6389a001fc1SVitaly Wool 6399a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 6409a001fc1SVitaly Wool switch (buddy) { 6419a001fc1SVitaly Wool case FIRST: 6429a001fc1SVitaly Wool addr += ZHDR_SIZE_ALIGNED; 6439a001fc1SVitaly Wool break; 6449a001fc1SVitaly Wool case MIDDLE: 6459a001fc1SVitaly Wool addr += zhdr->start_middle << CHUNK_SHIFT; 6469a001fc1SVitaly Wool set_bit(MIDDLE_CHUNK_MAPPED, &page->private); 6479a001fc1SVitaly Wool break; 6489a001fc1SVitaly Wool case LAST: 6499a001fc1SVitaly Wool addr += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT); 6509a001fc1SVitaly Wool break; 6519a001fc1SVitaly Wool default: 6529a001fc1SVitaly Wool pr_err("unknown buddy id %d\n", buddy); 6539a001fc1SVitaly Wool WARN_ON(1); 6549a001fc1SVitaly Wool addr = NULL; 6559a001fc1SVitaly Wool break; 6569a001fc1SVitaly Wool } 6579a001fc1SVitaly Wool out: 6589a001fc1SVitaly Wool spin_unlock(&pool->lock); 6599a001fc1SVitaly Wool return addr; 6609a001fc1SVitaly Wool } 6619a001fc1SVitaly Wool 6629a001fc1SVitaly Wool /** 6639a001fc1SVitaly Wool * z3fold_unmap() - unmaps the allocation associated with the given handle 6649a001fc1SVitaly Wool * @pool: pool in which the allocation resides 6659a001fc1SVitaly Wool * @handle: handle associated with the allocation to be unmapped 6669a001fc1SVitaly Wool */ 6679a001fc1SVitaly Wool static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) 6689a001fc1SVitaly Wool { 6699a001fc1SVitaly Wool struct z3fold_header *zhdr; 6709a001fc1SVitaly Wool struct page *page; 6719a001fc1SVitaly Wool enum buddy buddy; 6729a001fc1SVitaly Wool 6739a001fc1SVitaly Wool spin_lock(&pool->lock); 6749a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 6759a001fc1SVitaly Wool page = virt_to_page(zhdr); 6769a001fc1SVitaly Wool 6779a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 6789a001fc1SVitaly Wool spin_unlock(&pool->lock); 6799a001fc1SVitaly Wool return; 6809a001fc1SVitaly Wool } 6819a001fc1SVitaly Wool 6829a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 6839a001fc1SVitaly Wool if (buddy == MIDDLE) 6849a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 6859a001fc1SVitaly Wool spin_unlock(&pool->lock); 6869a001fc1SVitaly Wool } 6879a001fc1SVitaly Wool 6889a001fc1SVitaly Wool /** 6899a001fc1SVitaly Wool * z3fold_get_pool_size() - gets the z3fold pool size in pages 6909a001fc1SVitaly Wool * @pool: pool whose size is being queried 6919a001fc1SVitaly Wool * 69212d59ae6SVitaly Wool * Returns: size in pages of the given pool. 6939a001fc1SVitaly Wool */ 6949a001fc1SVitaly Wool static u64 z3fold_get_pool_size(struct z3fold_pool *pool) 6959a001fc1SVitaly Wool { 69612d59ae6SVitaly Wool return atomic64_read(&pool->pages_nr); 6979a001fc1SVitaly Wool } 6989a001fc1SVitaly Wool 6999a001fc1SVitaly Wool /***************** 7009a001fc1SVitaly Wool * zpool 7019a001fc1SVitaly Wool ****************/ 7029a001fc1SVitaly Wool 7039a001fc1SVitaly Wool static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle) 7049a001fc1SVitaly Wool { 7059a001fc1SVitaly Wool if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict) 7069a001fc1SVitaly Wool return pool->zpool_ops->evict(pool->zpool, handle); 7079a001fc1SVitaly Wool else 7089a001fc1SVitaly Wool return -ENOENT; 7099a001fc1SVitaly Wool } 7109a001fc1SVitaly Wool 7119a001fc1SVitaly Wool static const struct z3fold_ops z3fold_zpool_ops = { 7129a001fc1SVitaly Wool .evict = z3fold_zpool_evict 7139a001fc1SVitaly Wool }; 7149a001fc1SVitaly Wool 7159a001fc1SVitaly Wool static void *z3fold_zpool_create(const char *name, gfp_t gfp, 7169a001fc1SVitaly Wool const struct zpool_ops *zpool_ops, 7179a001fc1SVitaly Wool struct zpool *zpool) 7189a001fc1SVitaly Wool { 7199a001fc1SVitaly Wool struct z3fold_pool *pool; 7209a001fc1SVitaly Wool 7219a001fc1SVitaly Wool pool = z3fold_create_pool(gfp, zpool_ops ? &z3fold_zpool_ops : NULL); 7229a001fc1SVitaly Wool if (pool) { 7239a001fc1SVitaly Wool pool->zpool = zpool; 7249a001fc1SVitaly Wool pool->zpool_ops = zpool_ops; 7259a001fc1SVitaly Wool } 7269a001fc1SVitaly Wool return pool; 7279a001fc1SVitaly Wool } 7289a001fc1SVitaly Wool 7299a001fc1SVitaly Wool static void z3fold_zpool_destroy(void *pool) 7309a001fc1SVitaly Wool { 7319a001fc1SVitaly Wool z3fold_destroy_pool(pool); 7329a001fc1SVitaly Wool } 7339a001fc1SVitaly Wool 7349a001fc1SVitaly Wool static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp, 7359a001fc1SVitaly Wool unsigned long *handle) 7369a001fc1SVitaly Wool { 7379a001fc1SVitaly Wool return z3fold_alloc(pool, size, gfp, handle); 7389a001fc1SVitaly Wool } 7399a001fc1SVitaly Wool static void z3fold_zpool_free(void *pool, unsigned long handle) 7409a001fc1SVitaly Wool { 7419a001fc1SVitaly Wool z3fold_free(pool, handle); 7429a001fc1SVitaly Wool } 7439a001fc1SVitaly Wool 7449a001fc1SVitaly Wool static int z3fold_zpool_shrink(void *pool, unsigned int pages, 7459a001fc1SVitaly Wool unsigned int *reclaimed) 7469a001fc1SVitaly Wool { 7479a001fc1SVitaly Wool unsigned int total = 0; 7489a001fc1SVitaly Wool int ret = -EINVAL; 7499a001fc1SVitaly Wool 7509a001fc1SVitaly Wool while (total < pages) { 7519a001fc1SVitaly Wool ret = z3fold_reclaim_page(pool, 8); 7529a001fc1SVitaly Wool if (ret < 0) 7539a001fc1SVitaly Wool break; 7549a001fc1SVitaly Wool total++; 7559a001fc1SVitaly Wool } 7569a001fc1SVitaly Wool 7579a001fc1SVitaly Wool if (reclaimed) 7589a001fc1SVitaly Wool *reclaimed = total; 7599a001fc1SVitaly Wool 7609a001fc1SVitaly Wool return ret; 7619a001fc1SVitaly Wool } 7629a001fc1SVitaly Wool 7639a001fc1SVitaly Wool static void *z3fold_zpool_map(void *pool, unsigned long handle, 7649a001fc1SVitaly Wool enum zpool_mapmode mm) 7659a001fc1SVitaly Wool { 7669a001fc1SVitaly Wool return z3fold_map(pool, handle); 7679a001fc1SVitaly Wool } 7689a001fc1SVitaly Wool static void z3fold_zpool_unmap(void *pool, unsigned long handle) 7699a001fc1SVitaly Wool { 7709a001fc1SVitaly Wool z3fold_unmap(pool, handle); 7719a001fc1SVitaly Wool } 7729a001fc1SVitaly Wool 7739a001fc1SVitaly Wool static u64 z3fold_zpool_total_size(void *pool) 7749a001fc1SVitaly Wool { 7759a001fc1SVitaly Wool return z3fold_get_pool_size(pool) * PAGE_SIZE; 7769a001fc1SVitaly Wool } 7779a001fc1SVitaly Wool 7789a001fc1SVitaly Wool static struct zpool_driver z3fold_zpool_driver = { 7799a001fc1SVitaly Wool .type = "z3fold", 7809a001fc1SVitaly Wool .owner = THIS_MODULE, 7819a001fc1SVitaly Wool .create = z3fold_zpool_create, 7829a001fc1SVitaly Wool .destroy = z3fold_zpool_destroy, 7839a001fc1SVitaly Wool .malloc = z3fold_zpool_malloc, 7849a001fc1SVitaly Wool .free = z3fold_zpool_free, 7859a001fc1SVitaly Wool .shrink = z3fold_zpool_shrink, 7869a001fc1SVitaly Wool .map = z3fold_zpool_map, 7879a001fc1SVitaly Wool .unmap = z3fold_zpool_unmap, 7889a001fc1SVitaly Wool .total_size = z3fold_zpool_total_size, 7899a001fc1SVitaly Wool }; 7909a001fc1SVitaly Wool 7919a001fc1SVitaly Wool MODULE_ALIAS("zpool-z3fold"); 7929a001fc1SVitaly Wool 7939a001fc1SVitaly Wool static int __init init_z3fold(void) 7949a001fc1SVitaly Wool { 795*ede93213SVitaly Wool /* Make sure the z3fold header is not larger than the page size */ 796*ede93213SVitaly Wool BUILD_BUG_ON(ZHDR_SIZE_ALIGNED > PAGE_SIZE); 7979a001fc1SVitaly Wool zpool_register_driver(&z3fold_zpool_driver); 7989a001fc1SVitaly Wool 7999a001fc1SVitaly Wool return 0; 8009a001fc1SVitaly Wool } 8019a001fc1SVitaly Wool 8029a001fc1SVitaly Wool static void __exit exit_z3fold(void) 8039a001fc1SVitaly Wool { 8049a001fc1SVitaly Wool zpool_unregister_driver(&z3fold_zpool_driver); 8059a001fc1SVitaly Wool } 8069a001fc1SVitaly Wool 8079a001fc1SVitaly Wool module_init(init_z3fold); 8089a001fc1SVitaly Wool module_exit(exit_z3fold); 8099a001fc1SVitaly Wool 8109a001fc1SVitaly Wool MODULE_LICENSE("GPL"); 8119a001fc1SVitaly Wool MODULE_AUTHOR("Vitaly Wool <vitalywool@gmail.com>"); 8129a001fc1SVitaly Wool MODULE_DESCRIPTION("3-Fold Allocator for Compressed Pages"); 813