19a001fc1SVitaly Wool /* 29a001fc1SVitaly Wool * z3fold.c 39a001fc1SVitaly Wool * 49a001fc1SVitaly Wool * Author: Vitaly Wool <vitaly.wool@konsulko.com> 59a001fc1SVitaly Wool * Copyright (C) 2016, Sony Mobile Communications Inc. 69a001fc1SVitaly Wool * 79a001fc1SVitaly Wool * This implementation is based on zbud written by Seth Jennings. 89a001fc1SVitaly Wool * 99a001fc1SVitaly Wool * z3fold is an special purpose allocator for storing compressed pages. It 109a001fc1SVitaly Wool * can store up to three compressed pages per page which improves the 119a001fc1SVitaly Wool * compression ratio of zbud while retaining its main concepts (e. g. always 129a001fc1SVitaly Wool * storing an integral number of objects per page) and simplicity. 139a001fc1SVitaly Wool * It still has simple and deterministic reclaim properties that make it 149a001fc1SVitaly Wool * preferable to a higher density approach (with no requirement on integral 159a001fc1SVitaly Wool * number of object per page) when reclaim is used. 169a001fc1SVitaly Wool * 179a001fc1SVitaly Wool * As in zbud, pages are divided into "chunks". The size of the chunks is 189a001fc1SVitaly Wool * fixed at compile time and is determined by NCHUNKS_ORDER below. 199a001fc1SVitaly Wool * 209a001fc1SVitaly Wool * z3fold doesn't export any API and is meant to be used via zpool API. 219a001fc1SVitaly Wool */ 229a001fc1SVitaly Wool 239a001fc1SVitaly Wool #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 249a001fc1SVitaly Wool 259a001fc1SVitaly Wool #include <linux/atomic.h> 269a001fc1SVitaly Wool #include <linux/list.h> 279a001fc1SVitaly Wool #include <linux/mm.h> 289a001fc1SVitaly Wool #include <linux/module.h> 299a001fc1SVitaly Wool #include <linux/preempt.h> 309a001fc1SVitaly Wool #include <linux/slab.h> 319a001fc1SVitaly Wool #include <linux/spinlock.h> 329a001fc1SVitaly Wool #include <linux/zpool.h> 339a001fc1SVitaly Wool 349a001fc1SVitaly Wool /***************** 359a001fc1SVitaly Wool * Structures 369a001fc1SVitaly Wool *****************/ 379a001fc1SVitaly Wool /* 389a001fc1SVitaly Wool * NCHUNKS_ORDER determines the internal allocation granularity, effectively 399a001fc1SVitaly Wool * adjusting internal fragmentation. It also determines the number of 409a001fc1SVitaly Wool * freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the 419a001fc1SVitaly Wool * allocation granularity will be in chunks of size PAGE_SIZE/64. As one chunk 429a001fc1SVitaly Wool * in allocated page is occupied by z3fold header, NCHUNKS will be calculated 439a001fc1SVitaly Wool * to 63 which shows the max number of free chunks in z3fold page, also there 449a001fc1SVitaly Wool * will be 63 freelists per pool. 459a001fc1SVitaly Wool */ 469a001fc1SVitaly Wool #define NCHUNKS_ORDER 6 479a001fc1SVitaly Wool 489a001fc1SVitaly Wool #define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER) 499a001fc1SVitaly Wool #define CHUNK_SIZE (1 << CHUNK_SHIFT) 509a001fc1SVitaly Wool #define ZHDR_SIZE_ALIGNED CHUNK_SIZE 519a001fc1SVitaly Wool #define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT) 529a001fc1SVitaly Wool 53f201ebd8Szhong jiang #define BUDDY_MASK (0x3) 549a001fc1SVitaly Wool 559a001fc1SVitaly Wool struct z3fold_pool; 569a001fc1SVitaly Wool struct z3fold_ops { 579a001fc1SVitaly Wool int (*evict)(struct z3fold_pool *pool, unsigned long handle); 589a001fc1SVitaly Wool }; 599a001fc1SVitaly Wool 609a001fc1SVitaly Wool /** 619a001fc1SVitaly Wool * struct z3fold_pool - stores metadata for each z3fold pool 629a001fc1SVitaly Wool * @lock: protects all pool fields and first|last_chunk fields of any 639a001fc1SVitaly Wool * z3fold page in the pool 649a001fc1SVitaly Wool * @unbuddied: array of lists tracking z3fold pages that contain 2- buddies; 659a001fc1SVitaly Wool * the lists each z3fold page is added to depends on the size of 669a001fc1SVitaly Wool * its free region. 679a001fc1SVitaly Wool * @buddied: list tracking the z3fold pages that contain 3 buddies; 689a001fc1SVitaly Wool * these z3fold pages are full 699a001fc1SVitaly Wool * @lru: list tracking the z3fold pages in LRU order by most recently 709a001fc1SVitaly Wool * added buddy. 719a001fc1SVitaly Wool * @pages_nr: number of z3fold pages in the pool. 729a001fc1SVitaly Wool * @ops: pointer to a structure of user defined operations specified at 739a001fc1SVitaly Wool * pool creation time. 749a001fc1SVitaly Wool * 759a001fc1SVitaly Wool * This structure is allocated at pool creation time and maintains metadata 769a001fc1SVitaly Wool * pertaining to a particular z3fold pool. 779a001fc1SVitaly Wool */ 789a001fc1SVitaly Wool struct z3fold_pool { 799a001fc1SVitaly Wool spinlock_t lock; 809a001fc1SVitaly Wool struct list_head unbuddied[NCHUNKS]; 819a001fc1SVitaly Wool struct list_head buddied; 829a001fc1SVitaly Wool struct list_head lru; 83*12d59ae6SVitaly Wool atomic64_t pages_nr; 849a001fc1SVitaly Wool const struct z3fold_ops *ops; 859a001fc1SVitaly Wool struct zpool *zpool; 869a001fc1SVitaly Wool const struct zpool_ops *zpool_ops; 879a001fc1SVitaly Wool }; 889a001fc1SVitaly Wool 899a001fc1SVitaly Wool enum buddy { 909a001fc1SVitaly Wool HEADLESS = 0, 919a001fc1SVitaly Wool FIRST, 929a001fc1SVitaly Wool MIDDLE, 939a001fc1SVitaly Wool LAST, 949a001fc1SVitaly Wool BUDDIES_MAX 959a001fc1SVitaly Wool }; 969a001fc1SVitaly Wool 979a001fc1SVitaly Wool /* 989a001fc1SVitaly Wool * struct z3fold_header - z3fold page metadata occupying the first chunk of each 999a001fc1SVitaly Wool * z3fold page, except for HEADLESS pages 1009a001fc1SVitaly Wool * @buddy: links the z3fold page into the relevant list in the pool 1019a001fc1SVitaly Wool * @first_chunks: the size of the first buddy in chunks, 0 if free 1029a001fc1SVitaly Wool * @middle_chunks: the size of the middle buddy in chunks, 0 if free 1039a001fc1SVitaly Wool * @last_chunks: the size of the last buddy in chunks, 0 if free 1049a001fc1SVitaly Wool * @first_num: the starting number (for the first handle) 1059a001fc1SVitaly Wool */ 1069a001fc1SVitaly Wool struct z3fold_header { 1079a001fc1SVitaly Wool struct list_head buddy; 1089a001fc1SVitaly Wool unsigned short first_chunks; 1099a001fc1SVitaly Wool unsigned short middle_chunks; 1109a001fc1SVitaly Wool unsigned short last_chunks; 1119a001fc1SVitaly Wool unsigned short start_middle; 112f201ebd8Szhong jiang unsigned short first_num:2; 1139a001fc1SVitaly Wool }; 1149a001fc1SVitaly Wool 1159a001fc1SVitaly Wool /* 1169a001fc1SVitaly Wool * Internal z3fold page flags 1179a001fc1SVitaly Wool */ 1189a001fc1SVitaly Wool enum z3fold_page_flags { 1199a001fc1SVitaly Wool UNDER_RECLAIM = 0, 1209a001fc1SVitaly Wool PAGE_HEADLESS, 1219a001fc1SVitaly Wool MIDDLE_CHUNK_MAPPED, 1229a001fc1SVitaly Wool }; 1239a001fc1SVitaly Wool 1249a001fc1SVitaly Wool /***************** 1259a001fc1SVitaly Wool * Helpers 1269a001fc1SVitaly Wool *****************/ 1279a001fc1SVitaly Wool 1289a001fc1SVitaly Wool /* Converts an allocation size in bytes to size in z3fold chunks */ 1299a001fc1SVitaly Wool static int size_to_chunks(size_t size) 1309a001fc1SVitaly Wool { 1319a001fc1SVitaly Wool return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; 1329a001fc1SVitaly Wool } 1339a001fc1SVitaly Wool 1349a001fc1SVitaly Wool #define for_each_unbuddied_list(_iter, _begin) \ 1359a001fc1SVitaly Wool for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++) 1369a001fc1SVitaly Wool 1379a001fc1SVitaly Wool /* Initializes the z3fold header of a newly allocated z3fold page */ 1389a001fc1SVitaly Wool static struct z3fold_header *init_z3fold_page(struct page *page) 1399a001fc1SVitaly Wool { 1409a001fc1SVitaly Wool struct z3fold_header *zhdr = page_address(page); 1419a001fc1SVitaly Wool 1429a001fc1SVitaly Wool INIT_LIST_HEAD(&page->lru); 1439a001fc1SVitaly Wool clear_bit(UNDER_RECLAIM, &page->private); 1449a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 1459a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 1469a001fc1SVitaly Wool 1479a001fc1SVitaly Wool zhdr->first_chunks = 0; 1489a001fc1SVitaly Wool zhdr->middle_chunks = 0; 1499a001fc1SVitaly Wool zhdr->last_chunks = 0; 1509a001fc1SVitaly Wool zhdr->first_num = 0; 1519a001fc1SVitaly Wool zhdr->start_middle = 0; 1529a001fc1SVitaly Wool INIT_LIST_HEAD(&zhdr->buddy); 1539a001fc1SVitaly Wool return zhdr; 1549a001fc1SVitaly Wool } 1559a001fc1SVitaly Wool 1569a001fc1SVitaly Wool /* Resets the struct page fields and frees the page */ 1579a001fc1SVitaly Wool static void free_z3fold_page(struct z3fold_header *zhdr) 1589a001fc1SVitaly Wool { 1599a001fc1SVitaly Wool __free_page(virt_to_page(zhdr)); 1609a001fc1SVitaly Wool } 1619a001fc1SVitaly Wool 1629a001fc1SVitaly Wool /* 1639a001fc1SVitaly Wool * Encodes the handle of a particular buddy within a z3fold page 1649a001fc1SVitaly Wool * Pool lock should be held as this function accesses first_num 1659a001fc1SVitaly Wool */ 1669a001fc1SVitaly Wool static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) 1679a001fc1SVitaly Wool { 1689a001fc1SVitaly Wool unsigned long handle; 1699a001fc1SVitaly Wool 1709a001fc1SVitaly Wool handle = (unsigned long)zhdr; 1719a001fc1SVitaly Wool if (bud != HEADLESS) 1729a001fc1SVitaly Wool handle += (bud + zhdr->first_num) & BUDDY_MASK; 1739a001fc1SVitaly Wool return handle; 1749a001fc1SVitaly Wool } 1759a001fc1SVitaly Wool 1769a001fc1SVitaly Wool /* Returns the z3fold page where a given handle is stored */ 1779a001fc1SVitaly Wool static struct z3fold_header *handle_to_z3fold_header(unsigned long handle) 1789a001fc1SVitaly Wool { 1799a001fc1SVitaly Wool return (struct z3fold_header *)(handle & PAGE_MASK); 1809a001fc1SVitaly Wool } 1819a001fc1SVitaly Wool 182f201ebd8Szhong jiang /* 183f201ebd8Szhong jiang * (handle & BUDDY_MASK) < zhdr->first_num is possible in encode_handle 184f201ebd8Szhong jiang * but that doesn't matter. because the masking will result in the 185f201ebd8Szhong jiang * correct buddy number. 186f201ebd8Szhong jiang */ 1879a001fc1SVitaly Wool static enum buddy handle_to_buddy(unsigned long handle) 1889a001fc1SVitaly Wool { 1899a001fc1SVitaly Wool struct z3fold_header *zhdr = handle_to_z3fold_header(handle); 1909a001fc1SVitaly Wool return (handle - zhdr->first_num) & BUDDY_MASK; 1919a001fc1SVitaly Wool } 1929a001fc1SVitaly Wool 1939a001fc1SVitaly Wool /* 1949a001fc1SVitaly Wool * Returns the number of free chunks in a z3fold page. 1959a001fc1SVitaly Wool * NB: can't be used with HEADLESS pages. 1969a001fc1SVitaly Wool */ 1979a001fc1SVitaly Wool static int num_free_chunks(struct z3fold_header *zhdr) 1989a001fc1SVitaly Wool { 1999a001fc1SVitaly Wool int nfree; 2009a001fc1SVitaly Wool /* 2019a001fc1SVitaly Wool * If there is a middle object, pick up the bigger free space 2029a001fc1SVitaly Wool * either before or after it. Otherwise just subtract the number 2039a001fc1SVitaly Wool * of chunks occupied by the first and the last objects. 2049a001fc1SVitaly Wool */ 2059a001fc1SVitaly Wool if (zhdr->middle_chunks != 0) { 2069a001fc1SVitaly Wool int nfree_before = zhdr->first_chunks ? 2079a001fc1SVitaly Wool 0 : zhdr->start_middle - 1; 2089a001fc1SVitaly Wool int nfree_after = zhdr->last_chunks ? 2099a001fc1SVitaly Wool 0 : NCHUNKS - zhdr->start_middle - zhdr->middle_chunks; 2109a001fc1SVitaly Wool nfree = max(nfree_before, nfree_after); 2119a001fc1SVitaly Wool } else 2129a001fc1SVitaly Wool nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; 2139a001fc1SVitaly Wool return nfree; 2149a001fc1SVitaly Wool } 2159a001fc1SVitaly Wool 2169a001fc1SVitaly Wool /***************** 2179a001fc1SVitaly Wool * API Functions 2189a001fc1SVitaly Wool *****************/ 2199a001fc1SVitaly Wool /** 2209a001fc1SVitaly Wool * z3fold_create_pool() - create a new z3fold pool 2219a001fc1SVitaly Wool * @gfp: gfp flags when allocating the z3fold pool structure 2229a001fc1SVitaly Wool * @ops: user-defined operations for the z3fold pool 2239a001fc1SVitaly Wool * 2249a001fc1SVitaly Wool * Return: pointer to the new z3fold pool or NULL if the metadata allocation 2259a001fc1SVitaly Wool * failed. 2269a001fc1SVitaly Wool */ 2279a001fc1SVitaly Wool static struct z3fold_pool *z3fold_create_pool(gfp_t gfp, 2289a001fc1SVitaly Wool const struct z3fold_ops *ops) 2299a001fc1SVitaly Wool { 2309a001fc1SVitaly Wool struct z3fold_pool *pool; 2319a001fc1SVitaly Wool int i; 2329a001fc1SVitaly Wool 2339a001fc1SVitaly Wool pool = kzalloc(sizeof(struct z3fold_pool), gfp); 2349a001fc1SVitaly Wool if (!pool) 2359a001fc1SVitaly Wool return NULL; 2369a001fc1SVitaly Wool spin_lock_init(&pool->lock); 2379a001fc1SVitaly Wool for_each_unbuddied_list(i, 0) 2389a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->unbuddied[i]); 2399a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->buddied); 2409a001fc1SVitaly Wool INIT_LIST_HEAD(&pool->lru); 241*12d59ae6SVitaly Wool atomic64_set(&pool->pages_nr, 0); 2429a001fc1SVitaly Wool pool->ops = ops; 2439a001fc1SVitaly Wool return pool; 2449a001fc1SVitaly Wool } 2459a001fc1SVitaly Wool 2469a001fc1SVitaly Wool /** 2479a001fc1SVitaly Wool * z3fold_destroy_pool() - destroys an existing z3fold pool 2489a001fc1SVitaly Wool * @pool: the z3fold pool to be destroyed 2499a001fc1SVitaly Wool * 2509a001fc1SVitaly Wool * The pool should be emptied before this function is called. 2519a001fc1SVitaly Wool */ 2529a001fc1SVitaly Wool static void z3fold_destroy_pool(struct z3fold_pool *pool) 2539a001fc1SVitaly Wool { 2549a001fc1SVitaly Wool kfree(pool); 2559a001fc1SVitaly Wool } 2569a001fc1SVitaly Wool 2579a001fc1SVitaly Wool /* Has to be called with lock held */ 2589a001fc1SVitaly Wool static int z3fold_compact_page(struct z3fold_header *zhdr) 2599a001fc1SVitaly Wool { 2609a001fc1SVitaly Wool struct page *page = virt_to_page(zhdr); 2619a001fc1SVitaly Wool void *beg = zhdr; 2629a001fc1SVitaly Wool 2639a001fc1SVitaly Wool 2649a001fc1SVitaly Wool if (!test_bit(MIDDLE_CHUNK_MAPPED, &page->private) && 2659a001fc1SVitaly Wool zhdr->middle_chunks != 0 && 2669a001fc1SVitaly Wool zhdr->first_chunks == 0 && zhdr->last_chunks == 0) { 2679a001fc1SVitaly Wool memmove(beg + ZHDR_SIZE_ALIGNED, 2689a001fc1SVitaly Wool beg + (zhdr->start_middle << CHUNK_SHIFT), 2699a001fc1SVitaly Wool zhdr->middle_chunks << CHUNK_SHIFT); 2709a001fc1SVitaly Wool zhdr->first_chunks = zhdr->middle_chunks; 2719a001fc1SVitaly Wool zhdr->middle_chunks = 0; 2729a001fc1SVitaly Wool zhdr->start_middle = 0; 2739a001fc1SVitaly Wool zhdr->first_num++; 2749a001fc1SVitaly Wool return 1; 2759a001fc1SVitaly Wool } 2769a001fc1SVitaly Wool return 0; 2779a001fc1SVitaly Wool } 2789a001fc1SVitaly Wool 2799a001fc1SVitaly Wool /** 2809a001fc1SVitaly Wool * z3fold_alloc() - allocates a region of a given size 2819a001fc1SVitaly Wool * @pool: z3fold pool from which to allocate 2829a001fc1SVitaly Wool * @size: size in bytes of the desired allocation 2839a001fc1SVitaly Wool * @gfp: gfp flags used if the pool needs to grow 2849a001fc1SVitaly Wool * @handle: handle of the new allocation 2859a001fc1SVitaly Wool * 2869a001fc1SVitaly Wool * This function will attempt to find a free region in the pool large enough to 2879a001fc1SVitaly Wool * satisfy the allocation request. A search of the unbuddied lists is 2889a001fc1SVitaly Wool * performed first. If no suitable free region is found, then a new page is 2899a001fc1SVitaly Wool * allocated and added to the pool to satisfy the request. 2909a001fc1SVitaly Wool * 2919a001fc1SVitaly Wool * gfp should not set __GFP_HIGHMEM as highmem pages cannot be used 2929a001fc1SVitaly Wool * as z3fold pool pages. 2939a001fc1SVitaly Wool * 2949a001fc1SVitaly Wool * Return: 0 if success and handle is set, otherwise -EINVAL if the size or 2959a001fc1SVitaly Wool * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate 2969a001fc1SVitaly Wool * a new page. 2979a001fc1SVitaly Wool */ 2989a001fc1SVitaly Wool static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, 2999a001fc1SVitaly Wool unsigned long *handle) 3009a001fc1SVitaly Wool { 3019a001fc1SVitaly Wool int chunks = 0, i, freechunks; 3029a001fc1SVitaly Wool struct z3fold_header *zhdr = NULL; 3039a001fc1SVitaly Wool enum buddy bud; 3049a001fc1SVitaly Wool struct page *page; 3059a001fc1SVitaly Wool 3069a001fc1SVitaly Wool if (!size || (gfp & __GFP_HIGHMEM)) 3079a001fc1SVitaly Wool return -EINVAL; 3089a001fc1SVitaly Wool 3099a001fc1SVitaly Wool if (size > PAGE_SIZE) 3109a001fc1SVitaly Wool return -ENOSPC; 3119a001fc1SVitaly Wool 3129a001fc1SVitaly Wool if (size > PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE) 3139a001fc1SVitaly Wool bud = HEADLESS; 3149a001fc1SVitaly Wool else { 3159a001fc1SVitaly Wool chunks = size_to_chunks(size); 3169a001fc1SVitaly Wool spin_lock(&pool->lock); 3179a001fc1SVitaly Wool 3189a001fc1SVitaly Wool /* First, try to find an unbuddied z3fold page. */ 3199a001fc1SVitaly Wool zhdr = NULL; 3209a001fc1SVitaly Wool for_each_unbuddied_list(i, chunks) { 3219a001fc1SVitaly Wool if (!list_empty(&pool->unbuddied[i])) { 3229a001fc1SVitaly Wool zhdr = list_first_entry(&pool->unbuddied[i], 3239a001fc1SVitaly Wool struct z3fold_header, buddy); 3249a001fc1SVitaly Wool page = virt_to_page(zhdr); 3259a001fc1SVitaly Wool if (zhdr->first_chunks == 0) { 3269a001fc1SVitaly Wool if (zhdr->middle_chunks != 0 && 3279a001fc1SVitaly Wool chunks >= zhdr->start_middle) 3289a001fc1SVitaly Wool bud = LAST; 3299a001fc1SVitaly Wool else 3309a001fc1SVitaly Wool bud = FIRST; 3319a001fc1SVitaly Wool } else if (zhdr->last_chunks == 0) 3329a001fc1SVitaly Wool bud = LAST; 3339a001fc1SVitaly Wool else if (zhdr->middle_chunks == 0) 3349a001fc1SVitaly Wool bud = MIDDLE; 3359a001fc1SVitaly Wool else { 3369a001fc1SVitaly Wool pr_err("No free chunks in unbuddied\n"); 3379a001fc1SVitaly Wool WARN_ON(1); 3389a001fc1SVitaly Wool continue; 3399a001fc1SVitaly Wool } 3409a001fc1SVitaly Wool list_del(&zhdr->buddy); 3419a001fc1SVitaly Wool goto found; 3429a001fc1SVitaly Wool } 3439a001fc1SVitaly Wool } 3449a001fc1SVitaly Wool bud = FIRST; 3459a001fc1SVitaly Wool spin_unlock(&pool->lock); 3469a001fc1SVitaly Wool } 3479a001fc1SVitaly Wool 3489a001fc1SVitaly Wool /* Couldn't find unbuddied z3fold page, create new one */ 3499a001fc1SVitaly Wool page = alloc_page(gfp); 3509a001fc1SVitaly Wool if (!page) 3519a001fc1SVitaly Wool return -ENOMEM; 3529a001fc1SVitaly Wool spin_lock(&pool->lock); 353*12d59ae6SVitaly Wool atomic64_inc(&pool->pages_nr); 3549a001fc1SVitaly Wool zhdr = init_z3fold_page(page); 3559a001fc1SVitaly Wool 3569a001fc1SVitaly Wool if (bud == HEADLESS) { 3579a001fc1SVitaly Wool set_bit(PAGE_HEADLESS, &page->private); 3589a001fc1SVitaly Wool goto headless; 3599a001fc1SVitaly Wool } 3609a001fc1SVitaly Wool 3619a001fc1SVitaly Wool found: 3629a001fc1SVitaly Wool if (bud == FIRST) 3639a001fc1SVitaly Wool zhdr->first_chunks = chunks; 3649a001fc1SVitaly Wool else if (bud == LAST) 3659a001fc1SVitaly Wool zhdr->last_chunks = chunks; 3669a001fc1SVitaly Wool else { 3679a001fc1SVitaly Wool zhdr->middle_chunks = chunks; 3689a001fc1SVitaly Wool zhdr->start_middle = zhdr->first_chunks + 1; 3699a001fc1SVitaly Wool } 3709a001fc1SVitaly Wool 3719a001fc1SVitaly Wool if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || 3729a001fc1SVitaly Wool zhdr->middle_chunks == 0) { 3739a001fc1SVitaly Wool /* Add to unbuddied list */ 3749a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 3759a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->unbuddied[freechunks]); 3769a001fc1SVitaly Wool } else { 3779a001fc1SVitaly Wool /* Add to buddied list */ 3789a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->buddied); 3799a001fc1SVitaly Wool } 3809a001fc1SVitaly Wool 3819a001fc1SVitaly Wool headless: 3829a001fc1SVitaly Wool /* Add/move z3fold page to beginning of LRU */ 3839a001fc1SVitaly Wool if (!list_empty(&page->lru)) 3849a001fc1SVitaly Wool list_del(&page->lru); 3859a001fc1SVitaly Wool 3869a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 3879a001fc1SVitaly Wool 3889a001fc1SVitaly Wool *handle = encode_handle(zhdr, bud); 3899a001fc1SVitaly Wool spin_unlock(&pool->lock); 3909a001fc1SVitaly Wool 3919a001fc1SVitaly Wool return 0; 3929a001fc1SVitaly Wool } 3939a001fc1SVitaly Wool 3949a001fc1SVitaly Wool /** 3959a001fc1SVitaly Wool * z3fold_free() - frees the allocation associated with the given handle 3969a001fc1SVitaly Wool * @pool: pool in which the allocation resided 3979a001fc1SVitaly Wool * @handle: handle associated with the allocation returned by z3fold_alloc() 3989a001fc1SVitaly Wool * 3999a001fc1SVitaly Wool * In the case that the z3fold page in which the allocation resides is under 4009a001fc1SVitaly Wool * reclaim, as indicated by the PG_reclaim flag being set, this function 4019a001fc1SVitaly Wool * only sets the first|last_chunks to 0. The page is actually freed 4029a001fc1SVitaly Wool * once both buddies are evicted (see z3fold_reclaim_page() below). 4039a001fc1SVitaly Wool */ 4049a001fc1SVitaly Wool static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) 4059a001fc1SVitaly Wool { 4069a001fc1SVitaly Wool struct z3fold_header *zhdr; 4079a001fc1SVitaly Wool int freechunks; 4089a001fc1SVitaly Wool struct page *page; 4099a001fc1SVitaly Wool enum buddy bud; 4109a001fc1SVitaly Wool 4119a001fc1SVitaly Wool spin_lock(&pool->lock); 4129a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 4139a001fc1SVitaly Wool page = virt_to_page(zhdr); 4149a001fc1SVitaly Wool 4159a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 4169a001fc1SVitaly Wool /* HEADLESS page stored */ 4179a001fc1SVitaly Wool bud = HEADLESS; 4189a001fc1SVitaly Wool } else { 41943afc194SVitaly Wool bud = handle_to_buddy(handle); 4209a001fc1SVitaly Wool 4219a001fc1SVitaly Wool switch (bud) { 4229a001fc1SVitaly Wool case FIRST: 4239a001fc1SVitaly Wool zhdr->first_chunks = 0; 4249a001fc1SVitaly Wool break; 4259a001fc1SVitaly Wool case MIDDLE: 4269a001fc1SVitaly Wool zhdr->middle_chunks = 0; 4279a001fc1SVitaly Wool zhdr->start_middle = 0; 4289a001fc1SVitaly Wool break; 4299a001fc1SVitaly Wool case LAST: 4309a001fc1SVitaly Wool zhdr->last_chunks = 0; 4319a001fc1SVitaly Wool break; 4329a001fc1SVitaly Wool default: 4339a001fc1SVitaly Wool pr_err("%s: unknown bud %d\n", __func__, bud); 4349a001fc1SVitaly Wool WARN_ON(1); 4359a001fc1SVitaly Wool spin_unlock(&pool->lock); 4369a001fc1SVitaly Wool return; 4379a001fc1SVitaly Wool } 4389a001fc1SVitaly Wool } 4399a001fc1SVitaly Wool 4409a001fc1SVitaly Wool if (test_bit(UNDER_RECLAIM, &page->private)) { 4419a001fc1SVitaly Wool /* z3fold page is under reclaim, reclaim will free */ 4429a001fc1SVitaly Wool spin_unlock(&pool->lock); 4439a001fc1SVitaly Wool return; 4449a001fc1SVitaly Wool } 4459a001fc1SVitaly Wool 4469a001fc1SVitaly Wool /* Remove from existing buddy list */ 447*12d59ae6SVitaly Wool if (bud != HEADLESS) 4489a001fc1SVitaly Wool list_del(&zhdr->buddy); 4499a001fc1SVitaly Wool 4509a001fc1SVitaly Wool if (bud == HEADLESS || 4519a001fc1SVitaly Wool (zhdr->first_chunks == 0 && zhdr->middle_chunks == 0 && 4529a001fc1SVitaly Wool zhdr->last_chunks == 0)) { 4539a001fc1SVitaly Wool /* z3fold page is empty, free */ 4549a001fc1SVitaly Wool list_del(&page->lru); 4559a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 4569a001fc1SVitaly Wool free_z3fold_page(zhdr); 457*12d59ae6SVitaly Wool atomic64_dec(&pool->pages_nr); 4589a001fc1SVitaly Wool } else { 4599a001fc1SVitaly Wool z3fold_compact_page(zhdr); 4609a001fc1SVitaly Wool /* Add to the unbuddied list */ 4619a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 4629a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->unbuddied[freechunks]); 4639a001fc1SVitaly Wool } 4649a001fc1SVitaly Wool 4659a001fc1SVitaly Wool spin_unlock(&pool->lock); 4669a001fc1SVitaly Wool } 4679a001fc1SVitaly Wool 4689a001fc1SVitaly Wool /** 4699a001fc1SVitaly Wool * z3fold_reclaim_page() - evicts allocations from a pool page and frees it 4709a001fc1SVitaly Wool * @pool: pool from which a page will attempt to be evicted 4719a001fc1SVitaly Wool * @retires: number of pages on the LRU list for which eviction will 4729a001fc1SVitaly Wool * be attempted before failing 4739a001fc1SVitaly Wool * 4749a001fc1SVitaly Wool * z3fold reclaim is different from normal system reclaim in that it is done 4759a001fc1SVitaly Wool * from the bottom, up. This is because only the bottom layer, z3fold, has 4769a001fc1SVitaly Wool * information on how the allocations are organized within each z3fold page. 4779a001fc1SVitaly Wool * This has the potential to create interesting locking situations between 4789a001fc1SVitaly Wool * z3fold and the user, however. 4799a001fc1SVitaly Wool * 4809a001fc1SVitaly Wool * To avoid these, this is how z3fold_reclaim_page() should be called: 4819a001fc1SVitaly Wool 4829a001fc1SVitaly Wool * The user detects a page should be reclaimed and calls z3fold_reclaim_page(). 4839a001fc1SVitaly Wool * z3fold_reclaim_page() will remove a z3fold page from the pool LRU list and 4849a001fc1SVitaly Wool * call the user-defined eviction handler with the pool and handle as 4859a001fc1SVitaly Wool * arguments. 4869a001fc1SVitaly Wool * 4879a001fc1SVitaly Wool * If the handle can not be evicted, the eviction handler should return 4889a001fc1SVitaly Wool * non-zero. z3fold_reclaim_page() will add the z3fold page back to the 4899a001fc1SVitaly Wool * appropriate list and try the next z3fold page on the LRU up to 4909a001fc1SVitaly Wool * a user defined number of retries. 4919a001fc1SVitaly Wool * 4929a001fc1SVitaly Wool * If the handle is successfully evicted, the eviction handler should 4939a001fc1SVitaly Wool * return 0 _and_ should have called z3fold_free() on the handle. z3fold_free() 4949a001fc1SVitaly Wool * contains logic to delay freeing the page if the page is under reclaim, 4959a001fc1SVitaly Wool * as indicated by the setting of the PG_reclaim flag on the underlying page. 4969a001fc1SVitaly Wool * 4979a001fc1SVitaly Wool * If all buddies in the z3fold page are successfully evicted, then the 4989a001fc1SVitaly Wool * z3fold page can be freed. 4999a001fc1SVitaly Wool * 5009a001fc1SVitaly Wool * Returns: 0 if page is successfully freed, otherwise -EINVAL if there are 5019a001fc1SVitaly Wool * no pages to evict or an eviction handler is not registered, -EAGAIN if 5029a001fc1SVitaly Wool * the retry limit was hit. 5039a001fc1SVitaly Wool */ 5049a001fc1SVitaly Wool static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) 5059a001fc1SVitaly Wool { 5069a001fc1SVitaly Wool int i, ret = 0, freechunks; 5079a001fc1SVitaly Wool struct z3fold_header *zhdr; 5089a001fc1SVitaly Wool struct page *page; 5099a001fc1SVitaly Wool unsigned long first_handle = 0, middle_handle = 0, last_handle = 0; 5109a001fc1SVitaly Wool 5119a001fc1SVitaly Wool spin_lock(&pool->lock); 5129a001fc1SVitaly Wool if (!pool->ops || !pool->ops->evict || list_empty(&pool->lru) || 5139a001fc1SVitaly Wool retries == 0) { 5149a001fc1SVitaly Wool spin_unlock(&pool->lock); 5159a001fc1SVitaly Wool return -EINVAL; 5169a001fc1SVitaly Wool } 5179a001fc1SVitaly Wool for (i = 0; i < retries; i++) { 5189a001fc1SVitaly Wool page = list_last_entry(&pool->lru, struct page, lru); 5199a001fc1SVitaly Wool list_del(&page->lru); 5209a001fc1SVitaly Wool 5219a001fc1SVitaly Wool /* Protect z3fold page against free */ 5229a001fc1SVitaly Wool set_bit(UNDER_RECLAIM, &page->private); 5239a001fc1SVitaly Wool zhdr = page_address(page); 5249a001fc1SVitaly Wool if (!test_bit(PAGE_HEADLESS, &page->private)) { 5259a001fc1SVitaly Wool list_del(&zhdr->buddy); 5269a001fc1SVitaly Wool /* 5279a001fc1SVitaly Wool * We need encode the handles before unlocking, since 5289a001fc1SVitaly Wool * we can race with free that will set 5299a001fc1SVitaly Wool * (first|last)_chunks to 0 5309a001fc1SVitaly Wool */ 5319a001fc1SVitaly Wool first_handle = 0; 5329a001fc1SVitaly Wool last_handle = 0; 5339a001fc1SVitaly Wool middle_handle = 0; 5349a001fc1SVitaly Wool if (zhdr->first_chunks) 5359a001fc1SVitaly Wool first_handle = encode_handle(zhdr, FIRST); 5369a001fc1SVitaly Wool if (zhdr->middle_chunks) 5379a001fc1SVitaly Wool middle_handle = encode_handle(zhdr, MIDDLE); 5389a001fc1SVitaly Wool if (zhdr->last_chunks) 5399a001fc1SVitaly Wool last_handle = encode_handle(zhdr, LAST); 5409a001fc1SVitaly Wool } else { 5419a001fc1SVitaly Wool first_handle = encode_handle(zhdr, HEADLESS); 5429a001fc1SVitaly Wool last_handle = middle_handle = 0; 5439a001fc1SVitaly Wool } 5449a001fc1SVitaly Wool 5459a001fc1SVitaly Wool spin_unlock(&pool->lock); 5469a001fc1SVitaly Wool 5479a001fc1SVitaly Wool /* Issue the eviction callback(s) */ 5489a001fc1SVitaly Wool if (middle_handle) { 5499a001fc1SVitaly Wool ret = pool->ops->evict(pool, middle_handle); 5509a001fc1SVitaly Wool if (ret) 5519a001fc1SVitaly Wool goto next; 5529a001fc1SVitaly Wool } 5539a001fc1SVitaly Wool if (first_handle) { 5549a001fc1SVitaly Wool ret = pool->ops->evict(pool, first_handle); 5559a001fc1SVitaly Wool if (ret) 5569a001fc1SVitaly Wool goto next; 5579a001fc1SVitaly Wool } 5589a001fc1SVitaly Wool if (last_handle) { 5599a001fc1SVitaly Wool ret = pool->ops->evict(pool, last_handle); 5609a001fc1SVitaly Wool if (ret) 5619a001fc1SVitaly Wool goto next; 5629a001fc1SVitaly Wool } 5639a001fc1SVitaly Wool next: 5649a001fc1SVitaly Wool spin_lock(&pool->lock); 5659a001fc1SVitaly Wool clear_bit(UNDER_RECLAIM, &page->private); 5669a001fc1SVitaly Wool if ((test_bit(PAGE_HEADLESS, &page->private) && ret == 0) || 5679a001fc1SVitaly Wool (zhdr->first_chunks == 0 && zhdr->last_chunks == 0 && 5689a001fc1SVitaly Wool zhdr->middle_chunks == 0)) { 5699a001fc1SVitaly Wool /* 5709a001fc1SVitaly Wool * All buddies are now free, free the z3fold page and 5719a001fc1SVitaly Wool * return success. 5729a001fc1SVitaly Wool */ 5739a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 5749a001fc1SVitaly Wool free_z3fold_page(zhdr); 575*12d59ae6SVitaly Wool atomic64_dec(&pool->pages_nr); 5769a001fc1SVitaly Wool spin_unlock(&pool->lock); 5779a001fc1SVitaly Wool return 0; 57843afc194SVitaly Wool } else if (!test_bit(PAGE_HEADLESS, &page->private)) { 57943afc194SVitaly Wool if (zhdr->first_chunks != 0 && 58043afc194SVitaly Wool zhdr->last_chunks != 0 && 58143afc194SVitaly Wool zhdr->middle_chunks != 0) { 5829a001fc1SVitaly Wool /* Full, add to buddied list */ 5839a001fc1SVitaly Wool list_add(&zhdr->buddy, &pool->buddied); 58443afc194SVitaly Wool } else { 5859a001fc1SVitaly Wool z3fold_compact_page(zhdr); 5869a001fc1SVitaly Wool /* add to unbuddied list */ 5879a001fc1SVitaly Wool freechunks = num_free_chunks(zhdr); 58843afc194SVitaly Wool list_add(&zhdr->buddy, 58943afc194SVitaly Wool &pool->unbuddied[freechunks]); 59043afc194SVitaly Wool } 5919a001fc1SVitaly Wool } 5929a001fc1SVitaly Wool 5939a001fc1SVitaly Wool /* add to beginning of LRU */ 5949a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 5959a001fc1SVitaly Wool } 5969a001fc1SVitaly Wool spin_unlock(&pool->lock); 5979a001fc1SVitaly Wool return -EAGAIN; 5989a001fc1SVitaly Wool } 5999a001fc1SVitaly Wool 6009a001fc1SVitaly Wool /** 6019a001fc1SVitaly Wool * z3fold_map() - maps the allocation associated with the given handle 6029a001fc1SVitaly Wool * @pool: pool in which the allocation resides 6039a001fc1SVitaly Wool * @handle: handle associated with the allocation to be mapped 6049a001fc1SVitaly Wool * 6059a001fc1SVitaly Wool * Extracts the buddy number from handle and constructs the pointer to the 6069a001fc1SVitaly Wool * correct starting chunk within the page. 6079a001fc1SVitaly Wool * 6089a001fc1SVitaly Wool * Returns: a pointer to the mapped allocation 6099a001fc1SVitaly Wool */ 6109a001fc1SVitaly Wool static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) 6119a001fc1SVitaly Wool { 6129a001fc1SVitaly Wool struct z3fold_header *zhdr; 6139a001fc1SVitaly Wool struct page *page; 6149a001fc1SVitaly Wool void *addr; 6159a001fc1SVitaly Wool enum buddy buddy; 6169a001fc1SVitaly Wool 6179a001fc1SVitaly Wool spin_lock(&pool->lock); 6189a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 6199a001fc1SVitaly Wool addr = zhdr; 6209a001fc1SVitaly Wool page = virt_to_page(zhdr); 6219a001fc1SVitaly Wool 6229a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 6239a001fc1SVitaly Wool goto out; 6249a001fc1SVitaly Wool 6259a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 6269a001fc1SVitaly Wool switch (buddy) { 6279a001fc1SVitaly Wool case FIRST: 6289a001fc1SVitaly Wool addr += ZHDR_SIZE_ALIGNED; 6299a001fc1SVitaly Wool break; 6309a001fc1SVitaly Wool case MIDDLE: 6319a001fc1SVitaly Wool addr += zhdr->start_middle << CHUNK_SHIFT; 6329a001fc1SVitaly Wool set_bit(MIDDLE_CHUNK_MAPPED, &page->private); 6339a001fc1SVitaly Wool break; 6349a001fc1SVitaly Wool case LAST: 6359a001fc1SVitaly Wool addr += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT); 6369a001fc1SVitaly Wool break; 6379a001fc1SVitaly Wool default: 6389a001fc1SVitaly Wool pr_err("unknown buddy id %d\n", buddy); 6399a001fc1SVitaly Wool WARN_ON(1); 6409a001fc1SVitaly Wool addr = NULL; 6419a001fc1SVitaly Wool break; 6429a001fc1SVitaly Wool } 6439a001fc1SVitaly Wool out: 6449a001fc1SVitaly Wool spin_unlock(&pool->lock); 6459a001fc1SVitaly Wool return addr; 6469a001fc1SVitaly Wool } 6479a001fc1SVitaly Wool 6489a001fc1SVitaly Wool /** 6499a001fc1SVitaly Wool * z3fold_unmap() - unmaps the allocation associated with the given handle 6509a001fc1SVitaly Wool * @pool: pool in which the allocation resides 6519a001fc1SVitaly Wool * @handle: handle associated with the allocation to be unmapped 6529a001fc1SVitaly Wool */ 6539a001fc1SVitaly Wool static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) 6549a001fc1SVitaly Wool { 6559a001fc1SVitaly Wool struct z3fold_header *zhdr; 6569a001fc1SVitaly Wool struct page *page; 6579a001fc1SVitaly Wool enum buddy buddy; 6589a001fc1SVitaly Wool 6599a001fc1SVitaly Wool spin_lock(&pool->lock); 6609a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 6619a001fc1SVitaly Wool page = virt_to_page(zhdr); 6629a001fc1SVitaly Wool 6639a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 6649a001fc1SVitaly Wool spin_unlock(&pool->lock); 6659a001fc1SVitaly Wool return; 6669a001fc1SVitaly Wool } 6679a001fc1SVitaly Wool 6689a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 6699a001fc1SVitaly Wool if (buddy == MIDDLE) 6709a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 6719a001fc1SVitaly Wool spin_unlock(&pool->lock); 6729a001fc1SVitaly Wool } 6739a001fc1SVitaly Wool 6749a001fc1SVitaly Wool /** 6759a001fc1SVitaly Wool * z3fold_get_pool_size() - gets the z3fold pool size in pages 6769a001fc1SVitaly Wool * @pool: pool whose size is being queried 6779a001fc1SVitaly Wool * 678*12d59ae6SVitaly Wool * Returns: size in pages of the given pool. 6799a001fc1SVitaly Wool */ 6809a001fc1SVitaly Wool static u64 z3fold_get_pool_size(struct z3fold_pool *pool) 6819a001fc1SVitaly Wool { 682*12d59ae6SVitaly Wool return atomic64_read(&pool->pages_nr); 6839a001fc1SVitaly Wool } 6849a001fc1SVitaly Wool 6859a001fc1SVitaly Wool /***************** 6869a001fc1SVitaly Wool * zpool 6879a001fc1SVitaly Wool ****************/ 6889a001fc1SVitaly Wool 6899a001fc1SVitaly Wool static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle) 6909a001fc1SVitaly Wool { 6919a001fc1SVitaly Wool if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict) 6929a001fc1SVitaly Wool return pool->zpool_ops->evict(pool->zpool, handle); 6939a001fc1SVitaly Wool else 6949a001fc1SVitaly Wool return -ENOENT; 6959a001fc1SVitaly Wool } 6969a001fc1SVitaly Wool 6979a001fc1SVitaly Wool static const struct z3fold_ops z3fold_zpool_ops = { 6989a001fc1SVitaly Wool .evict = z3fold_zpool_evict 6999a001fc1SVitaly Wool }; 7009a001fc1SVitaly Wool 7019a001fc1SVitaly Wool static void *z3fold_zpool_create(const char *name, gfp_t gfp, 7029a001fc1SVitaly Wool const struct zpool_ops *zpool_ops, 7039a001fc1SVitaly Wool struct zpool *zpool) 7049a001fc1SVitaly Wool { 7059a001fc1SVitaly Wool struct z3fold_pool *pool; 7069a001fc1SVitaly Wool 7079a001fc1SVitaly Wool pool = z3fold_create_pool(gfp, zpool_ops ? &z3fold_zpool_ops : NULL); 7089a001fc1SVitaly Wool if (pool) { 7099a001fc1SVitaly Wool pool->zpool = zpool; 7109a001fc1SVitaly Wool pool->zpool_ops = zpool_ops; 7119a001fc1SVitaly Wool } 7129a001fc1SVitaly Wool return pool; 7139a001fc1SVitaly Wool } 7149a001fc1SVitaly Wool 7159a001fc1SVitaly Wool static void z3fold_zpool_destroy(void *pool) 7169a001fc1SVitaly Wool { 7179a001fc1SVitaly Wool z3fold_destroy_pool(pool); 7189a001fc1SVitaly Wool } 7199a001fc1SVitaly Wool 7209a001fc1SVitaly Wool static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp, 7219a001fc1SVitaly Wool unsigned long *handle) 7229a001fc1SVitaly Wool { 7239a001fc1SVitaly Wool return z3fold_alloc(pool, size, gfp, handle); 7249a001fc1SVitaly Wool } 7259a001fc1SVitaly Wool static void z3fold_zpool_free(void *pool, unsigned long handle) 7269a001fc1SVitaly Wool { 7279a001fc1SVitaly Wool z3fold_free(pool, handle); 7289a001fc1SVitaly Wool } 7299a001fc1SVitaly Wool 7309a001fc1SVitaly Wool static int z3fold_zpool_shrink(void *pool, unsigned int pages, 7319a001fc1SVitaly Wool unsigned int *reclaimed) 7329a001fc1SVitaly Wool { 7339a001fc1SVitaly Wool unsigned int total = 0; 7349a001fc1SVitaly Wool int ret = -EINVAL; 7359a001fc1SVitaly Wool 7369a001fc1SVitaly Wool while (total < pages) { 7379a001fc1SVitaly Wool ret = z3fold_reclaim_page(pool, 8); 7389a001fc1SVitaly Wool if (ret < 0) 7399a001fc1SVitaly Wool break; 7409a001fc1SVitaly Wool total++; 7419a001fc1SVitaly Wool } 7429a001fc1SVitaly Wool 7439a001fc1SVitaly Wool if (reclaimed) 7449a001fc1SVitaly Wool *reclaimed = total; 7459a001fc1SVitaly Wool 7469a001fc1SVitaly Wool return ret; 7479a001fc1SVitaly Wool } 7489a001fc1SVitaly Wool 7499a001fc1SVitaly Wool static void *z3fold_zpool_map(void *pool, unsigned long handle, 7509a001fc1SVitaly Wool enum zpool_mapmode mm) 7519a001fc1SVitaly Wool { 7529a001fc1SVitaly Wool return z3fold_map(pool, handle); 7539a001fc1SVitaly Wool } 7549a001fc1SVitaly Wool static void z3fold_zpool_unmap(void *pool, unsigned long handle) 7559a001fc1SVitaly Wool { 7569a001fc1SVitaly Wool z3fold_unmap(pool, handle); 7579a001fc1SVitaly Wool } 7589a001fc1SVitaly Wool 7599a001fc1SVitaly Wool static u64 z3fold_zpool_total_size(void *pool) 7609a001fc1SVitaly Wool { 7619a001fc1SVitaly Wool return z3fold_get_pool_size(pool) * PAGE_SIZE; 7629a001fc1SVitaly Wool } 7639a001fc1SVitaly Wool 7649a001fc1SVitaly Wool static struct zpool_driver z3fold_zpool_driver = { 7659a001fc1SVitaly Wool .type = "z3fold", 7669a001fc1SVitaly Wool .owner = THIS_MODULE, 7679a001fc1SVitaly Wool .create = z3fold_zpool_create, 7689a001fc1SVitaly Wool .destroy = z3fold_zpool_destroy, 7699a001fc1SVitaly Wool .malloc = z3fold_zpool_malloc, 7709a001fc1SVitaly Wool .free = z3fold_zpool_free, 7719a001fc1SVitaly Wool .shrink = z3fold_zpool_shrink, 7729a001fc1SVitaly Wool .map = z3fold_zpool_map, 7739a001fc1SVitaly Wool .unmap = z3fold_zpool_unmap, 7749a001fc1SVitaly Wool .total_size = z3fold_zpool_total_size, 7759a001fc1SVitaly Wool }; 7769a001fc1SVitaly Wool 7779a001fc1SVitaly Wool MODULE_ALIAS("zpool-z3fold"); 7789a001fc1SVitaly Wool 7799a001fc1SVitaly Wool static int __init init_z3fold(void) 7809a001fc1SVitaly Wool { 7819a001fc1SVitaly Wool /* Make sure the z3fold header will fit in one chunk */ 7829a001fc1SVitaly Wool BUILD_BUG_ON(sizeof(struct z3fold_header) > ZHDR_SIZE_ALIGNED); 7839a001fc1SVitaly Wool zpool_register_driver(&z3fold_zpool_driver); 7849a001fc1SVitaly Wool 7859a001fc1SVitaly Wool return 0; 7869a001fc1SVitaly Wool } 7879a001fc1SVitaly Wool 7889a001fc1SVitaly Wool static void __exit exit_z3fold(void) 7899a001fc1SVitaly Wool { 7909a001fc1SVitaly Wool zpool_unregister_driver(&z3fold_zpool_driver); 7919a001fc1SVitaly Wool } 7929a001fc1SVitaly Wool 7939a001fc1SVitaly Wool module_init(init_z3fold); 7949a001fc1SVitaly Wool module_exit(exit_z3fold); 7959a001fc1SVitaly Wool 7969a001fc1SVitaly Wool MODULE_LICENSE("GPL"); 7979a001fc1SVitaly Wool MODULE_AUTHOR("Vitaly Wool <vitalywool@gmail.com>"); 7989a001fc1SVitaly Wool MODULE_DESCRIPTION("3-Fold Allocator for Compressed Pages"); 799