19a001fc1SVitaly Wool /* 29a001fc1SVitaly Wool * z3fold.c 39a001fc1SVitaly Wool * 49a001fc1SVitaly Wool * Author: Vitaly Wool <vitaly.wool@konsulko.com> 59a001fc1SVitaly Wool * Copyright (C) 2016, Sony Mobile Communications Inc. 69a001fc1SVitaly Wool * 79a001fc1SVitaly Wool * This implementation is based on zbud written by Seth Jennings. 89a001fc1SVitaly Wool * 99a001fc1SVitaly Wool * z3fold is an special purpose allocator for storing compressed pages. It 109a001fc1SVitaly Wool * can store up to three compressed pages per page which improves the 119a001fc1SVitaly Wool * compression ratio of zbud while retaining its main concepts (e. g. always 129a001fc1SVitaly Wool * storing an integral number of objects per page) and simplicity. 139a001fc1SVitaly Wool * It still has simple and deterministic reclaim properties that make it 149a001fc1SVitaly Wool * preferable to a higher density approach (with no requirement on integral 159a001fc1SVitaly Wool * number of object per page) when reclaim is used. 169a001fc1SVitaly Wool * 179a001fc1SVitaly Wool * As in zbud, pages are divided into "chunks". The size of the chunks is 189a001fc1SVitaly Wool * fixed at compile time and is determined by NCHUNKS_ORDER below. 199a001fc1SVitaly Wool * 209a001fc1SVitaly Wool * z3fold doesn't export any API and is meant to be used via zpool API. 219a001fc1SVitaly Wool */ 229a001fc1SVitaly Wool 239a001fc1SVitaly Wool #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 249a001fc1SVitaly Wool 259a001fc1SVitaly Wool #include <linux/atomic.h> 26d30561c5SVitaly Wool #include <linux/sched.h> 279a001fc1SVitaly Wool #include <linux/list.h> 289a001fc1SVitaly Wool #include <linux/mm.h> 299a001fc1SVitaly Wool #include <linux/module.h> 30d30561c5SVitaly Wool #include <linux/percpu.h> 319a001fc1SVitaly Wool #include <linux/preempt.h> 32d30561c5SVitaly Wool #include <linux/workqueue.h> 339a001fc1SVitaly Wool #include <linux/slab.h> 349a001fc1SVitaly Wool #include <linux/spinlock.h> 359a001fc1SVitaly Wool #include <linux/zpool.h> 369a001fc1SVitaly Wool 379a001fc1SVitaly Wool /***************** 389a001fc1SVitaly Wool * Structures 399a001fc1SVitaly Wool *****************/ 40ede93213SVitaly Wool struct z3fold_pool; 41ede93213SVitaly Wool struct z3fold_ops { 42ede93213SVitaly Wool int (*evict)(struct z3fold_pool *pool, unsigned long handle); 43ede93213SVitaly Wool }; 44ede93213SVitaly Wool 45ede93213SVitaly Wool enum buddy { 46ede93213SVitaly Wool HEADLESS = 0, 47ede93213SVitaly Wool FIRST, 48ede93213SVitaly Wool MIDDLE, 49ede93213SVitaly Wool LAST, 50ede93213SVitaly Wool BUDDIES_MAX 51ede93213SVitaly Wool }; 52ede93213SVitaly Wool 53ede93213SVitaly Wool /* 54d30561c5SVitaly Wool * struct z3fold_header - z3fold page metadata occupying first chunks of each 55ede93213SVitaly Wool * z3fold page, except for HEADLESS pages 56d30561c5SVitaly Wool * @buddy: links the z3fold page into the relevant list in the 57d30561c5SVitaly Wool * pool 582f1e5e4dSVitaly Wool * @page_lock: per-page lock 59d30561c5SVitaly Wool * @refcount: reference count for the z3fold page 60d30561c5SVitaly Wool * @work: work_struct for page layout optimization 61d30561c5SVitaly Wool * @pool: pointer to the pool which this page belongs to 62d30561c5SVitaly Wool * @cpu: CPU which this page "belongs" to 63ede93213SVitaly Wool * @first_chunks: the size of the first buddy in chunks, 0 if free 64ede93213SVitaly Wool * @middle_chunks: the size of the middle buddy in chunks, 0 if free 65ede93213SVitaly Wool * @last_chunks: the size of the last buddy in chunks, 0 if free 66ede93213SVitaly Wool * @first_num: the starting number (for the first handle) 67ede93213SVitaly Wool */ 68ede93213SVitaly Wool struct z3fold_header { 69ede93213SVitaly Wool struct list_head buddy; 702f1e5e4dSVitaly Wool spinlock_t page_lock; 715a27aa82SVitaly Wool struct kref refcount; 72d30561c5SVitaly Wool struct work_struct work; 73d30561c5SVitaly Wool struct z3fold_pool *pool; 74d30561c5SVitaly Wool short cpu; 75ede93213SVitaly Wool unsigned short first_chunks; 76ede93213SVitaly Wool unsigned short middle_chunks; 77ede93213SVitaly Wool unsigned short last_chunks; 78ede93213SVitaly Wool unsigned short start_middle; 79ede93213SVitaly Wool unsigned short first_num:2; 80ede93213SVitaly Wool }; 81ede93213SVitaly Wool 829a001fc1SVitaly Wool /* 839a001fc1SVitaly Wool * NCHUNKS_ORDER determines the internal allocation granularity, effectively 849a001fc1SVitaly Wool * adjusting internal fragmentation. It also determines the number of 859a001fc1SVitaly Wool * freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the 86ede93213SVitaly Wool * allocation granularity will be in chunks of size PAGE_SIZE/64. Some chunks 87ede93213SVitaly Wool * in the beginning of an allocated page are occupied by z3fold header, so 88ede93213SVitaly Wool * NCHUNKS will be calculated to 63 (or 62 in case CONFIG_DEBUG_SPINLOCK=y), 89ede93213SVitaly Wool * which shows the max number of free chunks in z3fold page, also there will 90ede93213SVitaly Wool * be 63, or 62, respectively, freelists per pool. 919a001fc1SVitaly Wool */ 929a001fc1SVitaly Wool #define NCHUNKS_ORDER 6 939a001fc1SVitaly Wool 949a001fc1SVitaly Wool #define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER) 959a001fc1SVitaly Wool #define CHUNK_SIZE (1 << CHUNK_SHIFT) 96ede93213SVitaly Wool #define ZHDR_SIZE_ALIGNED round_up(sizeof(struct z3fold_header), CHUNK_SIZE) 97ede93213SVitaly Wool #define ZHDR_CHUNKS (ZHDR_SIZE_ALIGNED >> CHUNK_SHIFT) 98ede93213SVitaly Wool #define TOTAL_CHUNKS (PAGE_SIZE >> CHUNK_SHIFT) 999a001fc1SVitaly Wool #define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT) 1009a001fc1SVitaly Wool 101f201ebd8Szhong jiang #define BUDDY_MASK (0x3) 102ca0246bbSVitaly Wool #define BUDDY_SHIFT 2 1039a001fc1SVitaly Wool 1049a001fc1SVitaly Wool /** 1059a001fc1SVitaly Wool * struct z3fold_pool - stores metadata for each z3fold pool 106d30561c5SVitaly Wool * @name: pool name 107d30561c5SVitaly Wool * @lock: protects pool unbuddied/lru lists 108d30561c5SVitaly Wool * @stale_lock: protects pool stale page list 109d30561c5SVitaly Wool * @unbuddied: per-cpu array of lists tracking z3fold pages that contain 2- 110d30561c5SVitaly Wool * buddies; the list each z3fold page is added to depends on 111d30561c5SVitaly Wool * the size of its free region. 1129a001fc1SVitaly Wool * @lru: list tracking the z3fold pages in LRU order by most recently 1139a001fc1SVitaly Wool * added buddy. 114d30561c5SVitaly Wool * @stale: list of pages marked for freeing 1159a001fc1SVitaly Wool * @pages_nr: number of z3fold pages in the pool. 1169a001fc1SVitaly Wool * @ops: pointer to a structure of user defined operations specified at 1179a001fc1SVitaly Wool * pool creation time. 118d30561c5SVitaly Wool * @compact_wq: workqueue for page layout background optimization 119d30561c5SVitaly Wool * @release_wq: workqueue for safe page release 120d30561c5SVitaly Wool * @work: work_struct for safe page release 1219a001fc1SVitaly Wool * 1229a001fc1SVitaly Wool * This structure is allocated at pool creation time and maintains metadata 1239a001fc1SVitaly Wool * pertaining to a particular z3fold pool. 1249a001fc1SVitaly Wool */ 1259a001fc1SVitaly Wool struct z3fold_pool { 126d30561c5SVitaly Wool const char *name; 1279a001fc1SVitaly Wool spinlock_t lock; 128d30561c5SVitaly Wool spinlock_t stale_lock; 129d30561c5SVitaly Wool struct list_head *unbuddied; 1309a001fc1SVitaly Wool struct list_head lru; 131d30561c5SVitaly Wool struct list_head stale; 13212d59ae6SVitaly Wool atomic64_t pages_nr; 1339a001fc1SVitaly Wool const struct z3fold_ops *ops; 1349a001fc1SVitaly Wool struct zpool *zpool; 1359a001fc1SVitaly Wool const struct zpool_ops *zpool_ops; 136d30561c5SVitaly Wool struct workqueue_struct *compact_wq; 137d30561c5SVitaly Wool struct workqueue_struct *release_wq; 138d30561c5SVitaly Wool struct work_struct work; 1399a001fc1SVitaly Wool }; 1409a001fc1SVitaly Wool 1419a001fc1SVitaly Wool /* 1429a001fc1SVitaly Wool * Internal z3fold page flags 1439a001fc1SVitaly Wool */ 1449a001fc1SVitaly Wool enum z3fold_page_flags { 1455a27aa82SVitaly Wool PAGE_HEADLESS = 0, 1469a001fc1SVitaly Wool MIDDLE_CHUNK_MAPPED, 147d30561c5SVitaly Wool NEEDS_COMPACTING, 1486098d7e1SVitaly Wool PAGE_STALE, 149ca0246bbSVitaly Wool PAGE_CLAIMED, /* by either reclaim or free */ 1509a001fc1SVitaly Wool }; 1519a001fc1SVitaly Wool 1529a001fc1SVitaly Wool /***************** 1539a001fc1SVitaly Wool * Helpers 1549a001fc1SVitaly Wool *****************/ 1559a001fc1SVitaly Wool 1569a001fc1SVitaly Wool /* Converts an allocation size in bytes to size in z3fold chunks */ 1579a001fc1SVitaly Wool static int size_to_chunks(size_t size) 1589a001fc1SVitaly Wool { 1599a001fc1SVitaly Wool return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; 1609a001fc1SVitaly Wool } 1619a001fc1SVitaly Wool 1629a001fc1SVitaly Wool #define for_each_unbuddied_list(_iter, _begin) \ 1639a001fc1SVitaly Wool for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++) 1649a001fc1SVitaly Wool 165d30561c5SVitaly Wool static void compact_page_work(struct work_struct *w); 166d30561c5SVitaly Wool 1679a001fc1SVitaly Wool /* Initializes the z3fold header of a newly allocated z3fold page */ 168d30561c5SVitaly Wool static struct z3fold_header *init_z3fold_page(struct page *page, 169d30561c5SVitaly Wool struct z3fold_pool *pool) 1709a001fc1SVitaly Wool { 1719a001fc1SVitaly Wool struct z3fold_header *zhdr = page_address(page); 1729a001fc1SVitaly Wool 1739a001fc1SVitaly Wool INIT_LIST_HEAD(&page->lru); 1749a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 1759a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 176d30561c5SVitaly Wool clear_bit(NEEDS_COMPACTING, &page->private); 177d30561c5SVitaly Wool clear_bit(PAGE_STALE, &page->private); 178ca0246bbSVitaly Wool clear_bit(PAGE_CLAIMED, &page->private); 1799a001fc1SVitaly Wool 1802f1e5e4dSVitaly Wool spin_lock_init(&zhdr->page_lock); 1815a27aa82SVitaly Wool kref_init(&zhdr->refcount); 1829a001fc1SVitaly Wool zhdr->first_chunks = 0; 1839a001fc1SVitaly Wool zhdr->middle_chunks = 0; 1849a001fc1SVitaly Wool zhdr->last_chunks = 0; 1859a001fc1SVitaly Wool zhdr->first_num = 0; 1869a001fc1SVitaly Wool zhdr->start_middle = 0; 187d30561c5SVitaly Wool zhdr->cpu = -1; 188d30561c5SVitaly Wool zhdr->pool = pool; 1899a001fc1SVitaly Wool INIT_LIST_HEAD(&zhdr->buddy); 190d30561c5SVitaly Wool INIT_WORK(&zhdr->work, compact_page_work); 1919a001fc1SVitaly Wool return zhdr; 1929a001fc1SVitaly Wool } 1939a001fc1SVitaly Wool 1949a001fc1SVitaly Wool /* Resets the struct page fields and frees the page */ 1955a27aa82SVitaly Wool static void free_z3fold_page(struct page *page) 1969a001fc1SVitaly Wool { 1975a27aa82SVitaly Wool __free_page(page); 1985a27aa82SVitaly Wool } 1995a27aa82SVitaly Wool 2002f1e5e4dSVitaly Wool /* Lock a z3fold page */ 2012f1e5e4dSVitaly Wool static inline void z3fold_page_lock(struct z3fold_header *zhdr) 2022f1e5e4dSVitaly Wool { 2032f1e5e4dSVitaly Wool spin_lock(&zhdr->page_lock); 2042f1e5e4dSVitaly Wool } 2052f1e5e4dSVitaly Wool 20676e32a2aSVitaly Wool /* Try to lock a z3fold page */ 20776e32a2aSVitaly Wool static inline int z3fold_page_trylock(struct z3fold_header *zhdr) 20876e32a2aSVitaly Wool { 20976e32a2aSVitaly Wool return spin_trylock(&zhdr->page_lock); 21076e32a2aSVitaly Wool } 21176e32a2aSVitaly Wool 2122f1e5e4dSVitaly Wool /* Unlock a z3fold page */ 2132f1e5e4dSVitaly Wool static inline void z3fold_page_unlock(struct z3fold_header *zhdr) 2142f1e5e4dSVitaly Wool { 2152f1e5e4dSVitaly Wool spin_unlock(&zhdr->page_lock); 2162f1e5e4dSVitaly Wool } 2172f1e5e4dSVitaly Wool 2189a001fc1SVitaly Wool /* 2199a001fc1SVitaly Wool * Encodes the handle of a particular buddy within a z3fold page 2209a001fc1SVitaly Wool * Pool lock should be held as this function accesses first_num 2219a001fc1SVitaly Wool */ 2229a001fc1SVitaly Wool static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) 2239a001fc1SVitaly Wool { 2249a001fc1SVitaly Wool unsigned long handle; 2259a001fc1SVitaly Wool 2269a001fc1SVitaly Wool handle = (unsigned long)zhdr; 227ca0246bbSVitaly Wool if (bud != HEADLESS) { 228ca0246bbSVitaly Wool handle |= (bud + zhdr->first_num) & BUDDY_MASK; 229ca0246bbSVitaly Wool if (bud == LAST) 230ca0246bbSVitaly Wool handle |= (zhdr->last_chunks << BUDDY_SHIFT); 231ca0246bbSVitaly Wool } 2329a001fc1SVitaly Wool return handle; 2339a001fc1SVitaly Wool } 2349a001fc1SVitaly Wool 2359a001fc1SVitaly Wool /* Returns the z3fold page where a given handle is stored */ 2369a001fc1SVitaly Wool static struct z3fold_header *handle_to_z3fold_header(unsigned long handle) 2379a001fc1SVitaly Wool { 2389a001fc1SVitaly Wool return (struct z3fold_header *)(handle & PAGE_MASK); 2399a001fc1SVitaly Wool } 2409a001fc1SVitaly Wool 241ca0246bbSVitaly Wool /* only for LAST bud, returns zero otherwise */ 242ca0246bbSVitaly Wool static unsigned short handle_to_chunks(unsigned long handle) 243ca0246bbSVitaly Wool { 244ca0246bbSVitaly Wool return (handle & ~PAGE_MASK) >> BUDDY_SHIFT; 245ca0246bbSVitaly Wool } 246ca0246bbSVitaly Wool 247f201ebd8Szhong jiang /* 248f201ebd8Szhong jiang * (handle & BUDDY_MASK) < zhdr->first_num is possible in encode_handle 249f201ebd8Szhong jiang * but that doesn't matter. because the masking will result in the 250f201ebd8Szhong jiang * correct buddy number. 251f201ebd8Szhong jiang */ 2529a001fc1SVitaly Wool static enum buddy handle_to_buddy(unsigned long handle) 2539a001fc1SVitaly Wool { 2549a001fc1SVitaly Wool struct z3fold_header *zhdr = handle_to_z3fold_header(handle); 2559a001fc1SVitaly Wool return (handle - zhdr->first_num) & BUDDY_MASK; 2569a001fc1SVitaly Wool } 2579a001fc1SVitaly Wool 258*9050cce1SVitaly Wool static inline struct z3fold_pool *zhdr_to_pool(struct z3fold_header *zhdr) 259*9050cce1SVitaly Wool { 260*9050cce1SVitaly Wool return zhdr->pool; 261*9050cce1SVitaly Wool } 262*9050cce1SVitaly Wool 263d30561c5SVitaly Wool static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) 264d30561c5SVitaly Wool { 265d30561c5SVitaly Wool struct page *page = virt_to_page(zhdr); 266*9050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 267d30561c5SVitaly Wool 268d30561c5SVitaly Wool WARN_ON(!list_empty(&zhdr->buddy)); 269d30561c5SVitaly Wool set_bit(PAGE_STALE, &page->private); 27035529357SVitaly Wool clear_bit(NEEDS_COMPACTING, &page->private); 271d30561c5SVitaly Wool spin_lock(&pool->lock); 272d30561c5SVitaly Wool if (!list_empty(&page->lru)) 273d30561c5SVitaly Wool list_del(&page->lru); 274d30561c5SVitaly Wool spin_unlock(&pool->lock); 275d30561c5SVitaly Wool if (locked) 276d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 277d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 278d30561c5SVitaly Wool list_add(&zhdr->buddy, &pool->stale); 279d30561c5SVitaly Wool queue_work(pool->release_wq, &pool->work); 280d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 281d30561c5SVitaly Wool } 282d30561c5SVitaly Wool 283d30561c5SVitaly Wool static void __attribute__((__unused__)) 284d30561c5SVitaly Wool release_z3fold_page(struct kref *ref) 285d30561c5SVitaly Wool { 286d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 287d30561c5SVitaly Wool refcount); 288d30561c5SVitaly Wool __release_z3fold_page(zhdr, false); 289d30561c5SVitaly Wool } 290d30561c5SVitaly Wool 291d30561c5SVitaly Wool static void release_z3fold_page_locked(struct kref *ref) 292d30561c5SVitaly Wool { 293d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 294d30561c5SVitaly Wool refcount); 295d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 296d30561c5SVitaly Wool __release_z3fold_page(zhdr, true); 297d30561c5SVitaly Wool } 298d30561c5SVitaly Wool 299d30561c5SVitaly Wool static void release_z3fold_page_locked_list(struct kref *ref) 300d30561c5SVitaly Wool { 301d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 302d30561c5SVitaly Wool refcount); 303*9050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 304*9050cce1SVitaly Wool spin_lock(&pool->lock); 305d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 306*9050cce1SVitaly Wool spin_unlock(&pool->lock); 307d30561c5SVitaly Wool 308d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 309d30561c5SVitaly Wool __release_z3fold_page(zhdr, true); 310d30561c5SVitaly Wool } 311d30561c5SVitaly Wool 312d30561c5SVitaly Wool static void free_pages_work(struct work_struct *w) 313d30561c5SVitaly Wool { 314d30561c5SVitaly Wool struct z3fold_pool *pool = container_of(w, struct z3fold_pool, work); 315d30561c5SVitaly Wool 316d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 317d30561c5SVitaly Wool while (!list_empty(&pool->stale)) { 318d30561c5SVitaly Wool struct z3fold_header *zhdr = list_first_entry(&pool->stale, 319d30561c5SVitaly Wool struct z3fold_header, buddy); 320d30561c5SVitaly Wool struct page *page = virt_to_page(zhdr); 321d30561c5SVitaly Wool 322d30561c5SVitaly Wool list_del(&zhdr->buddy); 323d30561c5SVitaly Wool if (WARN_ON(!test_bit(PAGE_STALE, &page->private))) 324d30561c5SVitaly Wool continue; 325d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 326d30561c5SVitaly Wool cancel_work_sync(&zhdr->work); 327d30561c5SVitaly Wool free_z3fold_page(page); 328d30561c5SVitaly Wool cond_resched(); 329d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 330d30561c5SVitaly Wool } 331d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 332d30561c5SVitaly Wool } 333d30561c5SVitaly Wool 3349a001fc1SVitaly Wool /* 3359a001fc1SVitaly Wool * Returns the number of free chunks in a z3fold page. 3369a001fc1SVitaly Wool * NB: can't be used with HEADLESS pages. 3379a001fc1SVitaly Wool */ 3389a001fc1SVitaly Wool static int num_free_chunks(struct z3fold_header *zhdr) 3399a001fc1SVitaly Wool { 3409a001fc1SVitaly Wool int nfree; 3419a001fc1SVitaly Wool /* 3429a001fc1SVitaly Wool * If there is a middle object, pick up the bigger free space 3439a001fc1SVitaly Wool * either before or after it. Otherwise just subtract the number 3449a001fc1SVitaly Wool * of chunks occupied by the first and the last objects. 3459a001fc1SVitaly Wool */ 3469a001fc1SVitaly Wool if (zhdr->middle_chunks != 0) { 3479a001fc1SVitaly Wool int nfree_before = zhdr->first_chunks ? 348ede93213SVitaly Wool 0 : zhdr->start_middle - ZHDR_CHUNKS; 3499a001fc1SVitaly Wool int nfree_after = zhdr->last_chunks ? 350ede93213SVitaly Wool 0 : TOTAL_CHUNKS - 351ede93213SVitaly Wool (zhdr->start_middle + zhdr->middle_chunks); 3529a001fc1SVitaly Wool nfree = max(nfree_before, nfree_after); 3539a001fc1SVitaly Wool } else 3549a001fc1SVitaly Wool nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; 3559a001fc1SVitaly Wool return nfree; 3569a001fc1SVitaly Wool } 3579a001fc1SVitaly Wool 358*9050cce1SVitaly Wool /* Add to the appropriate unbuddied list */ 359*9050cce1SVitaly Wool static inline void add_to_unbuddied(struct z3fold_pool *pool, 360*9050cce1SVitaly Wool struct z3fold_header *zhdr) 361*9050cce1SVitaly Wool { 362*9050cce1SVitaly Wool if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || 363*9050cce1SVitaly Wool zhdr->middle_chunks == 0) { 364*9050cce1SVitaly Wool struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied); 365*9050cce1SVitaly Wool 366*9050cce1SVitaly Wool int freechunks = num_free_chunks(zhdr); 367*9050cce1SVitaly Wool spin_lock(&pool->lock); 368*9050cce1SVitaly Wool list_add(&zhdr->buddy, &unbuddied[freechunks]); 369*9050cce1SVitaly Wool spin_unlock(&pool->lock); 370*9050cce1SVitaly Wool zhdr->cpu = smp_processor_id(); 371*9050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 372*9050cce1SVitaly Wool } 373*9050cce1SVitaly Wool } 374*9050cce1SVitaly Wool 375ede93213SVitaly Wool static inline void *mchunk_memmove(struct z3fold_header *zhdr, 376ede93213SVitaly Wool unsigned short dst_chunk) 377ede93213SVitaly Wool { 378ede93213SVitaly Wool void *beg = zhdr; 379ede93213SVitaly Wool return memmove(beg + (dst_chunk << CHUNK_SHIFT), 380ede93213SVitaly Wool beg + (zhdr->start_middle << CHUNK_SHIFT), 381ede93213SVitaly Wool zhdr->middle_chunks << CHUNK_SHIFT); 382ede93213SVitaly Wool } 383ede93213SVitaly Wool 3841b096e5aSVitaly Wool #define BIG_CHUNK_GAP 3 3859a001fc1SVitaly Wool /* Has to be called with lock held */ 3869a001fc1SVitaly Wool static int z3fold_compact_page(struct z3fold_header *zhdr) 3879a001fc1SVitaly Wool { 3889a001fc1SVitaly Wool struct page *page = virt_to_page(zhdr); 3899a001fc1SVitaly Wool 390ede93213SVitaly Wool if (test_bit(MIDDLE_CHUNK_MAPPED, &page->private)) 391ede93213SVitaly Wool return 0; /* can't move middle chunk, it's used */ 3929a001fc1SVitaly Wool 393ede93213SVitaly Wool if (zhdr->middle_chunks == 0) 394ede93213SVitaly Wool return 0; /* nothing to compact */ 395ede93213SVitaly Wool 396ede93213SVitaly Wool if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) { 397ede93213SVitaly Wool /* move to the beginning */ 398ede93213SVitaly Wool mchunk_memmove(zhdr, ZHDR_CHUNKS); 3999a001fc1SVitaly Wool zhdr->first_chunks = zhdr->middle_chunks; 4009a001fc1SVitaly Wool zhdr->middle_chunks = 0; 4019a001fc1SVitaly Wool zhdr->start_middle = 0; 4029a001fc1SVitaly Wool zhdr->first_num++; 403ede93213SVitaly Wool return 1; 4049a001fc1SVitaly Wool } 4059a001fc1SVitaly Wool 4061b096e5aSVitaly Wool /* 4071b096e5aSVitaly Wool * moving data is expensive, so let's only do that if 4081b096e5aSVitaly Wool * there's substantial gain (at least BIG_CHUNK_GAP chunks) 4091b096e5aSVitaly Wool */ 4101b096e5aSVitaly Wool if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 && 4111b096e5aSVitaly Wool zhdr->start_middle - (zhdr->first_chunks + ZHDR_CHUNKS) >= 4121b096e5aSVitaly Wool BIG_CHUNK_GAP) { 4131b096e5aSVitaly Wool mchunk_memmove(zhdr, zhdr->first_chunks + ZHDR_CHUNKS); 4141b096e5aSVitaly Wool zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; 4151b096e5aSVitaly Wool return 1; 4161b096e5aSVitaly Wool } else if (zhdr->last_chunks != 0 && zhdr->first_chunks == 0 && 4171b096e5aSVitaly Wool TOTAL_CHUNKS - (zhdr->last_chunks + zhdr->start_middle 4181b096e5aSVitaly Wool + zhdr->middle_chunks) >= 4191b096e5aSVitaly Wool BIG_CHUNK_GAP) { 4201b096e5aSVitaly Wool unsigned short new_start = TOTAL_CHUNKS - zhdr->last_chunks - 4211b096e5aSVitaly Wool zhdr->middle_chunks; 4221b096e5aSVitaly Wool mchunk_memmove(zhdr, new_start); 4231b096e5aSVitaly Wool zhdr->start_middle = new_start; 4241b096e5aSVitaly Wool return 1; 4251b096e5aSVitaly Wool } 4261b096e5aSVitaly Wool 4271b096e5aSVitaly Wool return 0; 4281b096e5aSVitaly Wool } 4291b096e5aSVitaly Wool 430d30561c5SVitaly Wool static void do_compact_page(struct z3fold_header *zhdr, bool locked) 431d30561c5SVitaly Wool { 432*9050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 433d30561c5SVitaly Wool struct page *page; 434d30561c5SVitaly Wool 435d30561c5SVitaly Wool page = virt_to_page(zhdr); 436d30561c5SVitaly Wool if (locked) 437d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 438d30561c5SVitaly Wool else 439d30561c5SVitaly Wool z3fold_page_lock(zhdr); 4405d03a661SVitaly Wool if (WARN_ON(!test_and_clear_bit(NEEDS_COMPACTING, &page->private))) { 441d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 442d30561c5SVitaly Wool return; 443d30561c5SVitaly Wool } 444d30561c5SVitaly Wool spin_lock(&pool->lock); 445d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 446d30561c5SVitaly Wool spin_unlock(&pool->lock); 447d30561c5SVitaly Wool 4485d03a661SVitaly Wool if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { 4495d03a661SVitaly Wool atomic64_dec(&pool->pages_nr); 4505d03a661SVitaly Wool return; 4515d03a661SVitaly Wool } 4525d03a661SVitaly Wool 453d30561c5SVitaly Wool z3fold_compact_page(zhdr); 454*9050cce1SVitaly Wool add_to_unbuddied(pool, zhdr); 455d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 456d30561c5SVitaly Wool } 457d30561c5SVitaly Wool 458d30561c5SVitaly Wool static void compact_page_work(struct work_struct *w) 459d30561c5SVitaly Wool { 460d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(w, struct z3fold_header, 461d30561c5SVitaly Wool work); 462d30561c5SVitaly Wool 463d30561c5SVitaly Wool do_compact_page(zhdr, false); 464d30561c5SVitaly Wool } 465d30561c5SVitaly Wool 466*9050cce1SVitaly Wool /* returns _locked_ z3fold page header or NULL */ 467*9050cce1SVitaly Wool static inline struct z3fold_header *__z3fold_alloc(struct z3fold_pool *pool, 468*9050cce1SVitaly Wool size_t size, bool can_sleep) 469*9050cce1SVitaly Wool { 470*9050cce1SVitaly Wool struct z3fold_header *zhdr = NULL; 471*9050cce1SVitaly Wool struct page *page; 472*9050cce1SVitaly Wool struct list_head *unbuddied; 473*9050cce1SVitaly Wool int chunks = size_to_chunks(size), i; 474*9050cce1SVitaly Wool 475*9050cce1SVitaly Wool lookup: 476*9050cce1SVitaly Wool /* First, try to find an unbuddied z3fold page. */ 477*9050cce1SVitaly Wool unbuddied = get_cpu_ptr(pool->unbuddied); 478*9050cce1SVitaly Wool for_each_unbuddied_list(i, chunks) { 479*9050cce1SVitaly Wool struct list_head *l = &unbuddied[i]; 480*9050cce1SVitaly Wool 481*9050cce1SVitaly Wool zhdr = list_first_entry_or_null(READ_ONCE(l), 482*9050cce1SVitaly Wool struct z3fold_header, buddy); 483*9050cce1SVitaly Wool 484*9050cce1SVitaly Wool if (!zhdr) 485*9050cce1SVitaly Wool continue; 486*9050cce1SVitaly Wool 487*9050cce1SVitaly Wool /* Re-check under lock. */ 488*9050cce1SVitaly Wool spin_lock(&pool->lock); 489*9050cce1SVitaly Wool l = &unbuddied[i]; 490*9050cce1SVitaly Wool if (unlikely(zhdr != list_first_entry(READ_ONCE(l), 491*9050cce1SVitaly Wool struct z3fold_header, buddy)) || 492*9050cce1SVitaly Wool !z3fold_page_trylock(zhdr)) { 493*9050cce1SVitaly Wool spin_unlock(&pool->lock); 494*9050cce1SVitaly Wool zhdr = NULL; 495*9050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 496*9050cce1SVitaly Wool if (can_sleep) 497*9050cce1SVitaly Wool cond_resched(); 498*9050cce1SVitaly Wool goto lookup; 499*9050cce1SVitaly Wool } 500*9050cce1SVitaly Wool list_del_init(&zhdr->buddy); 501*9050cce1SVitaly Wool zhdr->cpu = -1; 502*9050cce1SVitaly Wool spin_unlock(&pool->lock); 503*9050cce1SVitaly Wool 504*9050cce1SVitaly Wool page = virt_to_page(zhdr); 505*9050cce1SVitaly Wool if (test_bit(NEEDS_COMPACTING, &page->private)) { 506*9050cce1SVitaly Wool z3fold_page_unlock(zhdr); 507*9050cce1SVitaly Wool zhdr = NULL; 508*9050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 509*9050cce1SVitaly Wool if (can_sleep) 510*9050cce1SVitaly Wool cond_resched(); 511*9050cce1SVitaly Wool goto lookup; 512*9050cce1SVitaly Wool } 513*9050cce1SVitaly Wool 514*9050cce1SVitaly Wool /* 515*9050cce1SVitaly Wool * this page could not be removed from its unbuddied 516*9050cce1SVitaly Wool * list while pool lock was held, and then we've taken 517*9050cce1SVitaly Wool * page lock so kref_put could not be called before 518*9050cce1SVitaly Wool * we got here, so it's safe to just call kref_get() 519*9050cce1SVitaly Wool */ 520*9050cce1SVitaly Wool kref_get(&zhdr->refcount); 521*9050cce1SVitaly Wool break; 522*9050cce1SVitaly Wool } 523*9050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 524*9050cce1SVitaly Wool 525*9050cce1SVitaly Wool return zhdr; 526*9050cce1SVitaly Wool } 527d30561c5SVitaly Wool 528d30561c5SVitaly Wool /* 529d30561c5SVitaly Wool * API Functions 530d30561c5SVitaly Wool */ 531d30561c5SVitaly Wool 532d30561c5SVitaly Wool /** 533d30561c5SVitaly Wool * z3fold_create_pool() - create a new z3fold pool 534d30561c5SVitaly Wool * @name: pool name 535d30561c5SVitaly Wool * @gfp: gfp flags when allocating the z3fold pool structure 536d30561c5SVitaly Wool * @ops: user-defined operations for the z3fold pool 537d30561c5SVitaly Wool * 538d30561c5SVitaly Wool * Return: pointer to the new z3fold pool or NULL if the metadata allocation 539d30561c5SVitaly Wool * failed. 540d30561c5SVitaly Wool */ 541d30561c5SVitaly Wool static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp, 542d30561c5SVitaly Wool const struct z3fold_ops *ops) 543d30561c5SVitaly Wool { 544d30561c5SVitaly Wool struct z3fold_pool *pool = NULL; 545d30561c5SVitaly Wool int i, cpu; 546d30561c5SVitaly Wool 547d30561c5SVitaly Wool pool = kzalloc(sizeof(struct z3fold_pool), gfp); 548d30561c5SVitaly Wool if (!pool) 549d30561c5SVitaly Wool goto out; 550d30561c5SVitaly Wool spin_lock_init(&pool->lock); 551d30561c5SVitaly Wool spin_lock_init(&pool->stale_lock); 552d30561c5SVitaly Wool pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2); 5531ec6995dSXidong Wang if (!pool->unbuddied) 5541ec6995dSXidong Wang goto out_pool; 555d30561c5SVitaly Wool for_each_possible_cpu(cpu) { 556d30561c5SVitaly Wool struct list_head *unbuddied = 557d30561c5SVitaly Wool per_cpu_ptr(pool->unbuddied, cpu); 558d30561c5SVitaly Wool for_each_unbuddied_list(i, 0) 559d30561c5SVitaly Wool INIT_LIST_HEAD(&unbuddied[i]); 560d30561c5SVitaly Wool } 561d30561c5SVitaly Wool INIT_LIST_HEAD(&pool->lru); 562d30561c5SVitaly Wool INIT_LIST_HEAD(&pool->stale); 563d30561c5SVitaly Wool atomic64_set(&pool->pages_nr, 0); 564d30561c5SVitaly Wool pool->name = name; 565d30561c5SVitaly Wool pool->compact_wq = create_singlethread_workqueue(pool->name); 566d30561c5SVitaly Wool if (!pool->compact_wq) 5671ec6995dSXidong Wang goto out_unbuddied; 568d30561c5SVitaly Wool pool->release_wq = create_singlethread_workqueue(pool->name); 569d30561c5SVitaly Wool if (!pool->release_wq) 570d30561c5SVitaly Wool goto out_wq; 571d30561c5SVitaly Wool INIT_WORK(&pool->work, free_pages_work); 572d30561c5SVitaly Wool pool->ops = ops; 573d30561c5SVitaly Wool return pool; 574d30561c5SVitaly Wool 575d30561c5SVitaly Wool out_wq: 576d30561c5SVitaly Wool destroy_workqueue(pool->compact_wq); 5771ec6995dSXidong Wang out_unbuddied: 5781ec6995dSXidong Wang free_percpu(pool->unbuddied); 5791ec6995dSXidong Wang out_pool: 580d30561c5SVitaly Wool kfree(pool); 5811ec6995dSXidong Wang out: 582d30561c5SVitaly Wool return NULL; 583d30561c5SVitaly Wool } 584d30561c5SVitaly Wool 585d30561c5SVitaly Wool /** 586d30561c5SVitaly Wool * z3fold_destroy_pool() - destroys an existing z3fold pool 587d30561c5SVitaly Wool * @pool: the z3fold pool to be destroyed 588d30561c5SVitaly Wool * 589d30561c5SVitaly Wool * The pool should be emptied before this function is called. 590d30561c5SVitaly Wool */ 591d30561c5SVitaly Wool static void z3fold_destroy_pool(struct z3fold_pool *pool) 592d30561c5SVitaly Wool { 593d30561c5SVitaly Wool destroy_workqueue(pool->release_wq); 594d30561c5SVitaly Wool destroy_workqueue(pool->compact_wq); 595d30561c5SVitaly Wool kfree(pool); 596d30561c5SVitaly Wool } 597d30561c5SVitaly Wool 5989a001fc1SVitaly Wool /** 5999a001fc1SVitaly Wool * z3fold_alloc() - allocates a region of a given size 6009a001fc1SVitaly Wool * @pool: z3fold pool from which to allocate 6019a001fc1SVitaly Wool * @size: size in bytes of the desired allocation 6029a001fc1SVitaly Wool * @gfp: gfp flags used if the pool needs to grow 6039a001fc1SVitaly Wool * @handle: handle of the new allocation 6049a001fc1SVitaly Wool * 6059a001fc1SVitaly Wool * This function will attempt to find a free region in the pool large enough to 6069a001fc1SVitaly Wool * satisfy the allocation request. A search of the unbuddied lists is 6079a001fc1SVitaly Wool * performed first. If no suitable free region is found, then a new page is 6089a001fc1SVitaly Wool * allocated and added to the pool to satisfy the request. 6099a001fc1SVitaly Wool * 6109a001fc1SVitaly Wool * gfp should not set __GFP_HIGHMEM as highmem pages cannot be used 6119a001fc1SVitaly Wool * as z3fold pool pages. 6129a001fc1SVitaly Wool * 6139a001fc1SVitaly Wool * Return: 0 if success and handle is set, otherwise -EINVAL if the size or 6149a001fc1SVitaly Wool * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate 6159a001fc1SVitaly Wool * a new page. 6169a001fc1SVitaly Wool */ 6179a001fc1SVitaly Wool static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, 6189a001fc1SVitaly Wool unsigned long *handle) 6199a001fc1SVitaly Wool { 620*9050cce1SVitaly Wool int chunks = size_to_chunks(size); 6219a001fc1SVitaly Wool struct z3fold_header *zhdr = NULL; 622d30561c5SVitaly Wool struct page *page = NULL; 6239a001fc1SVitaly Wool enum buddy bud; 6248a97ea54SMatthew Wilcox bool can_sleep = gfpflags_allow_blocking(gfp); 6259a001fc1SVitaly Wool 6269a001fc1SVitaly Wool if (!size || (gfp & __GFP_HIGHMEM)) 6279a001fc1SVitaly Wool return -EINVAL; 6289a001fc1SVitaly Wool 6299a001fc1SVitaly Wool if (size > PAGE_SIZE) 6309a001fc1SVitaly Wool return -ENOSPC; 6319a001fc1SVitaly Wool 6329a001fc1SVitaly Wool if (size > PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE) 6339a001fc1SVitaly Wool bud = HEADLESS; 6349a001fc1SVitaly Wool else { 635*9050cce1SVitaly Wool retry: 636*9050cce1SVitaly Wool zhdr = __z3fold_alloc(pool, size, can_sleep); 637d30561c5SVitaly Wool if (zhdr) { 6389a001fc1SVitaly Wool if (zhdr->first_chunks == 0) { 6399a001fc1SVitaly Wool if (zhdr->middle_chunks != 0 && 6409a001fc1SVitaly Wool chunks >= zhdr->start_middle) 6419a001fc1SVitaly Wool bud = LAST; 6429a001fc1SVitaly Wool else 6439a001fc1SVitaly Wool bud = FIRST; 6449a001fc1SVitaly Wool } else if (zhdr->last_chunks == 0) 6459a001fc1SVitaly Wool bud = LAST; 6469a001fc1SVitaly Wool else if (zhdr->middle_chunks == 0) 6479a001fc1SVitaly Wool bud = MIDDLE; 6489a001fc1SVitaly Wool else { 6495a27aa82SVitaly Wool if (kref_put(&zhdr->refcount, 650d30561c5SVitaly Wool release_z3fold_page_locked)) 6515a27aa82SVitaly Wool atomic64_dec(&pool->pages_nr); 652d30561c5SVitaly Wool else 653d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 6549a001fc1SVitaly Wool pr_err("No free chunks in unbuddied\n"); 6559a001fc1SVitaly Wool WARN_ON(1); 656*9050cce1SVitaly Wool goto retry; 6579a001fc1SVitaly Wool } 658*9050cce1SVitaly Wool page = virt_to_page(zhdr); 6599a001fc1SVitaly Wool goto found; 6609a001fc1SVitaly Wool } 6619a001fc1SVitaly Wool bud = FIRST; 6629a001fc1SVitaly Wool } 6639a001fc1SVitaly Wool 6645c9bab59SVitaly Wool page = NULL; 6655c9bab59SVitaly Wool if (can_sleep) { 666d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 667d30561c5SVitaly Wool zhdr = list_first_entry_or_null(&pool->stale, 668d30561c5SVitaly Wool struct z3fold_header, buddy); 669d30561c5SVitaly Wool /* 6705c9bab59SVitaly Wool * Before allocating a page, let's see if we can take one from 6715c9bab59SVitaly Wool * the stale pages list. cancel_work_sync() can sleep so we 6725c9bab59SVitaly Wool * limit this case to the contexts where we can sleep 673d30561c5SVitaly Wool */ 6745c9bab59SVitaly Wool if (zhdr) { 675d30561c5SVitaly Wool list_del(&zhdr->buddy); 676d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 677d30561c5SVitaly Wool cancel_work_sync(&zhdr->work); 678d30561c5SVitaly Wool page = virt_to_page(zhdr); 679d30561c5SVitaly Wool } else { 680d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 681d30561c5SVitaly Wool } 6825c9bab59SVitaly Wool } 6835c9bab59SVitaly Wool if (!page) 6845c9bab59SVitaly Wool page = alloc_page(gfp); 685d30561c5SVitaly Wool 6869a001fc1SVitaly Wool if (!page) 6879a001fc1SVitaly Wool return -ENOMEM; 6882f1e5e4dSVitaly Wool 689d30561c5SVitaly Wool zhdr = init_z3fold_page(page, pool); 690*9050cce1SVitaly Wool if (!zhdr) { 691*9050cce1SVitaly Wool __free_page(page); 692*9050cce1SVitaly Wool return -ENOMEM; 693*9050cce1SVitaly Wool } 694*9050cce1SVitaly Wool atomic64_inc(&pool->pages_nr); 6959a001fc1SVitaly Wool 6969a001fc1SVitaly Wool if (bud == HEADLESS) { 6979a001fc1SVitaly Wool set_bit(PAGE_HEADLESS, &page->private); 6989a001fc1SVitaly Wool goto headless; 6999a001fc1SVitaly Wool } 7002f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 7019a001fc1SVitaly Wool 7029a001fc1SVitaly Wool found: 7039a001fc1SVitaly Wool if (bud == FIRST) 7049a001fc1SVitaly Wool zhdr->first_chunks = chunks; 7059a001fc1SVitaly Wool else if (bud == LAST) 7069a001fc1SVitaly Wool zhdr->last_chunks = chunks; 7079a001fc1SVitaly Wool else { 7089a001fc1SVitaly Wool zhdr->middle_chunks = chunks; 709ede93213SVitaly Wool zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; 7109a001fc1SVitaly Wool } 711*9050cce1SVitaly Wool add_to_unbuddied(pool, zhdr); 7129a001fc1SVitaly Wool 7139a001fc1SVitaly Wool headless: 714d30561c5SVitaly Wool spin_lock(&pool->lock); 7159a001fc1SVitaly Wool /* Add/move z3fold page to beginning of LRU */ 7169a001fc1SVitaly Wool if (!list_empty(&page->lru)) 7179a001fc1SVitaly Wool list_del(&page->lru); 7189a001fc1SVitaly Wool 7199a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 7209a001fc1SVitaly Wool 7219a001fc1SVitaly Wool *handle = encode_handle(zhdr, bud); 7229a001fc1SVitaly Wool spin_unlock(&pool->lock); 7232f1e5e4dSVitaly Wool if (bud != HEADLESS) 7242f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 7259a001fc1SVitaly Wool 7269a001fc1SVitaly Wool return 0; 7279a001fc1SVitaly Wool } 7289a001fc1SVitaly Wool 7299a001fc1SVitaly Wool /** 7309a001fc1SVitaly Wool * z3fold_free() - frees the allocation associated with the given handle 7319a001fc1SVitaly Wool * @pool: pool in which the allocation resided 7329a001fc1SVitaly Wool * @handle: handle associated with the allocation returned by z3fold_alloc() 7339a001fc1SVitaly Wool * 7349a001fc1SVitaly Wool * In the case that the z3fold page in which the allocation resides is under 7359a001fc1SVitaly Wool * reclaim, as indicated by the PG_reclaim flag being set, this function 7369a001fc1SVitaly Wool * only sets the first|last_chunks to 0. The page is actually freed 7379a001fc1SVitaly Wool * once both buddies are evicted (see z3fold_reclaim_page() below). 7389a001fc1SVitaly Wool */ 7399a001fc1SVitaly Wool static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) 7409a001fc1SVitaly Wool { 7419a001fc1SVitaly Wool struct z3fold_header *zhdr; 7429a001fc1SVitaly Wool struct page *page; 7439a001fc1SVitaly Wool enum buddy bud; 7449a001fc1SVitaly Wool 7459a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 7469a001fc1SVitaly Wool page = virt_to_page(zhdr); 7479a001fc1SVitaly Wool 7489a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 749ca0246bbSVitaly Wool /* if a headless page is under reclaim, just leave. 750ca0246bbSVitaly Wool * NB: we use test_and_set_bit for a reason: if the bit 751ca0246bbSVitaly Wool * has not been set before, we release this page 752ca0246bbSVitaly Wool * immediately so we don't care about its value any more. 753ca0246bbSVitaly Wool */ 754ca0246bbSVitaly Wool if (!test_and_set_bit(PAGE_CLAIMED, &page->private)) { 755ca0246bbSVitaly Wool spin_lock(&pool->lock); 756ca0246bbSVitaly Wool list_del(&page->lru); 757ca0246bbSVitaly Wool spin_unlock(&pool->lock); 758ca0246bbSVitaly Wool free_z3fold_page(page); 759ca0246bbSVitaly Wool atomic64_dec(&pool->pages_nr); 760ca0246bbSVitaly Wool } 761ca0246bbSVitaly Wool return; 762ca0246bbSVitaly Wool } 763ca0246bbSVitaly Wool 764ca0246bbSVitaly Wool /* Non-headless case */ 7652f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 76643afc194SVitaly Wool bud = handle_to_buddy(handle); 7679a001fc1SVitaly Wool 7689a001fc1SVitaly Wool switch (bud) { 7699a001fc1SVitaly Wool case FIRST: 7709a001fc1SVitaly Wool zhdr->first_chunks = 0; 7719a001fc1SVitaly Wool break; 7729a001fc1SVitaly Wool case MIDDLE: 7739a001fc1SVitaly Wool zhdr->middle_chunks = 0; 7749a001fc1SVitaly Wool break; 7759a001fc1SVitaly Wool case LAST: 7769a001fc1SVitaly Wool zhdr->last_chunks = 0; 7779a001fc1SVitaly Wool break; 7789a001fc1SVitaly Wool default: 7799a001fc1SVitaly Wool pr_err("%s: unknown bud %d\n", __func__, bud); 7809a001fc1SVitaly Wool WARN_ON(1); 7812f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 7829a001fc1SVitaly Wool return; 7839a001fc1SVitaly Wool } 7849a001fc1SVitaly Wool 785d30561c5SVitaly Wool if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) { 786d30561c5SVitaly Wool atomic64_dec(&pool->pages_nr); 787d30561c5SVitaly Wool return; 788d30561c5SVitaly Wool } 789ca0246bbSVitaly Wool if (test_bit(PAGE_CLAIMED, &page->private)) { 7906098d7e1SVitaly Wool z3fold_page_unlock(zhdr); 7916098d7e1SVitaly Wool return; 7926098d7e1SVitaly Wool } 793d30561c5SVitaly Wool if (test_and_set_bit(NEEDS_COMPACTING, &page->private)) { 794d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 795d30561c5SVitaly Wool return; 796d30561c5SVitaly Wool } 797d30561c5SVitaly Wool if (zhdr->cpu < 0 || !cpu_online(zhdr->cpu)) { 798d30561c5SVitaly Wool spin_lock(&pool->lock); 799d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 800d30561c5SVitaly Wool spin_unlock(&pool->lock); 801d30561c5SVitaly Wool zhdr->cpu = -1; 8025d03a661SVitaly Wool kref_get(&zhdr->refcount); 803d30561c5SVitaly Wool do_compact_page(zhdr, true); 804d30561c5SVitaly Wool return; 805d30561c5SVitaly Wool } 8065d03a661SVitaly Wool kref_get(&zhdr->refcount); 807d30561c5SVitaly Wool queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work); 808d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 8099a001fc1SVitaly Wool } 8109a001fc1SVitaly Wool 8119a001fc1SVitaly Wool /** 8129a001fc1SVitaly Wool * z3fold_reclaim_page() - evicts allocations from a pool page and frees it 8139a001fc1SVitaly Wool * @pool: pool from which a page will attempt to be evicted 814f144c390SMike Rapoport * @retries: number of pages on the LRU list for which eviction will 8159a001fc1SVitaly Wool * be attempted before failing 8169a001fc1SVitaly Wool * 8179a001fc1SVitaly Wool * z3fold reclaim is different from normal system reclaim in that it is done 8189a001fc1SVitaly Wool * from the bottom, up. This is because only the bottom layer, z3fold, has 8199a001fc1SVitaly Wool * information on how the allocations are organized within each z3fold page. 8209a001fc1SVitaly Wool * This has the potential to create interesting locking situations between 8219a001fc1SVitaly Wool * z3fold and the user, however. 8229a001fc1SVitaly Wool * 8239a001fc1SVitaly Wool * To avoid these, this is how z3fold_reclaim_page() should be called: 824f144c390SMike Rapoport * 8259a001fc1SVitaly Wool * The user detects a page should be reclaimed and calls z3fold_reclaim_page(). 8269a001fc1SVitaly Wool * z3fold_reclaim_page() will remove a z3fold page from the pool LRU list and 8279a001fc1SVitaly Wool * call the user-defined eviction handler with the pool and handle as 8289a001fc1SVitaly Wool * arguments. 8299a001fc1SVitaly Wool * 8309a001fc1SVitaly Wool * If the handle can not be evicted, the eviction handler should return 8319a001fc1SVitaly Wool * non-zero. z3fold_reclaim_page() will add the z3fold page back to the 8329a001fc1SVitaly Wool * appropriate list and try the next z3fold page on the LRU up to 8339a001fc1SVitaly Wool * a user defined number of retries. 8349a001fc1SVitaly Wool * 8359a001fc1SVitaly Wool * If the handle is successfully evicted, the eviction handler should 8369a001fc1SVitaly Wool * return 0 _and_ should have called z3fold_free() on the handle. z3fold_free() 8379a001fc1SVitaly Wool * contains logic to delay freeing the page if the page is under reclaim, 8389a001fc1SVitaly Wool * as indicated by the setting of the PG_reclaim flag on the underlying page. 8399a001fc1SVitaly Wool * 8409a001fc1SVitaly Wool * If all buddies in the z3fold page are successfully evicted, then the 8419a001fc1SVitaly Wool * z3fold page can be freed. 8429a001fc1SVitaly Wool * 8439a001fc1SVitaly Wool * Returns: 0 if page is successfully freed, otherwise -EINVAL if there are 8449a001fc1SVitaly Wool * no pages to evict or an eviction handler is not registered, -EAGAIN if 8459a001fc1SVitaly Wool * the retry limit was hit. 8469a001fc1SVitaly Wool */ 8479a001fc1SVitaly Wool static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) 8489a001fc1SVitaly Wool { 849d30561c5SVitaly Wool int i, ret = 0; 850d30561c5SVitaly Wool struct z3fold_header *zhdr = NULL; 851d30561c5SVitaly Wool struct page *page = NULL; 852d30561c5SVitaly Wool struct list_head *pos; 8539a001fc1SVitaly Wool unsigned long first_handle = 0, middle_handle = 0, last_handle = 0; 8549a001fc1SVitaly Wool 8559a001fc1SVitaly Wool spin_lock(&pool->lock); 8562f1e5e4dSVitaly Wool if (!pool->ops || !pool->ops->evict || retries == 0) { 8579a001fc1SVitaly Wool spin_unlock(&pool->lock); 8589a001fc1SVitaly Wool return -EINVAL; 8599a001fc1SVitaly Wool } 8609a001fc1SVitaly Wool for (i = 0; i < retries; i++) { 8612f1e5e4dSVitaly Wool if (list_empty(&pool->lru)) { 8622f1e5e4dSVitaly Wool spin_unlock(&pool->lock); 8632f1e5e4dSVitaly Wool return -EINVAL; 8642f1e5e4dSVitaly Wool } 865d30561c5SVitaly Wool list_for_each_prev(pos, &pool->lru) { 866d30561c5SVitaly Wool page = list_entry(pos, struct page, lru); 867ca0246bbSVitaly Wool 868ca0246bbSVitaly Wool /* this bit could have been set by free, in which case 869ca0246bbSVitaly Wool * we pass over to the next page in the pool. 870ca0246bbSVitaly Wool */ 871ca0246bbSVitaly Wool if (test_and_set_bit(PAGE_CLAIMED, &page->private)) 872ca0246bbSVitaly Wool continue; 8739a001fc1SVitaly Wool 8749a001fc1SVitaly Wool zhdr = page_address(page); 875ca0246bbSVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 876ca0246bbSVitaly Wool break; 877ca0246bbSVitaly Wool 878ca0246bbSVitaly Wool if (!z3fold_page_trylock(zhdr)) { 879ca0246bbSVitaly Wool zhdr = NULL; 880d30561c5SVitaly Wool continue; /* can't evict at this point */ 881ca0246bbSVitaly Wool } 8825a27aa82SVitaly Wool kref_get(&zhdr->refcount); 883d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 884d30561c5SVitaly Wool zhdr->cpu = -1; 8856098d7e1SVitaly Wool break; 886d30561c5SVitaly Wool } 887d30561c5SVitaly Wool 888ca0246bbSVitaly Wool if (!zhdr) 889ca0246bbSVitaly Wool break; 890ca0246bbSVitaly Wool 891d30561c5SVitaly Wool list_del_init(&page->lru); 8922f1e5e4dSVitaly Wool spin_unlock(&pool->lock); 893d30561c5SVitaly Wool 894d30561c5SVitaly Wool if (!test_bit(PAGE_HEADLESS, &page->private)) { 8959a001fc1SVitaly Wool /* 8969a001fc1SVitaly Wool * We need encode the handles before unlocking, since 8979a001fc1SVitaly Wool * we can race with free that will set 8989a001fc1SVitaly Wool * (first|last)_chunks to 0 8999a001fc1SVitaly Wool */ 9009a001fc1SVitaly Wool first_handle = 0; 9019a001fc1SVitaly Wool last_handle = 0; 9029a001fc1SVitaly Wool middle_handle = 0; 9039a001fc1SVitaly Wool if (zhdr->first_chunks) 9049a001fc1SVitaly Wool first_handle = encode_handle(zhdr, FIRST); 9059a001fc1SVitaly Wool if (zhdr->middle_chunks) 9069a001fc1SVitaly Wool middle_handle = encode_handle(zhdr, MIDDLE); 9079a001fc1SVitaly Wool if (zhdr->last_chunks) 9089a001fc1SVitaly Wool last_handle = encode_handle(zhdr, LAST); 909d30561c5SVitaly Wool /* 910d30561c5SVitaly Wool * it's safe to unlock here because we hold a 911d30561c5SVitaly Wool * reference to this page 912d30561c5SVitaly Wool */ 9132f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 9149a001fc1SVitaly Wool } else { 9159a001fc1SVitaly Wool first_handle = encode_handle(zhdr, HEADLESS); 9169a001fc1SVitaly Wool last_handle = middle_handle = 0; 9172f1e5e4dSVitaly Wool } 9189a001fc1SVitaly Wool 9199a001fc1SVitaly Wool /* Issue the eviction callback(s) */ 9209a001fc1SVitaly Wool if (middle_handle) { 9219a001fc1SVitaly Wool ret = pool->ops->evict(pool, middle_handle); 9229a001fc1SVitaly Wool if (ret) 9239a001fc1SVitaly Wool goto next; 9249a001fc1SVitaly Wool } 9259a001fc1SVitaly Wool if (first_handle) { 9269a001fc1SVitaly Wool ret = pool->ops->evict(pool, first_handle); 9279a001fc1SVitaly Wool if (ret) 9289a001fc1SVitaly Wool goto next; 9299a001fc1SVitaly Wool } 9309a001fc1SVitaly Wool if (last_handle) { 9319a001fc1SVitaly Wool ret = pool->ops->evict(pool, last_handle); 9329a001fc1SVitaly Wool if (ret) 9339a001fc1SVitaly Wool goto next; 9349a001fc1SVitaly Wool } 9359a001fc1SVitaly Wool next: 9365a27aa82SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 9375a27aa82SVitaly Wool if (ret == 0) { 9385a27aa82SVitaly Wool free_z3fold_page(page); 939ca0246bbSVitaly Wool atomic64_dec(&pool->pages_nr); 9409a001fc1SVitaly Wool return 0; 9415a27aa82SVitaly Wool } 9426098d7e1SVitaly Wool spin_lock(&pool->lock); 9436098d7e1SVitaly Wool list_add(&page->lru, &pool->lru); 944d5567c9dSVitaly Wool spin_unlock(&pool->lock); 9456098d7e1SVitaly Wool } else { 9466098d7e1SVitaly Wool z3fold_page_lock(zhdr); 947ca0246bbSVitaly Wool clear_bit(PAGE_CLAIMED, &page->private); 9486098d7e1SVitaly Wool if (kref_put(&zhdr->refcount, 9496098d7e1SVitaly Wool release_z3fold_page_locked)) { 9506098d7e1SVitaly Wool atomic64_dec(&pool->pages_nr); 9515a27aa82SVitaly Wool return 0; 9525a27aa82SVitaly Wool } 9535a27aa82SVitaly Wool /* 9546098d7e1SVitaly Wool * if we are here, the page is still not completely 9556098d7e1SVitaly Wool * free. Take the global pool lock then to be able 9566098d7e1SVitaly Wool * to add it back to the lru list 9575a27aa82SVitaly Wool */ 9586098d7e1SVitaly Wool spin_lock(&pool->lock); 9599a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 9606098d7e1SVitaly Wool spin_unlock(&pool->lock); 9616098d7e1SVitaly Wool z3fold_page_unlock(zhdr); 9626098d7e1SVitaly Wool } 9636098d7e1SVitaly Wool 9646098d7e1SVitaly Wool /* We started off locked to we need to lock the pool back */ 9656098d7e1SVitaly Wool spin_lock(&pool->lock); 9669a001fc1SVitaly Wool } 9679a001fc1SVitaly Wool spin_unlock(&pool->lock); 9689a001fc1SVitaly Wool return -EAGAIN; 9699a001fc1SVitaly Wool } 9709a001fc1SVitaly Wool 9719a001fc1SVitaly Wool /** 9729a001fc1SVitaly Wool * z3fold_map() - maps the allocation associated with the given handle 9739a001fc1SVitaly Wool * @pool: pool in which the allocation resides 9749a001fc1SVitaly Wool * @handle: handle associated with the allocation to be mapped 9759a001fc1SVitaly Wool * 9769a001fc1SVitaly Wool * Extracts the buddy number from handle and constructs the pointer to the 9779a001fc1SVitaly Wool * correct starting chunk within the page. 9789a001fc1SVitaly Wool * 9799a001fc1SVitaly Wool * Returns: a pointer to the mapped allocation 9809a001fc1SVitaly Wool */ 9819a001fc1SVitaly Wool static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) 9829a001fc1SVitaly Wool { 9839a001fc1SVitaly Wool struct z3fold_header *zhdr; 9849a001fc1SVitaly Wool struct page *page; 9859a001fc1SVitaly Wool void *addr; 9869a001fc1SVitaly Wool enum buddy buddy; 9879a001fc1SVitaly Wool 9889a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 9899a001fc1SVitaly Wool addr = zhdr; 9909a001fc1SVitaly Wool page = virt_to_page(zhdr); 9919a001fc1SVitaly Wool 9929a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 9939a001fc1SVitaly Wool goto out; 9949a001fc1SVitaly Wool 9952f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 9969a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 9979a001fc1SVitaly Wool switch (buddy) { 9989a001fc1SVitaly Wool case FIRST: 9999a001fc1SVitaly Wool addr += ZHDR_SIZE_ALIGNED; 10009a001fc1SVitaly Wool break; 10019a001fc1SVitaly Wool case MIDDLE: 10029a001fc1SVitaly Wool addr += zhdr->start_middle << CHUNK_SHIFT; 10039a001fc1SVitaly Wool set_bit(MIDDLE_CHUNK_MAPPED, &page->private); 10049a001fc1SVitaly Wool break; 10059a001fc1SVitaly Wool case LAST: 1006ca0246bbSVitaly Wool addr += PAGE_SIZE - (handle_to_chunks(handle) << CHUNK_SHIFT); 10079a001fc1SVitaly Wool break; 10089a001fc1SVitaly Wool default: 10099a001fc1SVitaly Wool pr_err("unknown buddy id %d\n", buddy); 10109a001fc1SVitaly Wool WARN_ON(1); 10119a001fc1SVitaly Wool addr = NULL; 10129a001fc1SVitaly Wool break; 10139a001fc1SVitaly Wool } 10142f1e5e4dSVitaly Wool 10152f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 10169a001fc1SVitaly Wool out: 10179a001fc1SVitaly Wool return addr; 10189a001fc1SVitaly Wool } 10199a001fc1SVitaly Wool 10209a001fc1SVitaly Wool /** 10219a001fc1SVitaly Wool * z3fold_unmap() - unmaps the allocation associated with the given handle 10229a001fc1SVitaly Wool * @pool: pool in which the allocation resides 10239a001fc1SVitaly Wool * @handle: handle associated with the allocation to be unmapped 10249a001fc1SVitaly Wool */ 10259a001fc1SVitaly Wool static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) 10269a001fc1SVitaly Wool { 10279a001fc1SVitaly Wool struct z3fold_header *zhdr; 10289a001fc1SVitaly Wool struct page *page; 10299a001fc1SVitaly Wool enum buddy buddy; 10309a001fc1SVitaly Wool 10319a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 10329a001fc1SVitaly Wool page = virt_to_page(zhdr); 10339a001fc1SVitaly Wool 10342f1e5e4dSVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 10359a001fc1SVitaly Wool return; 10369a001fc1SVitaly Wool 10372f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 10389a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 10399a001fc1SVitaly Wool if (buddy == MIDDLE) 10409a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 10412f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 10429a001fc1SVitaly Wool } 10439a001fc1SVitaly Wool 10449a001fc1SVitaly Wool /** 10459a001fc1SVitaly Wool * z3fold_get_pool_size() - gets the z3fold pool size in pages 10469a001fc1SVitaly Wool * @pool: pool whose size is being queried 10479a001fc1SVitaly Wool * 104812d59ae6SVitaly Wool * Returns: size in pages of the given pool. 10499a001fc1SVitaly Wool */ 10509a001fc1SVitaly Wool static u64 z3fold_get_pool_size(struct z3fold_pool *pool) 10519a001fc1SVitaly Wool { 105212d59ae6SVitaly Wool return atomic64_read(&pool->pages_nr); 10539a001fc1SVitaly Wool } 10549a001fc1SVitaly Wool 10559a001fc1SVitaly Wool /***************** 10569a001fc1SVitaly Wool * zpool 10579a001fc1SVitaly Wool ****************/ 10589a001fc1SVitaly Wool 10599a001fc1SVitaly Wool static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle) 10609a001fc1SVitaly Wool { 10619a001fc1SVitaly Wool if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict) 10629a001fc1SVitaly Wool return pool->zpool_ops->evict(pool->zpool, handle); 10639a001fc1SVitaly Wool else 10649a001fc1SVitaly Wool return -ENOENT; 10659a001fc1SVitaly Wool } 10669a001fc1SVitaly Wool 10679a001fc1SVitaly Wool static const struct z3fold_ops z3fold_zpool_ops = { 10689a001fc1SVitaly Wool .evict = z3fold_zpool_evict 10699a001fc1SVitaly Wool }; 10709a001fc1SVitaly Wool 10719a001fc1SVitaly Wool static void *z3fold_zpool_create(const char *name, gfp_t gfp, 10729a001fc1SVitaly Wool const struct zpool_ops *zpool_ops, 10739a001fc1SVitaly Wool struct zpool *zpool) 10749a001fc1SVitaly Wool { 10759a001fc1SVitaly Wool struct z3fold_pool *pool; 10769a001fc1SVitaly Wool 1077d30561c5SVitaly Wool pool = z3fold_create_pool(name, gfp, 1078d30561c5SVitaly Wool zpool_ops ? &z3fold_zpool_ops : NULL); 10799a001fc1SVitaly Wool if (pool) { 10809a001fc1SVitaly Wool pool->zpool = zpool; 10819a001fc1SVitaly Wool pool->zpool_ops = zpool_ops; 10829a001fc1SVitaly Wool } 10839a001fc1SVitaly Wool return pool; 10849a001fc1SVitaly Wool } 10859a001fc1SVitaly Wool 10869a001fc1SVitaly Wool static void z3fold_zpool_destroy(void *pool) 10879a001fc1SVitaly Wool { 10889a001fc1SVitaly Wool z3fold_destroy_pool(pool); 10899a001fc1SVitaly Wool } 10909a001fc1SVitaly Wool 10919a001fc1SVitaly Wool static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp, 10929a001fc1SVitaly Wool unsigned long *handle) 10939a001fc1SVitaly Wool { 10949a001fc1SVitaly Wool return z3fold_alloc(pool, size, gfp, handle); 10959a001fc1SVitaly Wool } 10969a001fc1SVitaly Wool static void z3fold_zpool_free(void *pool, unsigned long handle) 10979a001fc1SVitaly Wool { 10989a001fc1SVitaly Wool z3fold_free(pool, handle); 10999a001fc1SVitaly Wool } 11009a001fc1SVitaly Wool 11019a001fc1SVitaly Wool static int z3fold_zpool_shrink(void *pool, unsigned int pages, 11029a001fc1SVitaly Wool unsigned int *reclaimed) 11039a001fc1SVitaly Wool { 11049a001fc1SVitaly Wool unsigned int total = 0; 11059a001fc1SVitaly Wool int ret = -EINVAL; 11069a001fc1SVitaly Wool 11079a001fc1SVitaly Wool while (total < pages) { 11089a001fc1SVitaly Wool ret = z3fold_reclaim_page(pool, 8); 11099a001fc1SVitaly Wool if (ret < 0) 11109a001fc1SVitaly Wool break; 11119a001fc1SVitaly Wool total++; 11129a001fc1SVitaly Wool } 11139a001fc1SVitaly Wool 11149a001fc1SVitaly Wool if (reclaimed) 11159a001fc1SVitaly Wool *reclaimed = total; 11169a001fc1SVitaly Wool 11179a001fc1SVitaly Wool return ret; 11189a001fc1SVitaly Wool } 11199a001fc1SVitaly Wool 11209a001fc1SVitaly Wool static void *z3fold_zpool_map(void *pool, unsigned long handle, 11219a001fc1SVitaly Wool enum zpool_mapmode mm) 11229a001fc1SVitaly Wool { 11239a001fc1SVitaly Wool return z3fold_map(pool, handle); 11249a001fc1SVitaly Wool } 11259a001fc1SVitaly Wool static void z3fold_zpool_unmap(void *pool, unsigned long handle) 11269a001fc1SVitaly Wool { 11279a001fc1SVitaly Wool z3fold_unmap(pool, handle); 11289a001fc1SVitaly Wool } 11299a001fc1SVitaly Wool 11309a001fc1SVitaly Wool static u64 z3fold_zpool_total_size(void *pool) 11319a001fc1SVitaly Wool { 11329a001fc1SVitaly Wool return z3fold_get_pool_size(pool) * PAGE_SIZE; 11339a001fc1SVitaly Wool } 11349a001fc1SVitaly Wool 11359a001fc1SVitaly Wool static struct zpool_driver z3fold_zpool_driver = { 11369a001fc1SVitaly Wool .type = "z3fold", 11379a001fc1SVitaly Wool .owner = THIS_MODULE, 11389a001fc1SVitaly Wool .create = z3fold_zpool_create, 11399a001fc1SVitaly Wool .destroy = z3fold_zpool_destroy, 11409a001fc1SVitaly Wool .malloc = z3fold_zpool_malloc, 11419a001fc1SVitaly Wool .free = z3fold_zpool_free, 11429a001fc1SVitaly Wool .shrink = z3fold_zpool_shrink, 11439a001fc1SVitaly Wool .map = z3fold_zpool_map, 11449a001fc1SVitaly Wool .unmap = z3fold_zpool_unmap, 11459a001fc1SVitaly Wool .total_size = z3fold_zpool_total_size, 11469a001fc1SVitaly Wool }; 11479a001fc1SVitaly Wool 11489a001fc1SVitaly Wool MODULE_ALIAS("zpool-z3fold"); 11499a001fc1SVitaly Wool 11509a001fc1SVitaly Wool static int __init init_z3fold(void) 11519a001fc1SVitaly Wool { 1152ede93213SVitaly Wool /* Make sure the z3fold header is not larger than the page size */ 1153ede93213SVitaly Wool BUILD_BUG_ON(ZHDR_SIZE_ALIGNED > PAGE_SIZE); 11549a001fc1SVitaly Wool zpool_register_driver(&z3fold_zpool_driver); 11559a001fc1SVitaly Wool 11569a001fc1SVitaly Wool return 0; 11579a001fc1SVitaly Wool } 11589a001fc1SVitaly Wool 11599a001fc1SVitaly Wool static void __exit exit_z3fold(void) 11609a001fc1SVitaly Wool { 11619a001fc1SVitaly Wool zpool_unregister_driver(&z3fold_zpool_driver); 11629a001fc1SVitaly Wool } 11639a001fc1SVitaly Wool 11649a001fc1SVitaly Wool module_init(init_z3fold); 11659a001fc1SVitaly Wool module_exit(exit_z3fold); 11669a001fc1SVitaly Wool 11679a001fc1SVitaly Wool MODULE_LICENSE("GPL"); 11689a001fc1SVitaly Wool MODULE_AUTHOR("Vitaly Wool <vitalywool@gmail.com>"); 11699a001fc1SVitaly Wool MODULE_DESCRIPTION("3-Fold Allocator for Compressed Pages"); 1170