1*09c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 29a001fc1SVitaly Wool /* 39a001fc1SVitaly Wool * z3fold.c 49a001fc1SVitaly Wool * 59a001fc1SVitaly Wool * Author: Vitaly Wool <vitaly.wool@konsulko.com> 69a001fc1SVitaly Wool * Copyright (C) 2016, Sony Mobile Communications Inc. 79a001fc1SVitaly Wool * 89a001fc1SVitaly Wool * This implementation is based on zbud written by Seth Jennings. 99a001fc1SVitaly Wool * 109a001fc1SVitaly Wool * z3fold is an special purpose allocator for storing compressed pages. It 119a001fc1SVitaly Wool * can store up to three compressed pages per page which improves the 129a001fc1SVitaly Wool * compression ratio of zbud while retaining its main concepts (e. g. always 139a001fc1SVitaly Wool * storing an integral number of objects per page) and simplicity. 149a001fc1SVitaly Wool * It still has simple and deterministic reclaim properties that make it 159a001fc1SVitaly Wool * preferable to a higher density approach (with no requirement on integral 169a001fc1SVitaly Wool * number of object per page) when reclaim is used. 179a001fc1SVitaly Wool * 189a001fc1SVitaly Wool * As in zbud, pages are divided into "chunks". The size of the chunks is 199a001fc1SVitaly Wool * fixed at compile time and is determined by NCHUNKS_ORDER below. 209a001fc1SVitaly Wool * 219a001fc1SVitaly Wool * z3fold doesn't export any API and is meant to be used via zpool API. 229a001fc1SVitaly Wool */ 239a001fc1SVitaly Wool 249a001fc1SVitaly Wool #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 259a001fc1SVitaly Wool 269a001fc1SVitaly Wool #include <linux/atomic.h> 27d30561c5SVitaly Wool #include <linux/sched.h> 281f862989SVitaly Wool #include <linux/cpumask.h> 291f862989SVitaly Wool #include <linux/dcache.h> 309a001fc1SVitaly Wool #include <linux/list.h> 319a001fc1SVitaly Wool #include <linux/mm.h> 329a001fc1SVitaly Wool #include <linux/module.h> 331f862989SVitaly Wool #include <linux/page-flags.h> 341f862989SVitaly Wool #include <linux/migrate.h> 351f862989SVitaly Wool #include <linux/node.h> 361f862989SVitaly Wool #include <linux/compaction.h> 37d30561c5SVitaly Wool #include <linux/percpu.h> 381f862989SVitaly Wool #include <linux/mount.h> 391f862989SVitaly Wool #include <linux/fs.h> 409a001fc1SVitaly Wool #include <linux/preempt.h> 41d30561c5SVitaly Wool #include <linux/workqueue.h> 429a001fc1SVitaly Wool #include <linux/slab.h> 439a001fc1SVitaly Wool #include <linux/spinlock.h> 449a001fc1SVitaly Wool #include <linux/zpool.h> 459a001fc1SVitaly Wool 469a001fc1SVitaly Wool /* 479a001fc1SVitaly Wool * NCHUNKS_ORDER determines the internal allocation granularity, effectively 489a001fc1SVitaly Wool * adjusting internal fragmentation. It also determines the number of 499a001fc1SVitaly Wool * freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the 50ede93213SVitaly Wool * allocation granularity will be in chunks of size PAGE_SIZE/64. Some chunks 51ede93213SVitaly Wool * in the beginning of an allocated page are occupied by z3fold header, so 52ede93213SVitaly Wool * NCHUNKS will be calculated to 63 (or 62 in case CONFIG_DEBUG_SPINLOCK=y), 53ede93213SVitaly Wool * which shows the max number of free chunks in z3fold page, also there will 54ede93213SVitaly Wool * be 63, or 62, respectively, freelists per pool. 559a001fc1SVitaly Wool */ 569a001fc1SVitaly Wool #define NCHUNKS_ORDER 6 579a001fc1SVitaly Wool 589a001fc1SVitaly Wool #define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER) 599a001fc1SVitaly Wool #define CHUNK_SIZE (1 << CHUNK_SHIFT) 60ede93213SVitaly Wool #define ZHDR_SIZE_ALIGNED round_up(sizeof(struct z3fold_header), CHUNK_SIZE) 61ede93213SVitaly Wool #define ZHDR_CHUNKS (ZHDR_SIZE_ALIGNED >> CHUNK_SHIFT) 62ede93213SVitaly Wool #define TOTAL_CHUNKS (PAGE_SIZE >> CHUNK_SHIFT) 639a001fc1SVitaly Wool #define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT) 649a001fc1SVitaly Wool 65f201ebd8Szhong jiang #define BUDDY_MASK (0x3) 66ca0246bbSVitaly Wool #define BUDDY_SHIFT 2 677c2b8baaSVitaly Wool #define SLOTS_ALIGN (0x40) 687c2b8baaSVitaly Wool 697c2b8baaSVitaly Wool /***************** 707c2b8baaSVitaly Wool * Structures 717c2b8baaSVitaly Wool *****************/ 727c2b8baaSVitaly Wool struct z3fold_pool; 737c2b8baaSVitaly Wool struct z3fold_ops { 747c2b8baaSVitaly Wool int (*evict)(struct z3fold_pool *pool, unsigned long handle); 757c2b8baaSVitaly Wool }; 767c2b8baaSVitaly Wool 777c2b8baaSVitaly Wool enum buddy { 787c2b8baaSVitaly Wool HEADLESS = 0, 797c2b8baaSVitaly Wool FIRST, 807c2b8baaSVitaly Wool MIDDLE, 817c2b8baaSVitaly Wool LAST, 827c2b8baaSVitaly Wool BUDDIES_MAX = LAST 837c2b8baaSVitaly Wool }; 847c2b8baaSVitaly Wool 857c2b8baaSVitaly Wool struct z3fold_buddy_slots { 867c2b8baaSVitaly Wool /* 877c2b8baaSVitaly Wool * we are using BUDDY_MASK in handle_to_buddy etc. so there should 887c2b8baaSVitaly Wool * be enough slots to hold all possible variants 897c2b8baaSVitaly Wool */ 907c2b8baaSVitaly Wool unsigned long slot[BUDDY_MASK + 1]; 917c2b8baaSVitaly Wool unsigned long pool; /* back link + flags */ 927c2b8baaSVitaly Wool }; 937c2b8baaSVitaly Wool #define HANDLE_FLAG_MASK (0x03) 947c2b8baaSVitaly Wool 957c2b8baaSVitaly Wool /* 967c2b8baaSVitaly Wool * struct z3fold_header - z3fold page metadata occupying first chunks of each 977c2b8baaSVitaly Wool * z3fold page, except for HEADLESS pages 987c2b8baaSVitaly Wool * @buddy: links the z3fold page into the relevant list in the 997c2b8baaSVitaly Wool * pool 1007c2b8baaSVitaly Wool * @page_lock: per-page lock 1017c2b8baaSVitaly Wool * @refcount: reference count for the z3fold page 1027c2b8baaSVitaly Wool * @work: work_struct for page layout optimization 1037c2b8baaSVitaly Wool * @slots: pointer to the structure holding buddy slots 1047c2b8baaSVitaly Wool * @cpu: CPU which this page "belongs" to 1057c2b8baaSVitaly Wool * @first_chunks: the size of the first buddy in chunks, 0 if free 1067c2b8baaSVitaly Wool * @middle_chunks: the size of the middle buddy in chunks, 0 if free 1077c2b8baaSVitaly Wool * @last_chunks: the size of the last buddy in chunks, 0 if free 1087c2b8baaSVitaly Wool * @first_num: the starting number (for the first handle) 1091f862989SVitaly Wool * @mapped_count: the number of objects currently mapped 1107c2b8baaSVitaly Wool */ 1117c2b8baaSVitaly Wool struct z3fold_header { 1127c2b8baaSVitaly Wool struct list_head buddy; 1137c2b8baaSVitaly Wool spinlock_t page_lock; 1147c2b8baaSVitaly Wool struct kref refcount; 1157c2b8baaSVitaly Wool struct work_struct work; 1167c2b8baaSVitaly Wool struct z3fold_buddy_slots *slots; 1177c2b8baaSVitaly Wool short cpu; 1187c2b8baaSVitaly Wool unsigned short first_chunks; 1197c2b8baaSVitaly Wool unsigned short middle_chunks; 1207c2b8baaSVitaly Wool unsigned short last_chunks; 1217c2b8baaSVitaly Wool unsigned short start_middle; 1227c2b8baaSVitaly Wool unsigned short first_num:2; 1231f862989SVitaly Wool unsigned short mapped_count:2; 1247c2b8baaSVitaly Wool }; 1259a001fc1SVitaly Wool 1269a001fc1SVitaly Wool /** 1279a001fc1SVitaly Wool * struct z3fold_pool - stores metadata for each z3fold pool 128d30561c5SVitaly Wool * @name: pool name 129d30561c5SVitaly Wool * @lock: protects pool unbuddied/lru lists 130d30561c5SVitaly Wool * @stale_lock: protects pool stale page list 131d30561c5SVitaly Wool * @unbuddied: per-cpu array of lists tracking z3fold pages that contain 2- 132d30561c5SVitaly Wool * buddies; the list each z3fold page is added to depends on 133d30561c5SVitaly Wool * the size of its free region. 1349a001fc1SVitaly Wool * @lru: list tracking the z3fold pages in LRU order by most recently 1359a001fc1SVitaly Wool * added buddy. 136d30561c5SVitaly Wool * @stale: list of pages marked for freeing 1379a001fc1SVitaly Wool * @pages_nr: number of z3fold pages in the pool. 1387c2b8baaSVitaly Wool * @c_handle: cache for z3fold_buddy_slots allocation 1399a001fc1SVitaly Wool * @ops: pointer to a structure of user defined operations specified at 1409a001fc1SVitaly Wool * pool creation time. 141d30561c5SVitaly Wool * @compact_wq: workqueue for page layout background optimization 142d30561c5SVitaly Wool * @release_wq: workqueue for safe page release 143d30561c5SVitaly Wool * @work: work_struct for safe page release 1441f862989SVitaly Wool * @inode: inode for z3fold pseudo filesystem 1459a001fc1SVitaly Wool * 1469a001fc1SVitaly Wool * This structure is allocated at pool creation time and maintains metadata 1479a001fc1SVitaly Wool * pertaining to a particular z3fold pool. 1489a001fc1SVitaly Wool */ 1499a001fc1SVitaly Wool struct z3fold_pool { 150d30561c5SVitaly Wool const char *name; 1519a001fc1SVitaly Wool spinlock_t lock; 152d30561c5SVitaly Wool spinlock_t stale_lock; 153d30561c5SVitaly Wool struct list_head *unbuddied; 1549a001fc1SVitaly Wool struct list_head lru; 155d30561c5SVitaly Wool struct list_head stale; 15612d59ae6SVitaly Wool atomic64_t pages_nr; 1577c2b8baaSVitaly Wool struct kmem_cache *c_handle; 1589a001fc1SVitaly Wool const struct z3fold_ops *ops; 1599a001fc1SVitaly Wool struct zpool *zpool; 1609a001fc1SVitaly Wool const struct zpool_ops *zpool_ops; 161d30561c5SVitaly Wool struct workqueue_struct *compact_wq; 162d30561c5SVitaly Wool struct workqueue_struct *release_wq; 163d30561c5SVitaly Wool struct work_struct work; 1641f862989SVitaly Wool struct inode *inode; 1659a001fc1SVitaly Wool }; 1669a001fc1SVitaly Wool 1679a001fc1SVitaly Wool /* 1689a001fc1SVitaly Wool * Internal z3fold page flags 1699a001fc1SVitaly Wool */ 1709a001fc1SVitaly Wool enum z3fold_page_flags { 1715a27aa82SVitaly Wool PAGE_HEADLESS = 0, 1729a001fc1SVitaly Wool MIDDLE_CHUNK_MAPPED, 173d30561c5SVitaly Wool NEEDS_COMPACTING, 1746098d7e1SVitaly Wool PAGE_STALE, 175ca0246bbSVitaly Wool PAGE_CLAIMED, /* by either reclaim or free */ 1769a001fc1SVitaly Wool }; 1779a001fc1SVitaly Wool 1789a001fc1SVitaly Wool /***************** 1799a001fc1SVitaly Wool * Helpers 1809a001fc1SVitaly Wool *****************/ 1819a001fc1SVitaly Wool 1829a001fc1SVitaly Wool /* Converts an allocation size in bytes to size in z3fold chunks */ 1839a001fc1SVitaly Wool static int size_to_chunks(size_t size) 1849a001fc1SVitaly Wool { 1859a001fc1SVitaly Wool return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; 1869a001fc1SVitaly Wool } 1879a001fc1SVitaly Wool 1889a001fc1SVitaly Wool #define for_each_unbuddied_list(_iter, _begin) \ 1899a001fc1SVitaly Wool for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++) 1909a001fc1SVitaly Wool 191d30561c5SVitaly Wool static void compact_page_work(struct work_struct *w); 192d30561c5SVitaly Wool 1937c2b8baaSVitaly Wool static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool) 1947c2b8baaSVitaly Wool { 1957c2b8baaSVitaly Wool struct z3fold_buddy_slots *slots = kmem_cache_alloc(pool->c_handle, 1967c2b8baaSVitaly Wool GFP_KERNEL); 1977c2b8baaSVitaly Wool 1987c2b8baaSVitaly Wool if (slots) { 1997c2b8baaSVitaly Wool memset(slots->slot, 0, sizeof(slots->slot)); 2007c2b8baaSVitaly Wool slots->pool = (unsigned long)pool; 2017c2b8baaSVitaly Wool } 2027c2b8baaSVitaly Wool 2037c2b8baaSVitaly Wool return slots; 2047c2b8baaSVitaly Wool } 2057c2b8baaSVitaly Wool 2067c2b8baaSVitaly Wool static inline struct z3fold_pool *slots_to_pool(struct z3fold_buddy_slots *s) 2077c2b8baaSVitaly Wool { 2087c2b8baaSVitaly Wool return (struct z3fold_pool *)(s->pool & ~HANDLE_FLAG_MASK); 2097c2b8baaSVitaly Wool } 2107c2b8baaSVitaly Wool 2117c2b8baaSVitaly Wool static inline struct z3fold_buddy_slots *handle_to_slots(unsigned long handle) 2127c2b8baaSVitaly Wool { 2137c2b8baaSVitaly Wool return (struct z3fold_buddy_slots *)(handle & ~(SLOTS_ALIGN - 1)); 2147c2b8baaSVitaly Wool } 2157c2b8baaSVitaly Wool 2167c2b8baaSVitaly Wool static inline void free_handle(unsigned long handle) 2177c2b8baaSVitaly Wool { 2187c2b8baaSVitaly Wool struct z3fold_buddy_slots *slots; 2197c2b8baaSVitaly Wool int i; 2207c2b8baaSVitaly Wool bool is_free; 2217c2b8baaSVitaly Wool 2227c2b8baaSVitaly Wool if (handle & (1 << PAGE_HEADLESS)) 2237c2b8baaSVitaly Wool return; 2247c2b8baaSVitaly Wool 2257c2b8baaSVitaly Wool WARN_ON(*(unsigned long *)handle == 0); 2267c2b8baaSVitaly Wool *(unsigned long *)handle = 0; 2277c2b8baaSVitaly Wool slots = handle_to_slots(handle); 2287c2b8baaSVitaly Wool is_free = true; 2297c2b8baaSVitaly Wool for (i = 0; i <= BUDDY_MASK; i++) { 2307c2b8baaSVitaly Wool if (slots->slot[i]) { 2317c2b8baaSVitaly Wool is_free = false; 2327c2b8baaSVitaly Wool break; 2337c2b8baaSVitaly Wool } 2347c2b8baaSVitaly Wool } 2357c2b8baaSVitaly Wool 2367c2b8baaSVitaly Wool if (is_free) { 2377c2b8baaSVitaly Wool struct z3fold_pool *pool = slots_to_pool(slots); 2387c2b8baaSVitaly Wool 2397c2b8baaSVitaly Wool kmem_cache_free(pool->c_handle, slots); 2407c2b8baaSVitaly Wool } 2417c2b8baaSVitaly Wool } 2427c2b8baaSVitaly Wool 2431f862989SVitaly Wool static struct dentry *z3fold_do_mount(struct file_system_type *fs_type, 2441f862989SVitaly Wool int flags, const char *dev_name, void *data) 2451f862989SVitaly Wool { 2461f862989SVitaly Wool static const struct dentry_operations ops = { 2471f862989SVitaly Wool .d_dname = simple_dname, 2481f862989SVitaly Wool }; 2491f862989SVitaly Wool 2501f862989SVitaly Wool return mount_pseudo(fs_type, "z3fold:", NULL, &ops, 0x33); 2511f862989SVitaly Wool } 2521f862989SVitaly Wool 2531f862989SVitaly Wool static struct file_system_type z3fold_fs = { 2541f862989SVitaly Wool .name = "z3fold", 2551f862989SVitaly Wool .mount = z3fold_do_mount, 2561f862989SVitaly Wool .kill_sb = kill_anon_super, 2571f862989SVitaly Wool }; 2581f862989SVitaly Wool 2591f862989SVitaly Wool static struct vfsmount *z3fold_mnt; 2601f862989SVitaly Wool static int z3fold_mount(void) 2611f862989SVitaly Wool { 2621f862989SVitaly Wool int ret = 0; 2631f862989SVitaly Wool 2641f862989SVitaly Wool z3fold_mnt = kern_mount(&z3fold_fs); 2651f862989SVitaly Wool if (IS_ERR(z3fold_mnt)) 2661f862989SVitaly Wool ret = PTR_ERR(z3fold_mnt); 2671f862989SVitaly Wool 2681f862989SVitaly Wool return ret; 2691f862989SVitaly Wool } 2701f862989SVitaly Wool 2711f862989SVitaly Wool static void z3fold_unmount(void) 2721f862989SVitaly Wool { 2731f862989SVitaly Wool kern_unmount(z3fold_mnt); 2741f862989SVitaly Wool } 2751f862989SVitaly Wool 2761f862989SVitaly Wool static const struct address_space_operations z3fold_aops; 2771f862989SVitaly Wool static int z3fold_register_migration(struct z3fold_pool *pool) 2781f862989SVitaly Wool { 2791f862989SVitaly Wool pool->inode = alloc_anon_inode(z3fold_mnt->mnt_sb); 2801f862989SVitaly Wool if (IS_ERR(pool->inode)) { 2811f862989SVitaly Wool pool->inode = NULL; 2821f862989SVitaly Wool return 1; 2831f862989SVitaly Wool } 2841f862989SVitaly Wool 2851f862989SVitaly Wool pool->inode->i_mapping->private_data = pool; 2861f862989SVitaly Wool pool->inode->i_mapping->a_ops = &z3fold_aops; 2871f862989SVitaly Wool return 0; 2881f862989SVitaly Wool } 2891f862989SVitaly Wool 2901f862989SVitaly Wool static void z3fold_unregister_migration(struct z3fold_pool *pool) 2911f862989SVitaly Wool { 2921f862989SVitaly Wool if (pool->inode) 2931f862989SVitaly Wool iput(pool->inode); 2941f862989SVitaly Wool } 2951f862989SVitaly Wool 2969a001fc1SVitaly Wool /* Initializes the z3fold header of a newly allocated z3fold page */ 297d30561c5SVitaly Wool static struct z3fold_header *init_z3fold_page(struct page *page, 298d30561c5SVitaly Wool struct z3fold_pool *pool) 2999a001fc1SVitaly Wool { 3009a001fc1SVitaly Wool struct z3fold_header *zhdr = page_address(page); 3017c2b8baaSVitaly Wool struct z3fold_buddy_slots *slots = alloc_slots(pool); 3027c2b8baaSVitaly Wool 3037c2b8baaSVitaly Wool if (!slots) 3047c2b8baaSVitaly Wool return NULL; 3059a001fc1SVitaly Wool 3069a001fc1SVitaly Wool INIT_LIST_HEAD(&page->lru); 3079a001fc1SVitaly Wool clear_bit(PAGE_HEADLESS, &page->private); 3089a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 309d30561c5SVitaly Wool clear_bit(NEEDS_COMPACTING, &page->private); 310d30561c5SVitaly Wool clear_bit(PAGE_STALE, &page->private); 311ca0246bbSVitaly Wool clear_bit(PAGE_CLAIMED, &page->private); 3129a001fc1SVitaly Wool 3132f1e5e4dSVitaly Wool spin_lock_init(&zhdr->page_lock); 3145a27aa82SVitaly Wool kref_init(&zhdr->refcount); 3159a001fc1SVitaly Wool zhdr->first_chunks = 0; 3169a001fc1SVitaly Wool zhdr->middle_chunks = 0; 3179a001fc1SVitaly Wool zhdr->last_chunks = 0; 3189a001fc1SVitaly Wool zhdr->first_num = 0; 3199a001fc1SVitaly Wool zhdr->start_middle = 0; 320d30561c5SVitaly Wool zhdr->cpu = -1; 3217c2b8baaSVitaly Wool zhdr->slots = slots; 3229a001fc1SVitaly Wool INIT_LIST_HEAD(&zhdr->buddy); 323d30561c5SVitaly Wool INIT_WORK(&zhdr->work, compact_page_work); 3249a001fc1SVitaly Wool return zhdr; 3259a001fc1SVitaly Wool } 3269a001fc1SVitaly Wool 3279a001fc1SVitaly Wool /* Resets the struct page fields and frees the page */ 3281f862989SVitaly Wool static void free_z3fold_page(struct page *page, bool headless) 3299a001fc1SVitaly Wool { 3301f862989SVitaly Wool if (!headless) { 3311f862989SVitaly Wool lock_page(page); 3321f862989SVitaly Wool __ClearPageMovable(page); 3331f862989SVitaly Wool unlock_page(page); 3341f862989SVitaly Wool } 3351f862989SVitaly Wool ClearPagePrivate(page); 3365a27aa82SVitaly Wool __free_page(page); 3375a27aa82SVitaly Wool } 3385a27aa82SVitaly Wool 3392f1e5e4dSVitaly Wool /* Lock a z3fold page */ 3402f1e5e4dSVitaly Wool static inline void z3fold_page_lock(struct z3fold_header *zhdr) 3412f1e5e4dSVitaly Wool { 3422f1e5e4dSVitaly Wool spin_lock(&zhdr->page_lock); 3432f1e5e4dSVitaly Wool } 3442f1e5e4dSVitaly Wool 34576e32a2aSVitaly Wool /* Try to lock a z3fold page */ 34676e32a2aSVitaly Wool static inline int z3fold_page_trylock(struct z3fold_header *zhdr) 34776e32a2aSVitaly Wool { 34876e32a2aSVitaly Wool return spin_trylock(&zhdr->page_lock); 34976e32a2aSVitaly Wool } 35076e32a2aSVitaly Wool 3512f1e5e4dSVitaly Wool /* Unlock a z3fold page */ 3522f1e5e4dSVitaly Wool static inline void z3fold_page_unlock(struct z3fold_header *zhdr) 3532f1e5e4dSVitaly Wool { 3542f1e5e4dSVitaly Wool spin_unlock(&zhdr->page_lock); 3552f1e5e4dSVitaly Wool } 3562f1e5e4dSVitaly Wool 3577c2b8baaSVitaly Wool /* Helper function to build the index */ 3587c2b8baaSVitaly Wool static inline int __idx(struct z3fold_header *zhdr, enum buddy bud) 3597c2b8baaSVitaly Wool { 3607c2b8baaSVitaly Wool return (bud + zhdr->first_num) & BUDDY_MASK; 3617c2b8baaSVitaly Wool } 3627c2b8baaSVitaly Wool 3639a001fc1SVitaly Wool /* 3649a001fc1SVitaly Wool * Encodes the handle of a particular buddy within a z3fold page 3659a001fc1SVitaly Wool * Pool lock should be held as this function accesses first_num 3669a001fc1SVitaly Wool */ 3679a001fc1SVitaly Wool static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) 3689a001fc1SVitaly Wool { 3697c2b8baaSVitaly Wool struct z3fold_buddy_slots *slots; 3707c2b8baaSVitaly Wool unsigned long h = (unsigned long)zhdr; 3717c2b8baaSVitaly Wool int idx = 0; 3729a001fc1SVitaly Wool 3737c2b8baaSVitaly Wool /* 3747c2b8baaSVitaly Wool * For a headless page, its handle is its pointer with the extra 3757c2b8baaSVitaly Wool * PAGE_HEADLESS bit set 3767c2b8baaSVitaly Wool */ 3777c2b8baaSVitaly Wool if (bud == HEADLESS) 3787c2b8baaSVitaly Wool return h | (1 << PAGE_HEADLESS); 3797c2b8baaSVitaly Wool 3807c2b8baaSVitaly Wool /* otherwise, return pointer to encoded handle */ 3817c2b8baaSVitaly Wool idx = __idx(zhdr, bud); 3827c2b8baaSVitaly Wool h += idx; 383ca0246bbSVitaly Wool if (bud == LAST) 3847c2b8baaSVitaly Wool h |= (zhdr->last_chunks << BUDDY_SHIFT); 3857c2b8baaSVitaly Wool 3867c2b8baaSVitaly Wool slots = zhdr->slots; 3877c2b8baaSVitaly Wool slots->slot[idx] = h; 3887c2b8baaSVitaly Wool return (unsigned long)&slots->slot[idx]; 3899a001fc1SVitaly Wool } 3909a001fc1SVitaly Wool 3919a001fc1SVitaly Wool /* Returns the z3fold page where a given handle is stored */ 3921f862989SVitaly Wool static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h) 3939a001fc1SVitaly Wool { 3941f862989SVitaly Wool unsigned long addr = h; 3957c2b8baaSVitaly Wool 3967c2b8baaSVitaly Wool if (!(addr & (1 << PAGE_HEADLESS))) 3971f862989SVitaly Wool addr = *(unsigned long *)h; 3987c2b8baaSVitaly Wool 3997c2b8baaSVitaly Wool return (struct z3fold_header *)(addr & PAGE_MASK); 4009a001fc1SVitaly Wool } 4019a001fc1SVitaly Wool 402ca0246bbSVitaly Wool /* only for LAST bud, returns zero otherwise */ 403ca0246bbSVitaly Wool static unsigned short handle_to_chunks(unsigned long handle) 404ca0246bbSVitaly Wool { 4057c2b8baaSVitaly Wool unsigned long addr = *(unsigned long *)handle; 4067c2b8baaSVitaly Wool 4077c2b8baaSVitaly Wool return (addr & ~PAGE_MASK) >> BUDDY_SHIFT; 408ca0246bbSVitaly Wool } 409ca0246bbSVitaly Wool 410f201ebd8Szhong jiang /* 411f201ebd8Szhong jiang * (handle & BUDDY_MASK) < zhdr->first_num is possible in encode_handle 412f201ebd8Szhong jiang * but that doesn't matter. because the masking will result in the 413f201ebd8Szhong jiang * correct buddy number. 414f201ebd8Szhong jiang */ 4159a001fc1SVitaly Wool static enum buddy handle_to_buddy(unsigned long handle) 4169a001fc1SVitaly Wool { 4177c2b8baaSVitaly Wool struct z3fold_header *zhdr; 4187c2b8baaSVitaly Wool unsigned long addr; 4197c2b8baaSVitaly Wool 4207c2b8baaSVitaly Wool WARN_ON(handle & (1 << PAGE_HEADLESS)); 4217c2b8baaSVitaly Wool addr = *(unsigned long *)handle; 4227c2b8baaSVitaly Wool zhdr = (struct z3fold_header *)(addr & PAGE_MASK); 4237c2b8baaSVitaly Wool return (addr - zhdr->first_num) & BUDDY_MASK; 4249a001fc1SVitaly Wool } 4259a001fc1SVitaly Wool 4269050cce1SVitaly Wool static inline struct z3fold_pool *zhdr_to_pool(struct z3fold_header *zhdr) 4279050cce1SVitaly Wool { 4287c2b8baaSVitaly Wool return slots_to_pool(zhdr->slots); 4299050cce1SVitaly Wool } 4309050cce1SVitaly Wool 431d30561c5SVitaly Wool static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) 432d30561c5SVitaly Wool { 433d30561c5SVitaly Wool struct page *page = virt_to_page(zhdr); 4349050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 435d30561c5SVitaly Wool 436d30561c5SVitaly Wool WARN_ON(!list_empty(&zhdr->buddy)); 437d30561c5SVitaly Wool set_bit(PAGE_STALE, &page->private); 43835529357SVitaly Wool clear_bit(NEEDS_COMPACTING, &page->private); 439d30561c5SVitaly Wool spin_lock(&pool->lock); 440d30561c5SVitaly Wool if (!list_empty(&page->lru)) 4411f862989SVitaly Wool list_del_init(&page->lru); 442d30561c5SVitaly Wool spin_unlock(&pool->lock); 443d30561c5SVitaly Wool if (locked) 444d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 445d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 446d30561c5SVitaly Wool list_add(&zhdr->buddy, &pool->stale); 447d30561c5SVitaly Wool queue_work(pool->release_wq, &pool->work); 448d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 449d30561c5SVitaly Wool } 450d30561c5SVitaly Wool 451d30561c5SVitaly Wool static void __attribute__((__unused__)) 452d30561c5SVitaly Wool release_z3fold_page(struct kref *ref) 453d30561c5SVitaly Wool { 454d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 455d30561c5SVitaly Wool refcount); 456d30561c5SVitaly Wool __release_z3fold_page(zhdr, false); 457d30561c5SVitaly Wool } 458d30561c5SVitaly Wool 459d30561c5SVitaly Wool static void release_z3fold_page_locked(struct kref *ref) 460d30561c5SVitaly Wool { 461d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 462d30561c5SVitaly Wool refcount); 463d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 464d30561c5SVitaly Wool __release_z3fold_page(zhdr, true); 465d30561c5SVitaly Wool } 466d30561c5SVitaly Wool 467d30561c5SVitaly Wool static void release_z3fold_page_locked_list(struct kref *ref) 468d30561c5SVitaly Wool { 469d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(ref, struct z3fold_header, 470d30561c5SVitaly Wool refcount); 4719050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 4729050cce1SVitaly Wool spin_lock(&pool->lock); 473d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 4749050cce1SVitaly Wool spin_unlock(&pool->lock); 475d30561c5SVitaly Wool 476d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 477d30561c5SVitaly Wool __release_z3fold_page(zhdr, true); 478d30561c5SVitaly Wool } 479d30561c5SVitaly Wool 480d30561c5SVitaly Wool static void free_pages_work(struct work_struct *w) 481d30561c5SVitaly Wool { 482d30561c5SVitaly Wool struct z3fold_pool *pool = container_of(w, struct z3fold_pool, work); 483d30561c5SVitaly Wool 484d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 485d30561c5SVitaly Wool while (!list_empty(&pool->stale)) { 486d30561c5SVitaly Wool struct z3fold_header *zhdr = list_first_entry(&pool->stale, 487d30561c5SVitaly Wool struct z3fold_header, buddy); 488d30561c5SVitaly Wool struct page *page = virt_to_page(zhdr); 489d30561c5SVitaly Wool 490d30561c5SVitaly Wool list_del(&zhdr->buddy); 491d30561c5SVitaly Wool if (WARN_ON(!test_bit(PAGE_STALE, &page->private))) 492d30561c5SVitaly Wool continue; 493d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 494d30561c5SVitaly Wool cancel_work_sync(&zhdr->work); 4951f862989SVitaly Wool free_z3fold_page(page, false); 496d30561c5SVitaly Wool cond_resched(); 497d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 498d30561c5SVitaly Wool } 499d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 500d30561c5SVitaly Wool } 501d30561c5SVitaly Wool 5029a001fc1SVitaly Wool /* 5039a001fc1SVitaly Wool * Returns the number of free chunks in a z3fold page. 5049a001fc1SVitaly Wool * NB: can't be used with HEADLESS pages. 5059a001fc1SVitaly Wool */ 5069a001fc1SVitaly Wool static int num_free_chunks(struct z3fold_header *zhdr) 5079a001fc1SVitaly Wool { 5089a001fc1SVitaly Wool int nfree; 5099a001fc1SVitaly Wool /* 5109a001fc1SVitaly Wool * If there is a middle object, pick up the bigger free space 5119a001fc1SVitaly Wool * either before or after it. Otherwise just subtract the number 5129a001fc1SVitaly Wool * of chunks occupied by the first and the last objects. 5139a001fc1SVitaly Wool */ 5149a001fc1SVitaly Wool if (zhdr->middle_chunks != 0) { 5159a001fc1SVitaly Wool int nfree_before = zhdr->first_chunks ? 516ede93213SVitaly Wool 0 : zhdr->start_middle - ZHDR_CHUNKS; 5179a001fc1SVitaly Wool int nfree_after = zhdr->last_chunks ? 518ede93213SVitaly Wool 0 : TOTAL_CHUNKS - 519ede93213SVitaly Wool (zhdr->start_middle + zhdr->middle_chunks); 5209a001fc1SVitaly Wool nfree = max(nfree_before, nfree_after); 5219a001fc1SVitaly Wool } else 5229a001fc1SVitaly Wool nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; 5239a001fc1SVitaly Wool return nfree; 5249a001fc1SVitaly Wool } 5259a001fc1SVitaly Wool 5269050cce1SVitaly Wool /* Add to the appropriate unbuddied list */ 5279050cce1SVitaly Wool static inline void add_to_unbuddied(struct z3fold_pool *pool, 5289050cce1SVitaly Wool struct z3fold_header *zhdr) 5299050cce1SVitaly Wool { 5309050cce1SVitaly Wool if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || 5319050cce1SVitaly Wool zhdr->middle_chunks == 0) { 5329050cce1SVitaly Wool struct list_head *unbuddied = get_cpu_ptr(pool->unbuddied); 5339050cce1SVitaly Wool 5349050cce1SVitaly Wool int freechunks = num_free_chunks(zhdr); 5359050cce1SVitaly Wool spin_lock(&pool->lock); 5369050cce1SVitaly Wool list_add(&zhdr->buddy, &unbuddied[freechunks]); 5379050cce1SVitaly Wool spin_unlock(&pool->lock); 5389050cce1SVitaly Wool zhdr->cpu = smp_processor_id(); 5399050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 5409050cce1SVitaly Wool } 5419050cce1SVitaly Wool } 5429050cce1SVitaly Wool 543ede93213SVitaly Wool static inline void *mchunk_memmove(struct z3fold_header *zhdr, 544ede93213SVitaly Wool unsigned short dst_chunk) 545ede93213SVitaly Wool { 546ede93213SVitaly Wool void *beg = zhdr; 547ede93213SVitaly Wool return memmove(beg + (dst_chunk << CHUNK_SHIFT), 548ede93213SVitaly Wool beg + (zhdr->start_middle << CHUNK_SHIFT), 549ede93213SVitaly Wool zhdr->middle_chunks << CHUNK_SHIFT); 550ede93213SVitaly Wool } 551ede93213SVitaly Wool 5521b096e5aSVitaly Wool #define BIG_CHUNK_GAP 3 5539a001fc1SVitaly Wool /* Has to be called with lock held */ 5549a001fc1SVitaly Wool static int z3fold_compact_page(struct z3fold_header *zhdr) 5559a001fc1SVitaly Wool { 5569a001fc1SVitaly Wool struct page *page = virt_to_page(zhdr); 5579a001fc1SVitaly Wool 558ede93213SVitaly Wool if (test_bit(MIDDLE_CHUNK_MAPPED, &page->private)) 559ede93213SVitaly Wool return 0; /* can't move middle chunk, it's used */ 5609a001fc1SVitaly Wool 5611f862989SVitaly Wool if (unlikely(PageIsolated(page))) 5621f862989SVitaly Wool return 0; 5631f862989SVitaly Wool 564ede93213SVitaly Wool if (zhdr->middle_chunks == 0) 565ede93213SVitaly Wool return 0; /* nothing to compact */ 566ede93213SVitaly Wool 567ede93213SVitaly Wool if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) { 568ede93213SVitaly Wool /* move to the beginning */ 569ede93213SVitaly Wool mchunk_memmove(zhdr, ZHDR_CHUNKS); 5709a001fc1SVitaly Wool zhdr->first_chunks = zhdr->middle_chunks; 5719a001fc1SVitaly Wool zhdr->middle_chunks = 0; 5729a001fc1SVitaly Wool zhdr->start_middle = 0; 5739a001fc1SVitaly Wool zhdr->first_num++; 574ede93213SVitaly Wool return 1; 5759a001fc1SVitaly Wool } 5769a001fc1SVitaly Wool 5771b096e5aSVitaly Wool /* 5781b096e5aSVitaly Wool * moving data is expensive, so let's only do that if 5791b096e5aSVitaly Wool * there's substantial gain (at least BIG_CHUNK_GAP chunks) 5801b096e5aSVitaly Wool */ 5811b096e5aSVitaly Wool if (zhdr->first_chunks != 0 && zhdr->last_chunks == 0 && 5821b096e5aSVitaly Wool zhdr->start_middle - (zhdr->first_chunks + ZHDR_CHUNKS) >= 5831b096e5aSVitaly Wool BIG_CHUNK_GAP) { 5841b096e5aSVitaly Wool mchunk_memmove(zhdr, zhdr->first_chunks + ZHDR_CHUNKS); 5851b096e5aSVitaly Wool zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; 5861b096e5aSVitaly Wool return 1; 5871b096e5aSVitaly Wool } else if (zhdr->last_chunks != 0 && zhdr->first_chunks == 0 && 5881b096e5aSVitaly Wool TOTAL_CHUNKS - (zhdr->last_chunks + zhdr->start_middle 5891b096e5aSVitaly Wool + zhdr->middle_chunks) >= 5901b096e5aSVitaly Wool BIG_CHUNK_GAP) { 5911b096e5aSVitaly Wool unsigned short new_start = TOTAL_CHUNKS - zhdr->last_chunks - 5921b096e5aSVitaly Wool zhdr->middle_chunks; 5931b096e5aSVitaly Wool mchunk_memmove(zhdr, new_start); 5941b096e5aSVitaly Wool zhdr->start_middle = new_start; 5951b096e5aSVitaly Wool return 1; 5961b096e5aSVitaly Wool } 5971b096e5aSVitaly Wool 5981b096e5aSVitaly Wool return 0; 5991b096e5aSVitaly Wool } 6001b096e5aSVitaly Wool 601d30561c5SVitaly Wool static void do_compact_page(struct z3fold_header *zhdr, bool locked) 602d30561c5SVitaly Wool { 6039050cce1SVitaly Wool struct z3fold_pool *pool = zhdr_to_pool(zhdr); 604d30561c5SVitaly Wool struct page *page; 605d30561c5SVitaly Wool 606d30561c5SVitaly Wool page = virt_to_page(zhdr); 607d30561c5SVitaly Wool if (locked) 608d30561c5SVitaly Wool WARN_ON(z3fold_page_trylock(zhdr)); 609d30561c5SVitaly Wool else 610d30561c5SVitaly Wool z3fold_page_lock(zhdr); 6115d03a661SVitaly Wool if (WARN_ON(!test_and_clear_bit(NEEDS_COMPACTING, &page->private))) { 612d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 613d30561c5SVitaly Wool return; 614d30561c5SVitaly Wool } 615d30561c5SVitaly Wool spin_lock(&pool->lock); 616d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 617d30561c5SVitaly Wool spin_unlock(&pool->lock); 618d30561c5SVitaly Wool 6195d03a661SVitaly Wool if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { 6205d03a661SVitaly Wool atomic64_dec(&pool->pages_nr); 6215d03a661SVitaly Wool return; 6225d03a661SVitaly Wool } 6235d03a661SVitaly Wool 6241f862989SVitaly Wool if (unlikely(PageIsolated(page) || 6251f862989SVitaly Wool test_bit(PAGE_STALE, &page->private))) { 6261f862989SVitaly Wool z3fold_page_unlock(zhdr); 6271f862989SVitaly Wool return; 6281f862989SVitaly Wool } 6291f862989SVitaly Wool 630d30561c5SVitaly Wool z3fold_compact_page(zhdr); 6319050cce1SVitaly Wool add_to_unbuddied(pool, zhdr); 632d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 633d30561c5SVitaly Wool } 634d30561c5SVitaly Wool 635d30561c5SVitaly Wool static void compact_page_work(struct work_struct *w) 636d30561c5SVitaly Wool { 637d30561c5SVitaly Wool struct z3fold_header *zhdr = container_of(w, struct z3fold_header, 638d30561c5SVitaly Wool work); 639d30561c5SVitaly Wool 640d30561c5SVitaly Wool do_compact_page(zhdr, false); 641d30561c5SVitaly Wool } 642d30561c5SVitaly Wool 6439050cce1SVitaly Wool /* returns _locked_ z3fold page header or NULL */ 6449050cce1SVitaly Wool static inline struct z3fold_header *__z3fold_alloc(struct z3fold_pool *pool, 6459050cce1SVitaly Wool size_t size, bool can_sleep) 6469050cce1SVitaly Wool { 6479050cce1SVitaly Wool struct z3fold_header *zhdr = NULL; 6489050cce1SVitaly Wool struct page *page; 6499050cce1SVitaly Wool struct list_head *unbuddied; 6509050cce1SVitaly Wool int chunks = size_to_chunks(size), i; 6519050cce1SVitaly Wool 6529050cce1SVitaly Wool lookup: 6539050cce1SVitaly Wool /* First, try to find an unbuddied z3fold page. */ 6549050cce1SVitaly Wool unbuddied = get_cpu_ptr(pool->unbuddied); 6559050cce1SVitaly Wool for_each_unbuddied_list(i, chunks) { 6569050cce1SVitaly Wool struct list_head *l = &unbuddied[i]; 6579050cce1SVitaly Wool 6589050cce1SVitaly Wool zhdr = list_first_entry_or_null(READ_ONCE(l), 6599050cce1SVitaly Wool struct z3fold_header, buddy); 6609050cce1SVitaly Wool 6619050cce1SVitaly Wool if (!zhdr) 6629050cce1SVitaly Wool continue; 6639050cce1SVitaly Wool 6649050cce1SVitaly Wool /* Re-check under lock. */ 6659050cce1SVitaly Wool spin_lock(&pool->lock); 6669050cce1SVitaly Wool l = &unbuddied[i]; 6679050cce1SVitaly Wool if (unlikely(zhdr != list_first_entry(READ_ONCE(l), 6689050cce1SVitaly Wool struct z3fold_header, buddy)) || 6699050cce1SVitaly Wool !z3fold_page_trylock(zhdr)) { 6709050cce1SVitaly Wool spin_unlock(&pool->lock); 6719050cce1SVitaly Wool zhdr = NULL; 6729050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 6739050cce1SVitaly Wool if (can_sleep) 6749050cce1SVitaly Wool cond_resched(); 6759050cce1SVitaly Wool goto lookup; 6769050cce1SVitaly Wool } 6779050cce1SVitaly Wool list_del_init(&zhdr->buddy); 6789050cce1SVitaly Wool zhdr->cpu = -1; 6799050cce1SVitaly Wool spin_unlock(&pool->lock); 6809050cce1SVitaly Wool 6819050cce1SVitaly Wool page = virt_to_page(zhdr); 6829050cce1SVitaly Wool if (test_bit(NEEDS_COMPACTING, &page->private)) { 6839050cce1SVitaly Wool z3fold_page_unlock(zhdr); 6849050cce1SVitaly Wool zhdr = NULL; 6859050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 6869050cce1SVitaly Wool if (can_sleep) 6879050cce1SVitaly Wool cond_resched(); 6889050cce1SVitaly Wool goto lookup; 6899050cce1SVitaly Wool } 6909050cce1SVitaly Wool 6919050cce1SVitaly Wool /* 6929050cce1SVitaly Wool * this page could not be removed from its unbuddied 6939050cce1SVitaly Wool * list while pool lock was held, and then we've taken 6949050cce1SVitaly Wool * page lock so kref_put could not be called before 6959050cce1SVitaly Wool * we got here, so it's safe to just call kref_get() 6969050cce1SVitaly Wool */ 6979050cce1SVitaly Wool kref_get(&zhdr->refcount); 6989050cce1SVitaly Wool break; 6999050cce1SVitaly Wool } 7009050cce1SVitaly Wool put_cpu_ptr(pool->unbuddied); 7019050cce1SVitaly Wool 702351618b2SVitaly Wool if (!zhdr) { 703351618b2SVitaly Wool int cpu; 704351618b2SVitaly Wool 705351618b2SVitaly Wool /* look for _exact_ match on other cpus' lists */ 706351618b2SVitaly Wool for_each_online_cpu(cpu) { 707351618b2SVitaly Wool struct list_head *l; 708351618b2SVitaly Wool 709351618b2SVitaly Wool unbuddied = per_cpu_ptr(pool->unbuddied, cpu); 710351618b2SVitaly Wool spin_lock(&pool->lock); 711351618b2SVitaly Wool l = &unbuddied[chunks]; 712351618b2SVitaly Wool 713351618b2SVitaly Wool zhdr = list_first_entry_or_null(READ_ONCE(l), 714351618b2SVitaly Wool struct z3fold_header, buddy); 715351618b2SVitaly Wool 716351618b2SVitaly Wool if (!zhdr || !z3fold_page_trylock(zhdr)) { 717351618b2SVitaly Wool spin_unlock(&pool->lock); 718351618b2SVitaly Wool zhdr = NULL; 719351618b2SVitaly Wool continue; 720351618b2SVitaly Wool } 721351618b2SVitaly Wool list_del_init(&zhdr->buddy); 722351618b2SVitaly Wool zhdr->cpu = -1; 723351618b2SVitaly Wool spin_unlock(&pool->lock); 724351618b2SVitaly Wool 725351618b2SVitaly Wool page = virt_to_page(zhdr); 726351618b2SVitaly Wool if (test_bit(NEEDS_COMPACTING, &page->private)) { 727351618b2SVitaly Wool z3fold_page_unlock(zhdr); 728351618b2SVitaly Wool zhdr = NULL; 729351618b2SVitaly Wool if (can_sleep) 730351618b2SVitaly Wool cond_resched(); 731351618b2SVitaly Wool continue; 732351618b2SVitaly Wool } 733351618b2SVitaly Wool kref_get(&zhdr->refcount); 734351618b2SVitaly Wool break; 735351618b2SVitaly Wool } 736351618b2SVitaly Wool } 737351618b2SVitaly Wool 7389050cce1SVitaly Wool return zhdr; 7399050cce1SVitaly Wool } 740d30561c5SVitaly Wool 741d30561c5SVitaly Wool /* 742d30561c5SVitaly Wool * API Functions 743d30561c5SVitaly Wool */ 744d30561c5SVitaly Wool 745d30561c5SVitaly Wool /** 746d30561c5SVitaly Wool * z3fold_create_pool() - create a new z3fold pool 747d30561c5SVitaly Wool * @name: pool name 748d30561c5SVitaly Wool * @gfp: gfp flags when allocating the z3fold pool structure 749d30561c5SVitaly Wool * @ops: user-defined operations for the z3fold pool 750d30561c5SVitaly Wool * 751d30561c5SVitaly Wool * Return: pointer to the new z3fold pool or NULL if the metadata allocation 752d30561c5SVitaly Wool * failed. 753d30561c5SVitaly Wool */ 754d30561c5SVitaly Wool static struct z3fold_pool *z3fold_create_pool(const char *name, gfp_t gfp, 755d30561c5SVitaly Wool const struct z3fold_ops *ops) 756d30561c5SVitaly Wool { 757d30561c5SVitaly Wool struct z3fold_pool *pool = NULL; 758d30561c5SVitaly Wool int i, cpu; 759d30561c5SVitaly Wool 760d30561c5SVitaly Wool pool = kzalloc(sizeof(struct z3fold_pool), gfp); 761d30561c5SVitaly Wool if (!pool) 762d30561c5SVitaly Wool goto out; 7637c2b8baaSVitaly Wool pool->c_handle = kmem_cache_create("z3fold_handle", 7647c2b8baaSVitaly Wool sizeof(struct z3fold_buddy_slots), 7657c2b8baaSVitaly Wool SLOTS_ALIGN, 0, NULL); 7667c2b8baaSVitaly Wool if (!pool->c_handle) 7677c2b8baaSVitaly Wool goto out_c; 768d30561c5SVitaly Wool spin_lock_init(&pool->lock); 769d30561c5SVitaly Wool spin_lock_init(&pool->stale_lock); 770d30561c5SVitaly Wool pool->unbuddied = __alloc_percpu(sizeof(struct list_head)*NCHUNKS, 2); 7711ec6995dSXidong Wang if (!pool->unbuddied) 7721ec6995dSXidong Wang goto out_pool; 773d30561c5SVitaly Wool for_each_possible_cpu(cpu) { 774d30561c5SVitaly Wool struct list_head *unbuddied = 775d30561c5SVitaly Wool per_cpu_ptr(pool->unbuddied, cpu); 776d30561c5SVitaly Wool for_each_unbuddied_list(i, 0) 777d30561c5SVitaly Wool INIT_LIST_HEAD(&unbuddied[i]); 778d30561c5SVitaly Wool } 779d30561c5SVitaly Wool INIT_LIST_HEAD(&pool->lru); 780d30561c5SVitaly Wool INIT_LIST_HEAD(&pool->stale); 781d30561c5SVitaly Wool atomic64_set(&pool->pages_nr, 0); 782d30561c5SVitaly Wool pool->name = name; 783d30561c5SVitaly Wool pool->compact_wq = create_singlethread_workqueue(pool->name); 784d30561c5SVitaly Wool if (!pool->compact_wq) 7851ec6995dSXidong Wang goto out_unbuddied; 786d30561c5SVitaly Wool pool->release_wq = create_singlethread_workqueue(pool->name); 787d30561c5SVitaly Wool if (!pool->release_wq) 788d30561c5SVitaly Wool goto out_wq; 7891f862989SVitaly Wool if (z3fold_register_migration(pool)) 7901f862989SVitaly Wool goto out_rwq; 791d30561c5SVitaly Wool INIT_WORK(&pool->work, free_pages_work); 792d30561c5SVitaly Wool pool->ops = ops; 793d30561c5SVitaly Wool return pool; 794d30561c5SVitaly Wool 7951f862989SVitaly Wool out_rwq: 7961f862989SVitaly Wool destroy_workqueue(pool->release_wq); 797d30561c5SVitaly Wool out_wq: 798d30561c5SVitaly Wool destroy_workqueue(pool->compact_wq); 7991ec6995dSXidong Wang out_unbuddied: 8001ec6995dSXidong Wang free_percpu(pool->unbuddied); 8011ec6995dSXidong Wang out_pool: 8027c2b8baaSVitaly Wool kmem_cache_destroy(pool->c_handle); 8037c2b8baaSVitaly Wool out_c: 804d30561c5SVitaly Wool kfree(pool); 8051ec6995dSXidong Wang out: 806d30561c5SVitaly Wool return NULL; 807d30561c5SVitaly Wool } 808d30561c5SVitaly Wool 809d30561c5SVitaly Wool /** 810d30561c5SVitaly Wool * z3fold_destroy_pool() - destroys an existing z3fold pool 811d30561c5SVitaly Wool * @pool: the z3fold pool to be destroyed 812d30561c5SVitaly Wool * 813d30561c5SVitaly Wool * The pool should be emptied before this function is called. 814d30561c5SVitaly Wool */ 815d30561c5SVitaly Wool static void z3fold_destroy_pool(struct z3fold_pool *pool) 816d30561c5SVitaly Wool { 8177c2b8baaSVitaly Wool kmem_cache_destroy(pool->c_handle); 8181f862989SVitaly Wool z3fold_unregister_migration(pool); 819d30561c5SVitaly Wool destroy_workqueue(pool->release_wq); 820d30561c5SVitaly Wool destroy_workqueue(pool->compact_wq); 821d30561c5SVitaly Wool kfree(pool); 822d30561c5SVitaly Wool } 823d30561c5SVitaly Wool 8249a001fc1SVitaly Wool /** 8259a001fc1SVitaly Wool * z3fold_alloc() - allocates a region of a given size 8269a001fc1SVitaly Wool * @pool: z3fold pool from which to allocate 8279a001fc1SVitaly Wool * @size: size in bytes of the desired allocation 8289a001fc1SVitaly Wool * @gfp: gfp flags used if the pool needs to grow 8299a001fc1SVitaly Wool * @handle: handle of the new allocation 8309a001fc1SVitaly Wool * 8319a001fc1SVitaly Wool * This function will attempt to find a free region in the pool large enough to 8329a001fc1SVitaly Wool * satisfy the allocation request. A search of the unbuddied lists is 8339a001fc1SVitaly Wool * performed first. If no suitable free region is found, then a new page is 8349a001fc1SVitaly Wool * allocated and added to the pool to satisfy the request. 8359a001fc1SVitaly Wool * 8369a001fc1SVitaly Wool * gfp should not set __GFP_HIGHMEM as highmem pages cannot be used 8379a001fc1SVitaly Wool * as z3fold pool pages. 8389a001fc1SVitaly Wool * 8399a001fc1SVitaly Wool * Return: 0 if success and handle is set, otherwise -EINVAL if the size or 8409a001fc1SVitaly Wool * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate 8419a001fc1SVitaly Wool * a new page. 8429a001fc1SVitaly Wool */ 8439a001fc1SVitaly Wool static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, 8449a001fc1SVitaly Wool unsigned long *handle) 8459a001fc1SVitaly Wool { 8469050cce1SVitaly Wool int chunks = size_to_chunks(size); 8479a001fc1SVitaly Wool struct z3fold_header *zhdr = NULL; 848d30561c5SVitaly Wool struct page *page = NULL; 8499a001fc1SVitaly Wool enum buddy bud; 8508a97ea54SMatthew Wilcox bool can_sleep = gfpflags_allow_blocking(gfp); 8519a001fc1SVitaly Wool 8529a001fc1SVitaly Wool if (!size || (gfp & __GFP_HIGHMEM)) 8539a001fc1SVitaly Wool return -EINVAL; 8549a001fc1SVitaly Wool 8559a001fc1SVitaly Wool if (size > PAGE_SIZE) 8569a001fc1SVitaly Wool return -ENOSPC; 8579a001fc1SVitaly Wool 8589a001fc1SVitaly Wool if (size > PAGE_SIZE - ZHDR_SIZE_ALIGNED - CHUNK_SIZE) 8599a001fc1SVitaly Wool bud = HEADLESS; 8609a001fc1SVitaly Wool else { 8619050cce1SVitaly Wool retry: 8629050cce1SVitaly Wool zhdr = __z3fold_alloc(pool, size, can_sleep); 863d30561c5SVitaly Wool if (zhdr) { 8649a001fc1SVitaly Wool if (zhdr->first_chunks == 0) { 8659a001fc1SVitaly Wool if (zhdr->middle_chunks != 0 && 8669a001fc1SVitaly Wool chunks >= zhdr->start_middle) 8679a001fc1SVitaly Wool bud = LAST; 8689a001fc1SVitaly Wool else 8699a001fc1SVitaly Wool bud = FIRST; 8709a001fc1SVitaly Wool } else if (zhdr->last_chunks == 0) 8719a001fc1SVitaly Wool bud = LAST; 8729a001fc1SVitaly Wool else if (zhdr->middle_chunks == 0) 8739a001fc1SVitaly Wool bud = MIDDLE; 8749a001fc1SVitaly Wool else { 8755a27aa82SVitaly Wool if (kref_put(&zhdr->refcount, 876d30561c5SVitaly Wool release_z3fold_page_locked)) 8775a27aa82SVitaly Wool atomic64_dec(&pool->pages_nr); 878d30561c5SVitaly Wool else 879d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 8809a001fc1SVitaly Wool pr_err("No free chunks in unbuddied\n"); 8819a001fc1SVitaly Wool WARN_ON(1); 8829050cce1SVitaly Wool goto retry; 8839a001fc1SVitaly Wool } 8849050cce1SVitaly Wool page = virt_to_page(zhdr); 8859a001fc1SVitaly Wool goto found; 8869a001fc1SVitaly Wool } 8879a001fc1SVitaly Wool bud = FIRST; 8889a001fc1SVitaly Wool } 8899a001fc1SVitaly Wool 8905c9bab59SVitaly Wool page = NULL; 8915c9bab59SVitaly Wool if (can_sleep) { 892d30561c5SVitaly Wool spin_lock(&pool->stale_lock); 893d30561c5SVitaly Wool zhdr = list_first_entry_or_null(&pool->stale, 894d30561c5SVitaly Wool struct z3fold_header, buddy); 895d30561c5SVitaly Wool /* 8965c9bab59SVitaly Wool * Before allocating a page, let's see if we can take one from 8975c9bab59SVitaly Wool * the stale pages list. cancel_work_sync() can sleep so we 8985c9bab59SVitaly Wool * limit this case to the contexts where we can sleep 899d30561c5SVitaly Wool */ 9005c9bab59SVitaly Wool if (zhdr) { 901d30561c5SVitaly Wool list_del(&zhdr->buddy); 902d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 903d30561c5SVitaly Wool cancel_work_sync(&zhdr->work); 904d30561c5SVitaly Wool page = virt_to_page(zhdr); 905d30561c5SVitaly Wool } else { 906d30561c5SVitaly Wool spin_unlock(&pool->stale_lock); 907d30561c5SVitaly Wool } 9085c9bab59SVitaly Wool } 9095c9bab59SVitaly Wool if (!page) 9105c9bab59SVitaly Wool page = alloc_page(gfp); 911d30561c5SVitaly Wool 9129a001fc1SVitaly Wool if (!page) 9139a001fc1SVitaly Wool return -ENOMEM; 9142f1e5e4dSVitaly Wool 915d30561c5SVitaly Wool zhdr = init_z3fold_page(page, pool); 9169050cce1SVitaly Wool if (!zhdr) { 9179050cce1SVitaly Wool __free_page(page); 9189050cce1SVitaly Wool return -ENOMEM; 9199050cce1SVitaly Wool } 9209050cce1SVitaly Wool atomic64_inc(&pool->pages_nr); 9219a001fc1SVitaly Wool 9229a001fc1SVitaly Wool if (bud == HEADLESS) { 9239a001fc1SVitaly Wool set_bit(PAGE_HEADLESS, &page->private); 9249a001fc1SVitaly Wool goto headless; 9259a001fc1SVitaly Wool } 9261f862989SVitaly Wool __SetPageMovable(page, pool->inode->i_mapping); 9272f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 9289a001fc1SVitaly Wool 9299a001fc1SVitaly Wool found: 9309a001fc1SVitaly Wool if (bud == FIRST) 9319a001fc1SVitaly Wool zhdr->first_chunks = chunks; 9329a001fc1SVitaly Wool else if (bud == LAST) 9339a001fc1SVitaly Wool zhdr->last_chunks = chunks; 9349a001fc1SVitaly Wool else { 9359a001fc1SVitaly Wool zhdr->middle_chunks = chunks; 936ede93213SVitaly Wool zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS; 9379a001fc1SVitaly Wool } 9389050cce1SVitaly Wool add_to_unbuddied(pool, zhdr); 9399a001fc1SVitaly Wool 9409a001fc1SVitaly Wool headless: 941d30561c5SVitaly Wool spin_lock(&pool->lock); 9429a001fc1SVitaly Wool /* Add/move z3fold page to beginning of LRU */ 9439a001fc1SVitaly Wool if (!list_empty(&page->lru)) 9449a001fc1SVitaly Wool list_del(&page->lru); 9459a001fc1SVitaly Wool 9469a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 9479a001fc1SVitaly Wool 9489a001fc1SVitaly Wool *handle = encode_handle(zhdr, bud); 9499a001fc1SVitaly Wool spin_unlock(&pool->lock); 9502f1e5e4dSVitaly Wool if (bud != HEADLESS) 9512f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 9529a001fc1SVitaly Wool 9539a001fc1SVitaly Wool return 0; 9549a001fc1SVitaly Wool } 9559a001fc1SVitaly Wool 9569a001fc1SVitaly Wool /** 9579a001fc1SVitaly Wool * z3fold_free() - frees the allocation associated with the given handle 9589a001fc1SVitaly Wool * @pool: pool in which the allocation resided 9599a001fc1SVitaly Wool * @handle: handle associated with the allocation returned by z3fold_alloc() 9609a001fc1SVitaly Wool * 9619a001fc1SVitaly Wool * In the case that the z3fold page in which the allocation resides is under 9629a001fc1SVitaly Wool * reclaim, as indicated by the PG_reclaim flag being set, this function 9639a001fc1SVitaly Wool * only sets the first|last_chunks to 0. The page is actually freed 9649a001fc1SVitaly Wool * once both buddies are evicted (see z3fold_reclaim_page() below). 9659a001fc1SVitaly Wool */ 9669a001fc1SVitaly Wool static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) 9679a001fc1SVitaly Wool { 9689a001fc1SVitaly Wool struct z3fold_header *zhdr; 9699a001fc1SVitaly Wool struct page *page; 9709a001fc1SVitaly Wool enum buddy bud; 9719a001fc1SVitaly Wool 9729a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 9739a001fc1SVitaly Wool page = virt_to_page(zhdr); 9749a001fc1SVitaly Wool 9759a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 976ca0246bbSVitaly Wool /* if a headless page is under reclaim, just leave. 977ca0246bbSVitaly Wool * NB: we use test_and_set_bit for a reason: if the bit 978ca0246bbSVitaly Wool * has not been set before, we release this page 979ca0246bbSVitaly Wool * immediately so we don't care about its value any more. 980ca0246bbSVitaly Wool */ 981ca0246bbSVitaly Wool if (!test_and_set_bit(PAGE_CLAIMED, &page->private)) { 982ca0246bbSVitaly Wool spin_lock(&pool->lock); 983ca0246bbSVitaly Wool list_del(&page->lru); 984ca0246bbSVitaly Wool spin_unlock(&pool->lock); 9851f862989SVitaly Wool free_z3fold_page(page, true); 986ca0246bbSVitaly Wool atomic64_dec(&pool->pages_nr); 987ca0246bbSVitaly Wool } 988ca0246bbSVitaly Wool return; 989ca0246bbSVitaly Wool } 990ca0246bbSVitaly Wool 991ca0246bbSVitaly Wool /* Non-headless case */ 9922f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 99343afc194SVitaly Wool bud = handle_to_buddy(handle); 9949a001fc1SVitaly Wool 9959a001fc1SVitaly Wool switch (bud) { 9969a001fc1SVitaly Wool case FIRST: 9979a001fc1SVitaly Wool zhdr->first_chunks = 0; 9989a001fc1SVitaly Wool break; 9999a001fc1SVitaly Wool case MIDDLE: 10009a001fc1SVitaly Wool zhdr->middle_chunks = 0; 10019a001fc1SVitaly Wool break; 10029a001fc1SVitaly Wool case LAST: 10039a001fc1SVitaly Wool zhdr->last_chunks = 0; 10049a001fc1SVitaly Wool break; 10059a001fc1SVitaly Wool default: 10069a001fc1SVitaly Wool pr_err("%s: unknown bud %d\n", __func__, bud); 10079a001fc1SVitaly Wool WARN_ON(1); 10082f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 10099a001fc1SVitaly Wool return; 10109a001fc1SVitaly Wool } 10119a001fc1SVitaly Wool 10127c2b8baaSVitaly Wool free_handle(handle); 1013d30561c5SVitaly Wool if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) { 1014d30561c5SVitaly Wool atomic64_dec(&pool->pages_nr); 1015d30561c5SVitaly Wool return; 1016d30561c5SVitaly Wool } 1017ca0246bbSVitaly Wool if (test_bit(PAGE_CLAIMED, &page->private)) { 10186098d7e1SVitaly Wool z3fold_page_unlock(zhdr); 10196098d7e1SVitaly Wool return; 10206098d7e1SVitaly Wool } 10211f862989SVitaly Wool if (unlikely(PageIsolated(page)) || 10221f862989SVitaly Wool test_and_set_bit(NEEDS_COMPACTING, &page->private)) { 1023d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 1024d30561c5SVitaly Wool return; 1025d30561c5SVitaly Wool } 1026d30561c5SVitaly Wool if (zhdr->cpu < 0 || !cpu_online(zhdr->cpu)) { 1027d30561c5SVitaly Wool spin_lock(&pool->lock); 1028d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 1029d30561c5SVitaly Wool spin_unlock(&pool->lock); 1030d30561c5SVitaly Wool zhdr->cpu = -1; 10315d03a661SVitaly Wool kref_get(&zhdr->refcount); 1032d30561c5SVitaly Wool do_compact_page(zhdr, true); 1033d30561c5SVitaly Wool return; 1034d30561c5SVitaly Wool } 10355d03a661SVitaly Wool kref_get(&zhdr->refcount); 1036d30561c5SVitaly Wool queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work); 1037d30561c5SVitaly Wool z3fold_page_unlock(zhdr); 10389a001fc1SVitaly Wool } 10399a001fc1SVitaly Wool 10409a001fc1SVitaly Wool /** 10419a001fc1SVitaly Wool * z3fold_reclaim_page() - evicts allocations from a pool page and frees it 10429a001fc1SVitaly Wool * @pool: pool from which a page will attempt to be evicted 1043f144c390SMike Rapoport * @retries: number of pages on the LRU list for which eviction will 10449a001fc1SVitaly Wool * be attempted before failing 10459a001fc1SVitaly Wool * 10469a001fc1SVitaly Wool * z3fold reclaim is different from normal system reclaim in that it is done 10479a001fc1SVitaly Wool * from the bottom, up. This is because only the bottom layer, z3fold, has 10489a001fc1SVitaly Wool * information on how the allocations are organized within each z3fold page. 10499a001fc1SVitaly Wool * This has the potential to create interesting locking situations between 10509a001fc1SVitaly Wool * z3fold and the user, however. 10519a001fc1SVitaly Wool * 10529a001fc1SVitaly Wool * To avoid these, this is how z3fold_reclaim_page() should be called: 1053f144c390SMike Rapoport * 10549a001fc1SVitaly Wool * The user detects a page should be reclaimed and calls z3fold_reclaim_page(). 10559a001fc1SVitaly Wool * z3fold_reclaim_page() will remove a z3fold page from the pool LRU list and 10569a001fc1SVitaly Wool * call the user-defined eviction handler with the pool and handle as 10579a001fc1SVitaly Wool * arguments. 10589a001fc1SVitaly Wool * 10599a001fc1SVitaly Wool * If the handle can not be evicted, the eviction handler should return 10609a001fc1SVitaly Wool * non-zero. z3fold_reclaim_page() will add the z3fold page back to the 10619a001fc1SVitaly Wool * appropriate list and try the next z3fold page on the LRU up to 10629a001fc1SVitaly Wool * a user defined number of retries. 10639a001fc1SVitaly Wool * 10649a001fc1SVitaly Wool * If the handle is successfully evicted, the eviction handler should 10659a001fc1SVitaly Wool * return 0 _and_ should have called z3fold_free() on the handle. z3fold_free() 10669a001fc1SVitaly Wool * contains logic to delay freeing the page if the page is under reclaim, 10679a001fc1SVitaly Wool * as indicated by the setting of the PG_reclaim flag on the underlying page. 10689a001fc1SVitaly Wool * 10699a001fc1SVitaly Wool * If all buddies in the z3fold page are successfully evicted, then the 10709a001fc1SVitaly Wool * z3fold page can be freed. 10719a001fc1SVitaly Wool * 10729a001fc1SVitaly Wool * Returns: 0 if page is successfully freed, otherwise -EINVAL if there are 10739a001fc1SVitaly Wool * no pages to evict or an eviction handler is not registered, -EAGAIN if 10749a001fc1SVitaly Wool * the retry limit was hit. 10759a001fc1SVitaly Wool */ 10769a001fc1SVitaly Wool static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) 10779a001fc1SVitaly Wool { 1078d30561c5SVitaly Wool int i, ret = 0; 1079d30561c5SVitaly Wool struct z3fold_header *zhdr = NULL; 1080d30561c5SVitaly Wool struct page *page = NULL; 1081d30561c5SVitaly Wool struct list_head *pos; 10829a001fc1SVitaly Wool unsigned long first_handle = 0, middle_handle = 0, last_handle = 0; 10839a001fc1SVitaly Wool 10849a001fc1SVitaly Wool spin_lock(&pool->lock); 10852f1e5e4dSVitaly Wool if (!pool->ops || !pool->ops->evict || retries == 0) { 10869a001fc1SVitaly Wool spin_unlock(&pool->lock); 10879a001fc1SVitaly Wool return -EINVAL; 10889a001fc1SVitaly Wool } 10899a001fc1SVitaly Wool for (i = 0; i < retries; i++) { 10902f1e5e4dSVitaly Wool if (list_empty(&pool->lru)) { 10912f1e5e4dSVitaly Wool spin_unlock(&pool->lock); 10922f1e5e4dSVitaly Wool return -EINVAL; 10932f1e5e4dSVitaly Wool } 1094d30561c5SVitaly Wool list_for_each_prev(pos, &pool->lru) { 1095d30561c5SVitaly Wool page = list_entry(pos, struct page, lru); 1096ca0246bbSVitaly Wool 1097ca0246bbSVitaly Wool /* this bit could have been set by free, in which case 1098ca0246bbSVitaly Wool * we pass over to the next page in the pool. 1099ca0246bbSVitaly Wool */ 1100ca0246bbSVitaly Wool if (test_and_set_bit(PAGE_CLAIMED, &page->private)) 1101ca0246bbSVitaly Wool continue; 11029a001fc1SVitaly Wool 11031f862989SVitaly Wool if (unlikely(PageIsolated(page))) 11041f862989SVitaly Wool continue; 1105ca0246bbSVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 1106ca0246bbSVitaly Wool break; 1107ca0246bbSVitaly Wool 11081f862989SVitaly Wool zhdr = page_address(page); 1109ca0246bbSVitaly Wool if (!z3fold_page_trylock(zhdr)) { 1110ca0246bbSVitaly Wool zhdr = NULL; 1111d30561c5SVitaly Wool continue; /* can't evict at this point */ 1112ca0246bbSVitaly Wool } 11135a27aa82SVitaly Wool kref_get(&zhdr->refcount); 1114d30561c5SVitaly Wool list_del_init(&zhdr->buddy); 1115d30561c5SVitaly Wool zhdr->cpu = -1; 11166098d7e1SVitaly Wool break; 1117d30561c5SVitaly Wool } 1118d30561c5SVitaly Wool 1119ca0246bbSVitaly Wool if (!zhdr) 1120ca0246bbSVitaly Wool break; 1121ca0246bbSVitaly Wool 1122d30561c5SVitaly Wool list_del_init(&page->lru); 11232f1e5e4dSVitaly Wool spin_unlock(&pool->lock); 1124d30561c5SVitaly Wool 1125d30561c5SVitaly Wool if (!test_bit(PAGE_HEADLESS, &page->private)) { 11269a001fc1SVitaly Wool /* 11279a001fc1SVitaly Wool * We need encode the handles before unlocking, since 11289a001fc1SVitaly Wool * we can race with free that will set 11299a001fc1SVitaly Wool * (first|last)_chunks to 0 11309a001fc1SVitaly Wool */ 11319a001fc1SVitaly Wool first_handle = 0; 11329a001fc1SVitaly Wool last_handle = 0; 11339a001fc1SVitaly Wool middle_handle = 0; 11349a001fc1SVitaly Wool if (zhdr->first_chunks) 11359a001fc1SVitaly Wool first_handle = encode_handle(zhdr, FIRST); 11369a001fc1SVitaly Wool if (zhdr->middle_chunks) 11379a001fc1SVitaly Wool middle_handle = encode_handle(zhdr, MIDDLE); 11389a001fc1SVitaly Wool if (zhdr->last_chunks) 11399a001fc1SVitaly Wool last_handle = encode_handle(zhdr, LAST); 1140d30561c5SVitaly Wool /* 1141d30561c5SVitaly Wool * it's safe to unlock here because we hold a 1142d30561c5SVitaly Wool * reference to this page 1143d30561c5SVitaly Wool */ 11442f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 11459a001fc1SVitaly Wool } else { 11469a001fc1SVitaly Wool first_handle = encode_handle(zhdr, HEADLESS); 11479a001fc1SVitaly Wool last_handle = middle_handle = 0; 11482f1e5e4dSVitaly Wool } 11499a001fc1SVitaly Wool 11509a001fc1SVitaly Wool /* Issue the eviction callback(s) */ 11519a001fc1SVitaly Wool if (middle_handle) { 11529a001fc1SVitaly Wool ret = pool->ops->evict(pool, middle_handle); 11539a001fc1SVitaly Wool if (ret) 11549a001fc1SVitaly Wool goto next; 11559a001fc1SVitaly Wool } 11569a001fc1SVitaly Wool if (first_handle) { 11579a001fc1SVitaly Wool ret = pool->ops->evict(pool, first_handle); 11589a001fc1SVitaly Wool if (ret) 11599a001fc1SVitaly Wool goto next; 11609a001fc1SVitaly Wool } 11619a001fc1SVitaly Wool if (last_handle) { 11629a001fc1SVitaly Wool ret = pool->ops->evict(pool, last_handle); 11639a001fc1SVitaly Wool if (ret) 11649a001fc1SVitaly Wool goto next; 11659a001fc1SVitaly Wool } 11669a001fc1SVitaly Wool next: 11675a27aa82SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) { 11685a27aa82SVitaly Wool if (ret == 0) { 11691f862989SVitaly Wool free_z3fold_page(page, true); 1170ca0246bbSVitaly Wool atomic64_dec(&pool->pages_nr); 11719a001fc1SVitaly Wool return 0; 11725a27aa82SVitaly Wool } 11736098d7e1SVitaly Wool spin_lock(&pool->lock); 11746098d7e1SVitaly Wool list_add(&page->lru, &pool->lru); 1175d5567c9dSVitaly Wool spin_unlock(&pool->lock); 11766098d7e1SVitaly Wool } else { 11776098d7e1SVitaly Wool z3fold_page_lock(zhdr); 1178ca0246bbSVitaly Wool clear_bit(PAGE_CLAIMED, &page->private); 11796098d7e1SVitaly Wool if (kref_put(&zhdr->refcount, 11806098d7e1SVitaly Wool release_z3fold_page_locked)) { 11816098d7e1SVitaly Wool atomic64_dec(&pool->pages_nr); 11825a27aa82SVitaly Wool return 0; 11835a27aa82SVitaly Wool } 11845a27aa82SVitaly Wool /* 11856098d7e1SVitaly Wool * if we are here, the page is still not completely 11866098d7e1SVitaly Wool * free. Take the global pool lock then to be able 11876098d7e1SVitaly Wool * to add it back to the lru list 11885a27aa82SVitaly Wool */ 11896098d7e1SVitaly Wool spin_lock(&pool->lock); 11909a001fc1SVitaly Wool list_add(&page->lru, &pool->lru); 11916098d7e1SVitaly Wool spin_unlock(&pool->lock); 11926098d7e1SVitaly Wool z3fold_page_unlock(zhdr); 11936098d7e1SVitaly Wool } 11946098d7e1SVitaly Wool 11956098d7e1SVitaly Wool /* We started off locked to we need to lock the pool back */ 11966098d7e1SVitaly Wool spin_lock(&pool->lock); 11979a001fc1SVitaly Wool } 11989a001fc1SVitaly Wool spin_unlock(&pool->lock); 11999a001fc1SVitaly Wool return -EAGAIN; 12009a001fc1SVitaly Wool } 12019a001fc1SVitaly Wool 12029a001fc1SVitaly Wool /** 12039a001fc1SVitaly Wool * z3fold_map() - maps the allocation associated with the given handle 12049a001fc1SVitaly Wool * @pool: pool in which the allocation resides 12059a001fc1SVitaly Wool * @handle: handle associated with the allocation to be mapped 12069a001fc1SVitaly Wool * 12079a001fc1SVitaly Wool * Extracts the buddy number from handle and constructs the pointer to the 12089a001fc1SVitaly Wool * correct starting chunk within the page. 12099a001fc1SVitaly Wool * 12109a001fc1SVitaly Wool * Returns: a pointer to the mapped allocation 12119a001fc1SVitaly Wool */ 12129a001fc1SVitaly Wool static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) 12139a001fc1SVitaly Wool { 12149a001fc1SVitaly Wool struct z3fold_header *zhdr; 12159a001fc1SVitaly Wool struct page *page; 12169a001fc1SVitaly Wool void *addr; 12179a001fc1SVitaly Wool enum buddy buddy; 12189a001fc1SVitaly Wool 12199a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 12209a001fc1SVitaly Wool addr = zhdr; 12219a001fc1SVitaly Wool page = virt_to_page(zhdr); 12229a001fc1SVitaly Wool 12239a001fc1SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 12249a001fc1SVitaly Wool goto out; 12259a001fc1SVitaly Wool 12262f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 12279a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 12289a001fc1SVitaly Wool switch (buddy) { 12299a001fc1SVitaly Wool case FIRST: 12309a001fc1SVitaly Wool addr += ZHDR_SIZE_ALIGNED; 12319a001fc1SVitaly Wool break; 12329a001fc1SVitaly Wool case MIDDLE: 12339a001fc1SVitaly Wool addr += zhdr->start_middle << CHUNK_SHIFT; 12349a001fc1SVitaly Wool set_bit(MIDDLE_CHUNK_MAPPED, &page->private); 12359a001fc1SVitaly Wool break; 12369a001fc1SVitaly Wool case LAST: 1237ca0246bbSVitaly Wool addr += PAGE_SIZE - (handle_to_chunks(handle) << CHUNK_SHIFT); 12389a001fc1SVitaly Wool break; 12399a001fc1SVitaly Wool default: 12409a001fc1SVitaly Wool pr_err("unknown buddy id %d\n", buddy); 12419a001fc1SVitaly Wool WARN_ON(1); 12429a001fc1SVitaly Wool addr = NULL; 12439a001fc1SVitaly Wool break; 12449a001fc1SVitaly Wool } 12452f1e5e4dSVitaly Wool 12461f862989SVitaly Wool if (addr) 12471f862989SVitaly Wool zhdr->mapped_count++; 12482f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 12499a001fc1SVitaly Wool out: 12509a001fc1SVitaly Wool return addr; 12519a001fc1SVitaly Wool } 12529a001fc1SVitaly Wool 12539a001fc1SVitaly Wool /** 12549a001fc1SVitaly Wool * z3fold_unmap() - unmaps the allocation associated with the given handle 12559a001fc1SVitaly Wool * @pool: pool in which the allocation resides 12569a001fc1SVitaly Wool * @handle: handle associated with the allocation to be unmapped 12579a001fc1SVitaly Wool */ 12589a001fc1SVitaly Wool static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) 12599a001fc1SVitaly Wool { 12609a001fc1SVitaly Wool struct z3fold_header *zhdr; 12619a001fc1SVitaly Wool struct page *page; 12629a001fc1SVitaly Wool enum buddy buddy; 12639a001fc1SVitaly Wool 12649a001fc1SVitaly Wool zhdr = handle_to_z3fold_header(handle); 12659a001fc1SVitaly Wool page = virt_to_page(zhdr); 12669a001fc1SVitaly Wool 12672f1e5e4dSVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 12689a001fc1SVitaly Wool return; 12699a001fc1SVitaly Wool 12702f1e5e4dSVitaly Wool z3fold_page_lock(zhdr); 12719a001fc1SVitaly Wool buddy = handle_to_buddy(handle); 12729a001fc1SVitaly Wool if (buddy == MIDDLE) 12739a001fc1SVitaly Wool clear_bit(MIDDLE_CHUNK_MAPPED, &page->private); 12741f862989SVitaly Wool zhdr->mapped_count--; 12752f1e5e4dSVitaly Wool z3fold_page_unlock(zhdr); 12769a001fc1SVitaly Wool } 12779a001fc1SVitaly Wool 12789a001fc1SVitaly Wool /** 12799a001fc1SVitaly Wool * z3fold_get_pool_size() - gets the z3fold pool size in pages 12809a001fc1SVitaly Wool * @pool: pool whose size is being queried 12819a001fc1SVitaly Wool * 128212d59ae6SVitaly Wool * Returns: size in pages of the given pool. 12839a001fc1SVitaly Wool */ 12849a001fc1SVitaly Wool static u64 z3fold_get_pool_size(struct z3fold_pool *pool) 12859a001fc1SVitaly Wool { 128612d59ae6SVitaly Wool return atomic64_read(&pool->pages_nr); 12879a001fc1SVitaly Wool } 12889a001fc1SVitaly Wool 12891f862989SVitaly Wool static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode) 12901f862989SVitaly Wool { 12911f862989SVitaly Wool struct z3fold_header *zhdr; 12921f862989SVitaly Wool struct z3fold_pool *pool; 12931f862989SVitaly Wool 12941f862989SVitaly Wool VM_BUG_ON_PAGE(!PageMovable(page), page); 12951f862989SVitaly Wool VM_BUG_ON_PAGE(PageIsolated(page), page); 12961f862989SVitaly Wool 12971f862989SVitaly Wool if (test_bit(PAGE_HEADLESS, &page->private)) 12981f862989SVitaly Wool return false; 12991f862989SVitaly Wool 13001f862989SVitaly Wool zhdr = page_address(page); 13011f862989SVitaly Wool z3fold_page_lock(zhdr); 13021f862989SVitaly Wool if (test_bit(NEEDS_COMPACTING, &page->private) || 13031f862989SVitaly Wool test_bit(PAGE_STALE, &page->private)) 13041f862989SVitaly Wool goto out; 13051f862989SVitaly Wool 13061f862989SVitaly Wool pool = zhdr_to_pool(zhdr); 13071f862989SVitaly Wool 13081f862989SVitaly Wool if (zhdr->mapped_count == 0) { 13091f862989SVitaly Wool kref_get(&zhdr->refcount); 13101f862989SVitaly Wool if (!list_empty(&zhdr->buddy)) 13111f862989SVitaly Wool list_del_init(&zhdr->buddy); 13121f862989SVitaly Wool spin_lock(&pool->lock); 13131f862989SVitaly Wool if (!list_empty(&page->lru)) 13141f862989SVitaly Wool list_del(&page->lru); 13151f862989SVitaly Wool spin_unlock(&pool->lock); 13161f862989SVitaly Wool z3fold_page_unlock(zhdr); 13171f862989SVitaly Wool return true; 13181f862989SVitaly Wool } 13191f862989SVitaly Wool out: 13201f862989SVitaly Wool z3fold_page_unlock(zhdr); 13211f862989SVitaly Wool return false; 13221f862989SVitaly Wool } 13231f862989SVitaly Wool 13241f862989SVitaly Wool static int z3fold_page_migrate(struct address_space *mapping, struct page *newpage, 13251f862989SVitaly Wool struct page *page, enum migrate_mode mode) 13261f862989SVitaly Wool { 13271f862989SVitaly Wool struct z3fold_header *zhdr, *new_zhdr; 13281f862989SVitaly Wool struct z3fold_pool *pool; 13291f862989SVitaly Wool struct address_space *new_mapping; 13301f862989SVitaly Wool 13311f862989SVitaly Wool VM_BUG_ON_PAGE(!PageMovable(page), page); 13321f862989SVitaly Wool VM_BUG_ON_PAGE(!PageIsolated(page), page); 13331f862989SVitaly Wool 13341f862989SVitaly Wool zhdr = page_address(page); 13351f862989SVitaly Wool pool = zhdr_to_pool(zhdr); 13361f862989SVitaly Wool 13371f862989SVitaly Wool if (!trylock_page(page)) 13381f862989SVitaly Wool return -EAGAIN; 13391f862989SVitaly Wool 13401f862989SVitaly Wool if (!z3fold_page_trylock(zhdr)) { 13411f862989SVitaly Wool unlock_page(page); 13421f862989SVitaly Wool return -EAGAIN; 13431f862989SVitaly Wool } 13441f862989SVitaly Wool if (zhdr->mapped_count != 0) { 13451f862989SVitaly Wool z3fold_page_unlock(zhdr); 13461f862989SVitaly Wool unlock_page(page); 13471f862989SVitaly Wool return -EBUSY; 13481f862989SVitaly Wool } 13491f862989SVitaly Wool new_zhdr = page_address(newpage); 13501f862989SVitaly Wool memcpy(new_zhdr, zhdr, PAGE_SIZE); 13511f862989SVitaly Wool newpage->private = page->private; 13521f862989SVitaly Wool page->private = 0; 13531f862989SVitaly Wool z3fold_page_unlock(zhdr); 13541f862989SVitaly Wool spin_lock_init(&new_zhdr->page_lock); 13551f862989SVitaly Wool new_mapping = page_mapping(page); 13561f862989SVitaly Wool __ClearPageMovable(page); 13571f862989SVitaly Wool ClearPagePrivate(page); 13581f862989SVitaly Wool 13591f862989SVitaly Wool get_page(newpage); 13601f862989SVitaly Wool z3fold_page_lock(new_zhdr); 13611f862989SVitaly Wool if (new_zhdr->first_chunks) 13621f862989SVitaly Wool encode_handle(new_zhdr, FIRST); 13631f862989SVitaly Wool if (new_zhdr->last_chunks) 13641f862989SVitaly Wool encode_handle(new_zhdr, LAST); 13651f862989SVitaly Wool if (new_zhdr->middle_chunks) 13661f862989SVitaly Wool encode_handle(new_zhdr, MIDDLE); 13671f862989SVitaly Wool set_bit(NEEDS_COMPACTING, &newpage->private); 13681f862989SVitaly Wool new_zhdr->cpu = smp_processor_id(); 13691f862989SVitaly Wool spin_lock(&pool->lock); 13701f862989SVitaly Wool list_add(&newpage->lru, &pool->lru); 13711f862989SVitaly Wool spin_unlock(&pool->lock); 13721f862989SVitaly Wool __SetPageMovable(newpage, new_mapping); 13731f862989SVitaly Wool z3fold_page_unlock(new_zhdr); 13741f862989SVitaly Wool 13751f862989SVitaly Wool queue_work_on(new_zhdr->cpu, pool->compact_wq, &new_zhdr->work); 13761f862989SVitaly Wool 13771f862989SVitaly Wool page_mapcount_reset(page); 13781f862989SVitaly Wool unlock_page(page); 13791f862989SVitaly Wool put_page(page); 13801f862989SVitaly Wool return 0; 13811f862989SVitaly Wool } 13821f862989SVitaly Wool 13831f862989SVitaly Wool static void z3fold_page_putback(struct page *page) 13841f862989SVitaly Wool { 13851f862989SVitaly Wool struct z3fold_header *zhdr; 13861f862989SVitaly Wool struct z3fold_pool *pool; 13871f862989SVitaly Wool 13881f862989SVitaly Wool zhdr = page_address(page); 13891f862989SVitaly Wool pool = zhdr_to_pool(zhdr); 13901f862989SVitaly Wool 13911f862989SVitaly Wool z3fold_page_lock(zhdr); 13921f862989SVitaly Wool if (!list_empty(&zhdr->buddy)) 13931f862989SVitaly Wool list_del_init(&zhdr->buddy); 13941f862989SVitaly Wool INIT_LIST_HEAD(&page->lru); 13951f862989SVitaly Wool if (kref_put(&zhdr->refcount, release_z3fold_page_locked)) { 13961f862989SVitaly Wool atomic64_dec(&pool->pages_nr); 13971f862989SVitaly Wool return; 13981f862989SVitaly Wool } 13991f862989SVitaly Wool spin_lock(&pool->lock); 14001f862989SVitaly Wool list_add(&page->lru, &pool->lru); 14011f862989SVitaly Wool spin_unlock(&pool->lock); 14021f862989SVitaly Wool z3fold_page_unlock(zhdr); 14031f862989SVitaly Wool } 14041f862989SVitaly Wool 14051f862989SVitaly Wool static const struct address_space_operations z3fold_aops = { 14061f862989SVitaly Wool .isolate_page = z3fold_page_isolate, 14071f862989SVitaly Wool .migratepage = z3fold_page_migrate, 14081f862989SVitaly Wool .putback_page = z3fold_page_putback, 14091f862989SVitaly Wool }; 14101f862989SVitaly Wool 14119a001fc1SVitaly Wool /***************** 14129a001fc1SVitaly Wool * zpool 14139a001fc1SVitaly Wool ****************/ 14149a001fc1SVitaly Wool 14159a001fc1SVitaly Wool static int z3fold_zpool_evict(struct z3fold_pool *pool, unsigned long handle) 14169a001fc1SVitaly Wool { 14179a001fc1SVitaly Wool if (pool->zpool && pool->zpool_ops && pool->zpool_ops->evict) 14189a001fc1SVitaly Wool return pool->zpool_ops->evict(pool->zpool, handle); 14199a001fc1SVitaly Wool else 14209a001fc1SVitaly Wool return -ENOENT; 14219a001fc1SVitaly Wool } 14229a001fc1SVitaly Wool 14239a001fc1SVitaly Wool static const struct z3fold_ops z3fold_zpool_ops = { 14249a001fc1SVitaly Wool .evict = z3fold_zpool_evict 14259a001fc1SVitaly Wool }; 14269a001fc1SVitaly Wool 14279a001fc1SVitaly Wool static void *z3fold_zpool_create(const char *name, gfp_t gfp, 14289a001fc1SVitaly Wool const struct zpool_ops *zpool_ops, 14299a001fc1SVitaly Wool struct zpool *zpool) 14309a001fc1SVitaly Wool { 14319a001fc1SVitaly Wool struct z3fold_pool *pool; 14329a001fc1SVitaly Wool 1433d30561c5SVitaly Wool pool = z3fold_create_pool(name, gfp, 1434d30561c5SVitaly Wool zpool_ops ? &z3fold_zpool_ops : NULL); 14359a001fc1SVitaly Wool if (pool) { 14369a001fc1SVitaly Wool pool->zpool = zpool; 14379a001fc1SVitaly Wool pool->zpool_ops = zpool_ops; 14389a001fc1SVitaly Wool } 14399a001fc1SVitaly Wool return pool; 14409a001fc1SVitaly Wool } 14419a001fc1SVitaly Wool 14429a001fc1SVitaly Wool static void z3fold_zpool_destroy(void *pool) 14439a001fc1SVitaly Wool { 14449a001fc1SVitaly Wool z3fold_destroy_pool(pool); 14459a001fc1SVitaly Wool } 14469a001fc1SVitaly Wool 14479a001fc1SVitaly Wool static int z3fold_zpool_malloc(void *pool, size_t size, gfp_t gfp, 14489a001fc1SVitaly Wool unsigned long *handle) 14499a001fc1SVitaly Wool { 14509a001fc1SVitaly Wool return z3fold_alloc(pool, size, gfp, handle); 14519a001fc1SVitaly Wool } 14529a001fc1SVitaly Wool static void z3fold_zpool_free(void *pool, unsigned long handle) 14539a001fc1SVitaly Wool { 14549a001fc1SVitaly Wool z3fold_free(pool, handle); 14559a001fc1SVitaly Wool } 14569a001fc1SVitaly Wool 14579a001fc1SVitaly Wool static int z3fold_zpool_shrink(void *pool, unsigned int pages, 14589a001fc1SVitaly Wool unsigned int *reclaimed) 14599a001fc1SVitaly Wool { 14609a001fc1SVitaly Wool unsigned int total = 0; 14619a001fc1SVitaly Wool int ret = -EINVAL; 14629a001fc1SVitaly Wool 14639a001fc1SVitaly Wool while (total < pages) { 14649a001fc1SVitaly Wool ret = z3fold_reclaim_page(pool, 8); 14659a001fc1SVitaly Wool if (ret < 0) 14669a001fc1SVitaly Wool break; 14679a001fc1SVitaly Wool total++; 14689a001fc1SVitaly Wool } 14699a001fc1SVitaly Wool 14709a001fc1SVitaly Wool if (reclaimed) 14719a001fc1SVitaly Wool *reclaimed = total; 14729a001fc1SVitaly Wool 14739a001fc1SVitaly Wool return ret; 14749a001fc1SVitaly Wool } 14759a001fc1SVitaly Wool 14769a001fc1SVitaly Wool static void *z3fold_zpool_map(void *pool, unsigned long handle, 14779a001fc1SVitaly Wool enum zpool_mapmode mm) 14789a001fc1SVitaly Wool { 14799a001fc1SVitaly Wool return z3fold_map(pool, handle); 14809a001fc1SVitaly Wool } 14819a001fc1SVitaly Wool static void z3fold_zpool_unmap(void *pool, unsigned long handle) 14829a001fc1SVitaly Wool { 14839a001fc1SVitaly Wool z3fold_unmap(pool, handle); 14849a001fc1SVitaly Wool } 14859a001fc1SVitaly Wool 14869a001fc1SVitaly Wool static u64 z3fold_zpool_total_size(void *pool) 14879a001fc1SVitaly Wool { 14889a001fc1SVitaly Wool return z3fold_get_pool_size(pool) * PAGE_SIZE; 14899a001fc1SVitaly Wool } 14909a001fc1SVitaly Wool 14919a001fc1SVitaly Wool static struct zpool_driver z3fold_zpool_driver = { 14929a001fc1SVitaly Wool .type = "z3fold", 14939a001fc1SVitaly Wool .owner = THIS_MODULE, 14949a001fc1SVitaly Wool .create = z3fold_zpool_create, 14959a001fc1SVitaly Wool .destroy = z3fold_zpool_destroy, 14969a001fc1SVitaly Wool .malloc = z3fold_zpool_malloc, 14979a001fc1SVitaly Wool .free = z3fold_zpool_free, 14989a001fc1SVitaly Wool .shrink = z3fold_zpool_shrink, 14999a001fc1SVitaly Wool .map = z3fold_zpool_map, 15009a001fc1SVitaly Wool .unmap = z3fold_zpool_unmap, 15019a001fc1SVitaly Wool .total_size = z3fold_zpool_total_size, 15029a001fc1SVitaly Wool }; 15039a001fc1SVitaly Wool 15049a001fc1SVitaly Wool MODULE_ALIAS("zpool-z3fold"); 15059a001fc1SVitaly Wool 15069a001fc1SVitaly Wool static int __init init_z3fold(void) 15079a001fc1SVitaly Wool { 15081f862989SVitaly Wool int ret; 15091f862989SVitaly Wool 1510ede93213SVitaly Wool /* Make sure the z3fold header is not larger than the page size */ 1511ede93213SVitaly Wool BUILD_BUG_ON(ZHDR_SIZE_ALIGNED > PAGE_SIZE); 15121f862989SVitaly Wool ret = z3fold_mount(); 15131f862989SVitaly Wool if (ret) 15141f862989SVitaly Wool return ret; 15151f862989SVitaly Wool 15169a001fc1SVitaly Wool zpool_register_driver(&z3fold_zpool_driver); 15179a001fc1SVitaly Wool 15189a001fc1SVitaly Wool return 0; 15199a001fc1SVitaly Wool } 15209a001fc1SVitaly Wool 15219a001fc1SVitaly Wool static void __exit exit_z3fold(void) 15229a001fc1SVitaly Wool { 15231f862989SVitaly Wool z3fold_unmount(); 15249a001fc1SVitaly Wool zpool_unregister_driver(&z3fold_zpool_driver); 15259a001fc1SVitaly Wool } 15269a001fc1SVitaly Wool 15279a001fc1SVitaly Wool module_init(init_z3fold); 15289a001fc1SVitaly Wool module_exit(exit_z3fold); 15299a001fc1SVitaly Wool 15309a001fc1SVitaly Wool MODULE_LICENSE("GPL"); 15319a001fc1SVitaly Wool MODULE_AUTHOR("Vitaly Wool <vitalywool@gmail.com>"); 15329a001fc1SVitaly Wool MODULE_DESCRIPTION("3-Fold Allocator for Compressed Pages"); 1533