1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 22b281117SSeth Jennings /* 32b281117SSeth Jennings * zswap.c - zswap driver file 42b281117SSeth Jennings * 542c06a0eSJohannes Weiner * zswap is a cache that takes pages that are in the process 62b281117SSeth Jennings * of being swapped out and attempts to compress and store them in a 72b281117SSeth Jennings * RAM-based memory pool. This can result in a significant I/O reduction on 82b281117SSeth Jennings * the swap device and, in the case where decompressing from RAM is faster 92b281117SSeth Jennings * than reading from the swap device, can also improve workload performance. 102b281117SSeth Jennings * 112b281117SSeth Jennings * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 122b281117SSeth Jennings */ 132b281117SSeth Jennings 142b281117SSeth Jennings #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 152b281117SSeth Jennings 162b281117SSeth Jennings #include <linux/module.h> 172b281117SSeth Jennings #include <linux/cpu.h> 182b281117SSeth Jennings #include <linux/highmem.h> 192b281117SSeth Jennings #include <linux/slab.h> 202b281117SSeth Jennings #include <linux/spinlock.h> 212b281117SSeth Jennings #include <linux/types.h> 222b281117SSeth Jennings #include <linux/atomic.h> 232b281117SSeth Jennings #include <linux/rbtree.h> 242b281117SSeth Jennings #include <linux/swap.h> 252b281117SSeth Jennings #include <linux/crypto.h> 261ec3b5feSBarry Song #include <linux/scatterlist.h> 272b281117SSeth Jennings #include <linux/mempool.h> 2812d79d64SDan Streetman #include <linux/zpool.h> 291ec3b5feSBarry Song #include <crypto/acompress.h> 3042c06a0eSJohannes Weiner #include <linux/zswap.h> 312b281117SSeth Jennings #include <linux/mm_types.h> 322b281117SSeth Jennings #include <linux/page-flags.h> 332b281117SSeth Jennings #include <linux/swapops.h> 342b281117SSeth Jennings #include <linux/writeback.h> 352b281117SSeth Jennings #include <linux/pagemap.h> 3645190f01SVitaly Wool #include <linux/workqueue.h> 372b281117SSeth Jennings 38014bb1deSNeilBrown #include "swap.h" 39e0228d59SDomenico Cerasuolo #include "internal.h" 40014bb1deSNeilBrown 412b281117SSeth Jennings /********************************* 422b281117SSeth Jennings * statistics 432b281117SSeth Jennings **********************************/ 4412d79d64SDan Streetman /* Total bytes used by the compressed storage */ 45f6498b77SJohannes Weiner u64 zswap_pool_total_size; 462b281117SSeth Jennings /* The number of compressed pages currently stored in zswap */ 47f6498b77SJohannes Weiner atomic_t zswap_stored_pages = ATOMIC_INIT(0); 48a85f878bSSrividya Desireddy /* The number of same-value filled pages currently stored in zswap */ 49a85f878bSSrividya Desireddy static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0); 502b281117SSeth Jennings 512b281117SSeth Jennings /* 522b281117SSeth Jennings * The statistics below are not protected from concurrent access for 532b281117SSeth Jennings * performance reasons so they may not be a 100% accurate. However, 542b281117SSeth Jennings * they do provide useful information on roughly how many times a 552b281117SSeth Jennings * certain event is occurring. 562b281117SSeth Jennings */ 572b281117SSeth Jennings 582b281117SSeth Jennings /* Pool limit was hit (see zswap_max_pool_percent) */ 592b281117SSeth Jennings static u64 zswap_pool_limit_hit; 602b281117SSeth Jennings /* Pages written back when pool limit was reached */ 612b281117SSeth Jennings static u64 zswap_written_back_pages; 622b281117SSeth Jennings /* Store failed due to a reclaim failure after pool limit was reached */ 632b281117SSeth Jennings static u64 zswap_reject_reclaim_fail; 642b281117SSeth Jennings /* Compressed page was too big for the allocator to (optimally) store */ 652b281117SSeth Jennings static u64 zswap_reject_compress_poor; 662b281117SSeth Jennings /* Store failed because underlying allocator could not get memory */ 672b281117SSeth Jennings static u64 zswap_reject_alloc_fail; 682b281117SSeth Jennings /* Store failed because the entry metadata could not be allocated (rare) */ 692b281117SSeth Jennings static u64 zswap_reject_kmemcache_fail; 702b281117SSeth Jennings /* Duplicate store was encountered (rare) */ 712b281117SSeth Jennings static u64 zswap_duplicate_entry; 722b281117SSeth Jennings 7345190f01SVitaly Wool /* Shrinker work queue */ 7445190f01SVitaly Wool static struct workqueue_struct *shrink_wq; 7545190f01SVitaly Wool /* Pool limit was hit, we need to calm down */ 7645190f01SVitaly Wool static bool zswap_pool_reached_full; 7745190f01SVitaly Wool 782b281117SSeth Jennings /********************************* 792b281117SSeth Jennings * tunables 802b281117SSeth Jennings **********************************/ 81c00ed16aSDan Streetman 82bae21db8SDan Streetman #define ZSWAP_PARAM_UNSET "" 83bae21db8SDan Streetman 84141fdeecSLiu Shixin static int zswap_setup(void); 85141fdeecSLiu Shixin 86bb8b93b5SMaciej S. Szmigiero /* Enable/disable zswap */ 87bb8b93b5SMaciej S. Szmigiero static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); 88d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *, 89d7b028f5SDan Streetman const struct kernel_param *); 9083aed6cdSJoe Perches static const struct kernel_param_ops zswap_enabled_param_ops = { 91d7b028f5SDan Streetman .set = zswap_enabled_param_set, 92d7b028f5SDan Streetman .get = param_get_bool, 93d7b028f5SDan Streetman }; 94d7b028f5SDan Streetman module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); 952b281117SSeth Jennings 9690b0fc26SDan Streetman /* Crypto compressor to use */ 97bb8b93b5SMaciej S. Szmigiero static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 9890b0fc26SDan Streetman static int zswap_compressor_param_set(const char *, 9990b0fc26SDan Streetman const struct kernel_param *); 10083aed6cdSJoe Perches static const struct kernel_param_ops zswap_compressor_param_ops = { 10190b0fc26SDan Streetman .set = zswap_compressor_param_set, 102c99b42c3SDan Streetman .get = param_get_charp, 103c99b42c3SDan Streetman .free = param_free_charp, 10490b0fc26SDan Streetman }; 10590b0fc26SDan Streetman module_param_cb(compressor, &zswap_compressor_param_ops, 106c99b42c3SDan Streetman &zswap_compressor, 0644); 10790b0fc26SDan Streetman 10890b0fc26SDan Streetman /* Compressed storage zpool to use */ 109bb8b93b5SMaciej S. Szmigiero static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 11090b0fc26SDan Streetman static int zswap_zpool_param_set(const char *, const struct kernel_param *); 11183aed6cdSJoe Perches static const struct kernel_param_ops zswap_zpool_param_ops = { 11290b0fc26SDan Streetman .set = zswap_zpool_param_set, 113c99b42c3SDan Streetman .get = param_get_charp, 114c99b42c3SDan Streetman .free = param_free_charp, 11590b0fc26SDan Streetman }; 116c99b42c3SDan Streetman module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 1172b281117SSeth Jennings 1182b281117SSeth Jennings /* The maximum percentage of memory that the compressed pool can occupy */ 1192b281117SSeth Jennings static unsigned int zswap_max_pool_percent = 20; 12090b0fc26SDan Streetman module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 12160105e12SMinchan Kim 12245190f01SVitaly Wool /* The threshold for accepting new pages after the max_pool_percent was hit */ 12345190f01SVitaly Wool static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ 12445190f01SVitaly Wool module_param_named(accept_threshold_percent, zswap_accept_thr_percent, 12545190f01SVitaly Wool uint, 0644); 12645190f01SVitaly Wool 127cb325dddSMaciej S. Szmigiero /* 128cb325dddSMaciej S. Szmigiero * Enable/disable handling same-value filled pages (enabled by default). 129cb325dddSMaciej S. Szmigiero * If disabled every page is considered non-same-value filled. 130cb325dddSMaciej S. Szmigiero */ 131a85f878bSSrividya Desireddy static bool zswap_same_filled_pages_enabled = true; 132a85f878bSSrividya Desireddy module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled, 133a85f878bSSrividya Desireddy bool, 0644); 134a85f878bSSrividya Desireddy 135cb325dddSMaciej S. Szmigiero /* Enable/disable handling non-same-value filled pages (enabled by default) */ 136cb325dddSMaciej S. Szmigiero static bool zswap_non_same_filled_pages_enabled = true; 137cb325dddSMaciej S. Szmigiero module_param_named(non_same_filled_pages_enabled, zswap_non_same_filled_pages_enabled, 138cb325dddSMaciej S. Szmigiero bool, 0644); 139cb325dddSMaciej S. Szmigiero 140b9c91c43SYosry Ahmed static bool zswap_exclusive_loads_enabled = IS_ENABLED( 141b9c91c43SYosry Ahmed CONFIG_ZSWAP_EXCLUSIVE_LOADS_DEFAULT_ON); 142b9c91c43SYosry Ahmed module_param_named(exclusive_loads, zswap_exclusive_loads_enabled, bool, 0644); 143b9c91c43SYosry Ahmed 144b8cf32dcSYosry Ahmed /* Number of zpools in zswap_pool (empirically determined for scalability) */ 145b8cf32dcSYosry Ahmed #define ZSWAP_NR_ZPOOLS 32 146b8cf32dcSYosry Ahmed 1472b281117SSeth Jennings /********************************* 1482b281117SSeth Jennings * data structures 1492b281117SSeth Jennings **********************************/ 150f1c54846SDan Streetman 1511ec3b5feSBarry Song struct crypto_acomp_ctx { 1521ec3b5feSBarry Song struct crypto_acomp *acomp; 1531ec3b5feSBarry Song struct acomp_req *req; 1541ec3b5feSBarry Song struct crypto_wait wait; 1551ec3b5feSBarry Song u8 *dstmem; 1561ec3b5feSBarry Song struct mutex *mutex; 1571ec3b5feSBarry Song }; 1581ec3b5feSBarry Song 159f999f38bSDomenico Cerasuolo /* 160f999f38bSDomenico Cerasuolo * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock. 161f999f38bSDomenico Cerasuolo * The only case where lru_lock is not acquired while holding tree.lock is 162f999f38bSDomenico Cerasuolo * when a zswap_entry is taken off the lru for writeback, in that case it 163f999f38bSDomenico Cerasuolo * needs to be verified that it's still valid in the tree. 164f999f38bSDomenico Cerasuolo */ 165f1c54846SDan Streetman struct zswap_pool { 166b8cf32dcSYosry Ahmed struct zpool *zpools[ZSWAP_NR_ZPOOLS]; 1671ec3b5feSBarry Song struct crypto_acomp_ctx __percpu *acomp_ctx; 168f1c54846SDan Streetman struct kref kref; 169f1c54846SDan Streetman struct list_head list; 17045190f01SVitaly Wool struct work_struct release_work; 17145190f01SVitaly Wool struct work_struct shrink_work; 172cab7a7e5SSebastian Andrzej Siewior struct hlist_node node; 173f1c54846SDan Streetman char tfm_name[CRYPTO_MAX_ALG_NAME]; 174f999f38bSDomenico Cerasuolo struct list_head lru; 175f999f38bSDomenico Cerasuolo spinlock_t lru_lock; 176f1c54846SDan Streetman }; 177f1c54846SDan Streetman 1782b281117SSeth Jennings /* 1792b281117SSeth Jennings * struct zswap_entry 1802b281117SSeth Jennings * 1812b281117SSeth Jennings * This structure contains the metadata for tracking a single compressed 1822b281117SSeth Jennings * page within zswap. 1832b281117SSeth Jennings * 1842b281117SSeth Jennings * rbnode - links the entry into red-black tree for the appropriate swap type 18597157d89SXiu Jianfeng * swpentry - associated swap entry, the offset indexes into the red-black tree 1862b281117SSeth Jennings * refcount - the number of outstanding reference to the entry. This is needed 1872b281117SSeth Jennings * to protect against premature freeing of the entry by code 1886b452516SSeongJae Park * concurrent calls to load, invalidate, and writeback. The lock 1892b281117SSeth Jennings * for the zswap_tree structure that contains the entry must 1902b281117SSeth Jennings * be held while changing the refcount. Since the lock must 1912b281117SSeth Jennings * be held, there is no reason to also make refcount atomic. 1922b281117SSeth Jennings * length - the length in bytes of the compressed page data. Needed during 193f999f38bSDomenico Cerasuolo * decompression. For a same value filled page length is 0, and both 194f999f38bSDomenico Cerasuolo * pool and lru are invalid and must be ignored. 195f1c54846SDan Streetman * pool - the zswap_pool the entry's data is in 196f1c54846SDan Streetman * handle - zpool allocation handle that stores the compressed page data 197a85f878bSSrividya Desireddy * value - value of the same-value filled pages which have same content 19897157d89SXiu Jianfeng * objcg - the obj_cgroup that the compressed memory is charged to 199f999f38bSDomenico Cerasuolo * lru - handle to the pool's lru used to evict pages. 2002b281117SSeth Jennings */ 2012b281117SSeth Jennings struct zswap_entry { 2022b281117SSeth Jennings struct rb_node rbnode; 2030bb48849SDomenico Cerasuolo swp_entry_t swpentry; 2042b281117SSeth Jennings int refcount; 2052b281117SSeth Jennings unsigned int length; 206f1c54846SDan Streetman struct zswap_pool *pool; 207a85f878bSSrividya Desireddy union { 2082b281117SSeth Jennings unsigned long handle; 209a85f878bSSrividya Desireddy unsigned long value; 210a85f878bSSrividya Desireddy }; 211f4840ccfSJohannes Weiner struct obj_cgroup *objcg; 212f999f38bSDomenico Cerasuolo struct list_head lru; 2132b281117SSeth Jennings }; 2142b281117SSeth Jennings 2152b281117SSeth Jennings /* 2162b281117SSeth Jennings * The tree lock in the zswap_tree struct protects a few things: 2172b281117SSeth Jennings * - the rbtree 2182b281117SSeth Jennings * - the refcount field of each entry in the tree 2192b281117SSeth Jennings */ 2202b281117SSeth Jennings struct zswap_tree { 2212b281117SSeth Jennings struct rb_root rbroot; 2222b281117SSeth Jennings spinlock_t lock; 2232b281117SSeth Jennings }; 2242b281117SSeth Jennings 2252b281117SSeth Jennings static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; 2262b281117SSeth Jennings 227f1c54846SDan Streetman /* RCU-protected iteration */ 228f1c54846SDan Streetman static LIST_HEAD(zswap_pools); 229f1c54846SDan Streetman /* protects zswap_pools list modification */ 230f1c54846SDan Streetman static DEFINE_SPINLOCK(zswap_pools_lock); 23132a4e169SDan Streetman /* pool counter to provide unique names to zpool */ 23232a4e169SDan Streetman static atomic_t zswap_pools_count = ATOMIC_INIT(0); 233f1c54846SDan Streetman 2349021ccecSLiu Shixin enum zswap_init_type { 2359021ccecSLiu Shixin ZSWAP_UNINIT, 2369021ccecSLiu Shixin ZSWAP_INIT_SUCCEED, 2379021ccecSLiu Shixin ZSWAP_INIT_FAILED 2389021ccecSLiu Shixin }; 23990b0fc26SDan Streetman 2409021ccecSLiu Shixin static enum zswap_init_type zswap_init_state; 241d7b028f5SDan Streetman 242141fdeecSLiu Shixin /* used to ensure the integrity of initialization */ 243141fdeecSLiu Shixin static DEFINE_MUTEX(zswap_init_lock); 244f1c54846SDan Streetman 245ae3d89a7SDan Streetman /* init completed, but couldn't create the initial pool */ 246ae3d89a7SDan Streetman static bool zswap_has_pool; 247ae3d89a7SDan Streetman 248f1c54846SDan Streetman /********************************* 249f1c54846SDan Streetman * helpers and fwd declarations 250f1c54846SDan Streetman **********************************/ 251f1c54846SDan Streetman 252f1c54846SDan Streetman #define zswap_pool_debug(msg, p) \ 253f1c54846SDan Streetman pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 254b8cf32dcSYosry Ahmed zpool_get_type((p)->zpools[0])) 255f1c54846SDan Streetman 2560bb48849SDomenico Cerasuolo static int zswap_writeback_entry(struct zswap_entry *entry, 257ff9d5ba2SDomenico Cerasuolo struct zswap_tree *tree); 258f1c54846SDan Streetman static int zswap_pool_get(struct zswap_pool *pool); 259f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool); 260f1c54846SDan Streetman 261f1c54846SDan Streetman static bool zswap_is_full(void) 262f1c54846SDan Streetman { 263ca79b0c2SArun KS return totalram_pages() * zswap_max_pool_percent / 100 < 264f1c54846SDan Streetman DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 265f1c54846SDan Streetman } 266f1c54846SDan Streetman 26745190f01SVitaly Wool static bool zswap_can_accept(void) 26845190f01SVitaly Wool { 26945190f01SVitaly Wool return totalram_pages() * zswap_accept_thr_percent / 100 * 27045190f01SVitaly Wool zswap_max_pool_percent / 100 > 27145190f01SVitaly Wool DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 27245190f01SVitaly Wool } 27345190f01SVitaly Wool 274f1c54846SDan Streetman static void zswap_update_total_size(void) 275f1c54846SDan Streetman { 276f1c54846SDan Streetman struct zswap_pool *pool; 277f1c54846SDan Streetman u64 total = 0; 278b8cf32dcSYosry Ahmed int i; 279f1c54846SDan Streetman 280f1c54846SDan Streetman rcu_read_lock(); 281f1c54846SDan Streetman 282f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 283b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) 284b8cf32dcSYosry Ahmed total += zpool_get_total_size(pool->zpools[i]); 285f1c54846SDan Streetman 286f1c54846SDan Streetman rcu_read_unlock(); 287f1c54846SDan Streetman 288f1c54846SDan Streetman zswap_pool_total_size = total; 289f1c54846SDan Streetman } 290f1c54846SDan Streetman 2912b281117SSeth Jennings /********************************* 2922b281117SSeth Jennings * zswap entry functions 2932b281117SSeth Jennings **********************************/ 2942b281117SSeth Jennings static struct kmem_cache *zswap_entry_cache; 2952b281117SSeth Jennings 2962b281117SSeth Jennings static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) 2972b281117SSeth Jennings { 2982b281117SSeth Jennings struct zswap_entry *entry; 2992b281117SSeth Jennings entry = kmem_cache_alloc(zswap_entry_cache, gfp); 3002b281117SSeth Jennings if (!entry) 3012b281117SSeth Jennings return NULL; 3022b281117SSeth Jennings entry->refcount = 1; 3030ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 3042b281117SSeth Jennings return entry; 3052b281117SSeth Jennings } 3062b281117SSeth Jennings 3072b281117SSeth Jennings static void zswap_entry_cache_free(struct zswap_entry *entry) 3082b281117SSeth Jennings { 3092b281117SSeth Jennings kmem_cache_free(zswap_entry_cache, entry); 3102b281117SSeth Jennings } 3112b281117SSeth Jennings 3122b281117SSeth Jennings /********************************* 3132b281117SSeth Jennings * rbtree functions 3142b281117SSeth Jennings **********************************/ 3152b281117SSeth Jennings static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) 3162b281117SSeth Jennings { 3172b281117SSeth Jennings struct rb_node *node = root->rb_node; 3182b281117SSeth Jennings struct zswap_entry *entry; 3190bb48849SDomenico Cerasuolo pgoff_t entry_offset; 3202b281117SSeth Jennings 3212b281117SSeth Jennings while (node) { 3222b281117SSeth Jennings entry = rb_entry(node, struct zswap_entry, rbnode); 3230bb48849SDomenico Cerasuolo entry_offset = swp_offset(entry->swpentry); 3240bb48849SDomenico Cerasuolo if (entry_offset > offset) 3252b281117SSeth Jennings node = node->rb_left; 3260bb48849SDomenico Cerasuolo else if (entry_offset < offset) 3272b281117SSeth Jennings node = node->rb_right; 3282b281117SSeth Jennings else 3292b281117SSeth Jennings return entry; 3302b281117SSeth Jennings } 3312b281117SSeth Jennings return NULL; 3322b281117SSeth Jennings } 3332b281117SSeth Jennings 3342b281117SSeth Jennings /* 3352b281117SSeth Jennings * In the case that a entry with the same offset is found, a pointer to 3362b281117SSeth Jennings * the existing entry is stored in dupentry and the function returns -EEXIST 3372b281117SSeth Jennings */ 3382b281117SSeth Jennings static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, 3392b281117SSeth Jennings struct zswap_entry **dupentry) 3402b281117SSeth Jennings { 3412b281117SSeth Jennings struct rb_node **link = &root->rb_node, *parent = NULL; 3422b281117SSeth Jennings struct zswap_entry *myentry; 3430bb48849SDomenico Cerasuolo pgoff_t myentry_offset, entry_offset = swp_offset(entry->swpentry); 3442b281117SSeth Jennings 3452b281117SSeth Jennings while (*link) { 3462b281117SSeth Jennings parent = *link; 3472b281117SSeth Jennings myentry = rb_entry(parent, struct zswap_entry, rbnode); 3480bb48849SDomenico Cerasuolo myentry_offset = swp_offset(myentry->swpentry); 3490bb48849SDomenico Cerasuolo if (myentry_offset > entry_offset) 3502b281117SSeth Jennings link = &(*link)->rb_left; 3510bb48849SDomenico Cerasuolo else if (myentry_offset < entry_offset) 3522b281117SSeth Jennings link = &(*link)->rb_right; 3532b281117SSeth Jennings else { 3542b281117SSeth Jennings *dupentry = myentry; 3552b281117SSeth Jennings return -EEXIST; 3562b281117SSeth Jennings } 3572b281117SSeth Jennings } 3582b281117SSeth Jennings rb_link_node(&entry->rbnode, parent, link); 3592b281117SSeth Jennings rb_insert_color(&entry->rbnode, root); 3602b281117SSeth Jennings return 0; 3612b281117SSeth Jennings } 3622b281117SSeth Jennings 36318a93707SYosry Ahmed static bool zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) 3640ab0abcfSWeijie Yang { 3650ab0abcfSWeijie Yang if (!RB_EMPTY_NODE(&entry->rbnode)) { 3660ab0abcfSWeijie Yang rb_erase(&entry->rbnode, root); 3670ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 36818a93707SYosry Ahmed return true; 3690ab0abcfSWeijie Yang } 37018a93707SYosry Ahmed return false; 3710ab0abcfSWeijie Yang } 3720ab0abcfSWeijie Yang 373b8cf32dcSYosry Ahmed static struct zpool *zswap_find_zpool(struct zswap_entry *entry) 374b8cf32dcSYosry Ahmed { 375b8cf32dcSYosry Ahmed int i = 0; 376b8cf32dcSYosry Ahmed 377b8cf32dcSYosry Ahmed if (ZSWAP_NR_ZPOOLS > 1) 378b8cf32dcSYosry Ahmed i = hash_ptr(entry, ilog2(ZSWAP_NR_ZPOOLS)); 379b8cf32dcSYosry Ahmed 380b8cf32dcSYosry Ahmed return entry->pool->zpools[i]; 381b8cf32dcSYosry Ahmed } 382b8cf32dcSYosry Ahmed 3830ab0abcfSWeijie Yang /* 38412d79d64SDan Streetman * Carries out the common pattern of freeing and entry's zpool allocation, 3850ab0abcfSWeijie Yang * freeing the entry itself, and decrementing the number of stored pages. 3860ab0abcfSWeijie Yang */ 38760105e12SMinchan Kim static void zswap_free_entry(struct zswap_entry *entry) 3880ab0abcfSWeijie Yang { 389f4840ccfSJohannes Weiner if (entry->objcg) { 390f4840ccfSJohannes Weiner obj_cgroup_uncharge_zswap(entry->objcg, entry->length); 391f4840ccfSJohannes Weiner obj_cgroup_put(entry->objcg); 392f4840ccfSJohannes Weiner } 393a85f878bSSrividya Desireddy if (!entry->length) 394a85f878bSSrividya Desireddy atomic_dec(&zswap_same_filled_pages); 395a85f878bSSrividya Desireddy else { 396f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 397f999f38bSDomenico Cerasuolo list_del(&entry->lru); 398f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 399b8cf32dcSYosry Ahmed zpool_free(zswap_find_zpool(entry), entry->handle); 400f1c54846SDan Streetman zswap_pool_put(entry->pool); 401a85f878bSSrividya Desireddy } 4020ab0abcfSWeijie Yang zswap_entry_cache_free(entry); 4030ab0abcfSWeijie Yang atomic_dec(&zswap_stored_pages); 404f1c54846SDan Streetman zswap_update_total_size(); 4050ab0abcfSWeijie Yang } 4060ab0abcfSWeijie Yang 4070ab0abcfSWeijie Yang /* caller must hold the tree lock */ 4080ab0abcfSWeijie Yang static void zswap_entry_get(struct zswap_entry *entry) 4090ab0abcfSWeijie Yang { 4100ab0abcfSWeijie Yang entry->refcount++; 4110ab0abcfSWeijie Yang } 4120ab0abcfSWeijie Yang 4130ab0abcfSWeijie Yang /* caller must hold the tree lock 4140ab0abcfSWeijie Yang * remove from the tree and free it, if nobody reference the entry 4150ab0abcfSWeijie Yang */ 4160ab0abcfSWeijie Yang static void zswap_entry_put(struct zswap_tree *tree, 4170ab0abcfSWeijie Yang struct zswap_entry *entry) 4180ab0abcfSWeijie Yang { 4190ab0abcfSWeijie Yang int refcount = --entry->refcount; 4200ab0abcfSWeijie Yang 42173108957SJohannes Weiner WARN_ON_ONCE(refcount < 0); 4220ab0abcfSWeijie Yang if (refcount == 0) { 42373108957SJohannes Weiner WARN_ON_ONCE(!RB_EMPTY_NODE(&entry->rbnode)); 42460105e12SMinchan Kim zswap_free_entry(entry); 4250ab0abcfSWeijie Yang } 4260ab0abcfSWeijie Yang } 4270ab0abcfSWeijie Yang 4280ab0abcfSWeijie Yang /* caller must hold the tree lock */ 4290ab0abcfSWeijie Yang static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, 4300ab0abcfSWeijie Yang pgoff_t offset) 4310ab0abcfSWeijie Yang { 432b0c9865fSAlexey Klimov struct zswap_entry *entry; 4330ab0abcfSWeijie Yang 4340ab0abcfSWeijie Yang entry = zswap_rb_search(root, offset); 4350ab0abcfSWeijie Yang if (entry) 4360ab0abcfSWeijie Yang zswap_entry_get(entry); 4370ab0abcfSWeijie Yang 4380ab0abcfSWeijie Yang return entry; 4390ab0abcfSWeijie Yang } 4400ab0abcfSWeijie Yang 4412b281117SSeth Jennings /********************************* 4422b281117SSeth Jennings * per-cpu code 4432b281117SSeth Jennings **********************************/ 4442b281117SSeth Jennings static DEFINE_PER_CPU(u8 *, zswap_dstmem); 4451ec3b5feSBarry Song /* 4461ec3b5feSBarry Song * If users dynamically change the zpool type and compressor at runtime, i.e. 4471ec3b5feSBarry Song * zswap is running, zswap can have more than one zpool on one cpu, but they 4481ec3b5feSBarry Song * are sharing dtsmem. So we need this mutex to be per-cpu. 4491ec3b5feSBarry Song */ 4501ec3b5feSBarry Song static DEFINE_PER_CPU(struct mutex *, zswap_mutex); 4512b281117SSeth Jennings 452ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_prepare(unsigned int cpu) 4532b281117SSeth Jennings { 4541ec3b5feSBarry Song struct mutex *mutex; 4552b281117SSeth Jennings u8 *dst; 4562b281117SSeth Jennings 45772d09633SEric Dumazet dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); 4582b2695f5SMarkus Elfring if (!dst) 459ad7ed770SSebastian Andrzej Siewior return -ENOMEM; 4602b2695f5SMarkus Elfring 4611ec3b5feSBarry Song mutex = kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu)); 4621ec3b5feSBarry Song if (!mutex) { 4631ec3b5feSBarry Song kfree(dst); 4641ec3b5feSBarry Song return -ENOMEM; 4651ec3b5feSBarry Song } 4661ec3b5feSBarry Song 4671ec3b5feSBarry Song mutex_init(mutex); 4682b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = dst; 4691ec3b5feSBarry Song per_cpu(zswap_mutex, cpu) = mutex; 470ad7ed770SSebastian Andrzej Siewior return 0; 471ad7ed770SSebastian Andrzej Siewior } 472ad7ed770SSebastian Andrzej Siewior 473ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_dead(unsigned int cpu) 474ad7ed770SSebastian Andrzej Siewior { 4751ec3b5feSBarry Song struct mutex *mutex; 476ad7ed770SSebastian Andrzej Siewior u8 *dst; 477ad7ed770SSebastian Andrzej Siewior 4781ec3b5feSBarry Song mutex = per_cpu(zswap_mutex, cpu); 4791ec3b5feSBarry Song kfree(mutex); 4801ec3b5feSBarry Song per_cpu(zswap_mutex, cpu) = NULL; 4811ec3b5feSBarry Song 4822b281117SSeth Jennings dst = per_cpu(zswap_dstmem, cpu); 4832b281117SSeth Jennings kfree(dst); 4842b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = NULL; 4852b281117SSeth Jennings 4862b281117SSeth Jennings return 0; 487f1c54846SDan Streetman } 488f1c54846SDan Streetman 489cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 490f1c54846SDan Streetman { 491cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 4921ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 4931ec3b5feSBarry Song struct crypto_acomp *acomp; 4941ec3b5feSBarry Song struct acomp_req *req; 495f1c54846SDan Streetman 4961ec3b5feSBarry Song acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu)); 4971ec3b5feSBarry Song if (IS_ERR(acomp)) { 4981ec3b5feSBarry Song pr_err("could not alloc crypto acomp %s : %ld\n", 4991ec3b5feSBarry Song pool->tfm_name, PTR_ERR(acomp)); 5001ec3b5feSBarry Song return PTR_ERR(acomp); 5011ec3b5feSBarry Song } 5021ec3b5feSBarry Song acomp_ctx->acomp = acomp; 503cab7a7e5SSebastian Andrzej Siewior 5041ec3b5feSBarry Song req = acomp_request_alloc(acomp_ctx->acomp); 5051ec3b5feSBarry Song if (!req) { 5061ec3b5feSBarry Song pr_err("could not alloc crypto acomp_request %s\n", 5071ec3b5feSBarry Song pool->tfm_name); 5081ec3b5feSBarry Song crypto_free_acomp(acomp_ctx->acomp); 509cab7a7e5SSebastian Andrzej Siewior return -ENOMEM; 510f1c54846SDan Streetman } 5111ec3b5feSBarry Song acomp_ctx->req = req; 5121ec3b5feSBarry Song 5131ec3b5feSBarry Song crypto_init_wait(&acomp_ctx->wait); 5141ec3b5feSBarry Song /* 5151ec3b5feSBarry Song * if the backend of acomp is async zip, crypto_req_done() will wakeup 5161ec3b5feSBarry Song * crypto_wait_req(); if the backend of acomp is scomp, the callback 5171ec3b5feSBarry Song * won't be called, crypto_wait_req() will return without blocking. 5181ec3b5feSBarry Song */ 5191ec3b5feSBarry Song acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 5201ec3b5feSBarry Song crypto_req_done, &acomp_ctx->wait); 5211ec3b5feSBarry Song 5221ec3b5feSBarry Song acomp_ctx->mutex = per_cpu(zswap_mutex, cpu); 5231ec3b5feSBarry Song acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu); 5241ec3b5feSBarry Song 525cab7a7e5SSebastian Andrzej Siewior return 0; 526cab7a7e5SSebastian Andrzej Siewior } 527cab7a7e5SSebastian Andrzej Siewior 528cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 529cab7a7e5SSebastian Andrzej Siewior { 530cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 5311ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 532cab7a7e5SSebastian Andrzej Siewior 5331ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx)) { 5341ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx->req)) 5351ec3b5feSBarry Song acomp_request_free(acomp_ctx->req); 5361ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx->acomp)) 5371ec3b5feSBarry Song crypto_free_acomp(acomp_ctx->acomp); 5381ec3b5feSBarry Song } 5391ec3b5feSBarry Song 540f1c54846SDan Streetman return 0; 541f1c54846SDan Streetman } 542f1c54846SDan Streetman 543f1c54846SDan Streetman /********************************* 544f1c54846SDan Streetman * pool functions 545f1c54846SDan Streetman **********************************/ 546f1c54846SDan Streetman 547f1c54846SDan Streetman static struct zswap_pool *__zswap_pool_current(void) 548f1c54846SDan Streetman { 549f1c54846SDan Streetman struct zswap_pool *pool; 550f1c54846SDan Streetman 551f1c54846SDan Streetman pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 552ae3d89a7SDan Streetman WARN_ONCE(!pool && zswap_has_pool, 553ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 554f1c54846SDan Streetman 555f1c54846SDan Streetman return pool; 556f1c54846SDan Streetman } 557f1c54846SDan Streetman 558f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current(void) 559f1c54846SDan Streetman { 560f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 561f1c54846SDan Streetman 562f1c54846SDan Streetman return __zswap_pool_current(); 563f1c54846SDan Streetman } 564f1c54846SDan Streetman 565f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current_get(void) 566f1c54846SDan Streetman { 567f1c54846SDan Streetman struct zswap_pool *pool; 568f1c54846SDan Streetman 569f1c54846SDan Streetman rcu_read_lock(); 570f1c54846SDan Streetman 571f1c54846SDan Streetman pool = __zswap_pool_current(); 572ae3d89a7SDan Streetman if (!zswap_pool_get(pool)) 573f1c54846SDan Streetman pool = NULL; 574f1c54846SDan Streetman 575f1c54846SDan Streetman rcu_read_unlock(); 576f1c54846SDan Streetman 577f1c54846SDan Streetman return pool; 578f1c54846SDan Streetman } 579f1c54846SDan Streetman 580f1c54846SDan Streetman static struct zswap_pool *zswap_pool_last_get(void) 581f1c54846SDan Streetman { 582f1c54846SDan Streetman struct zswap_pool *pool, *last = NULL; 583f1c54846SDan Streetman 584f1c54846SDan Streetman rcu_read_lock(); 585f1c54846SDan Streetman 586f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 587f1c54846SDan Streetman last = pool; 588ae3d89a7SDan Streetman WARN_ONCE(!last && zswap_has_pool, 589ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 590ae3d89a7SDan Streetman if (!zswap_pool_get(last)) 591f1c54846SDan Streetman last = NULL; 592f1c54846SDan Streetman 593f1c54846SDan Streetman rcu_read_unlock(); 594f1c54846SDan Streetman 595f1c54846SDan Streetman return last; 596f1c54846SDan Streetman } 597f1c54846SDan Streetman 5988bc8b228SDan Streetman /* type and compressor must be null-terminated */ 599f1c54846SDan Streetman static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 600f1c54846SDan Streetman { 601f1c54846SDan Streetman struct zswap_pool *pool; 602f1c54846SDan Streetman 603f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 604f1c54846SDan Streetman 605f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) { 6068bc8b228SDan Streetman if (strcmp(pool->tfm_name, compressor)) 607f1c54846SDan Streetman continue; 608b8cf32dcSYosry Ahmed /* all zpools share the same type */ 609b8cf32dcSYosry Ahmed if (strcmp(zpool_get_type(pool->zpools[0]), type)) 610f1c54846SDan Streetman continue; 611f1c54846SDan Streetman /* if we can't get it, it's about to be destroyed */ 612f1c54846SDan Streetman if (!zswap_pool_get(pool)) 613f1c54846SDan Streetman continue; 614f1c54846SDan Streetman return pool; 615f1c54846SDan Streetman } 616f1c54846SDan Streetman 617f1c54846SDan Streetman return NULL; 618f1c54846SDan Streetman } 619f1c54846SDan Streetman 62018a93707SYosry Ahmed /* 62118a93707SYosry Ahmed * If the entry is still valid in the tree, drop the initial ref and remove it 62218a93707SYosry Ahmed * from the tree. This function must be called with an additional ref held, 62318a93707SYosry Ahmed * otherwise it may race with another invalidation freeing the entry. 62418a93707SYosry Ahmed */ 625418fd29dSDomenico Cerasuolo static void zswap_invalidate_entry(struct zswap_tree *tree, 626418fd29dSDomenico Cerasuolo struct zswap_entry *entry) 627418fd29dSDomenico Cerasuolo { 62818a93707SYosry Ahmed if (zswap_rb_erase(&tree->rbroot, entry)) 629418fd29dSDomenico Cerasuolo zswap_entry_put(tree, entry); 630418fd29dSDomenico Cerasuolo } 631418fd29dSDomenico Cerasuolo 632f999f38bSDomenico Cerasuolo static int zswap_reclaim_entry(struct zswap_pool *pool) 633f999f38bSDomenico Cerasuolo { 634f999f38bSDomenico Cerasuolo struct zswap_entry *entry; 635f999f38bSDomenico Cerasuolo struct zswap_tree *tree; 636f999f38bSDomenico Cerasuolo pgoff_t swpoffset; 637f999f38bSDomenico Cerasuolo int ret; 638f999f38bSDomenico Cerasuolo 639f999f38bSDomenico Cerasuolo /* Get an entry off the LRU */ 640f999f38bSDomenico Cerasuolo spin_lock(&pool->lru_lock); 641f999f38bSDomenico Cerasuolo if (list_empty(&pool->lru)) { 642f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 643f999f38bSDomenico Cerasuolo return -EINVAL; 644f999f38bSDomenico Cerasuolo } 645f999f38bSDomenico Cerasuolo entry = list_last_entry(&pool->lru, struct zswap_entry, lru); 646f999f38bSDomenico Cerasuolo list_del_init(&entry->lru); 647f999f38bSDomenico Cerasuolo /* 648f999f38bSDomenico Cerasuolo * Once the lru lock is dropped, the entry might get freed. The 649f999f38bSDomenico Cerasuolo * swpoffset is copied to the stack, and entry isn't deref'd again 650f999f38bSDomenico Cerasuolo * until the entry is verified to still be alive in the tree. 651f999f38bSDomenico Cerasuolo */ 6520bb48849SDomenico Cerasuolo swpoffset = swp_offset(entry->swpentry); 6530bb48849SDomenico Cerasuolo tree = zswap_trees[swp_type(entry->swpentry)]; 654f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 655f999f38bSDomenico Cerasuolo 656f999f38bSDomenico Cerasuolo /* Check for invalidate() race */ 657f999f38bSDomenico Cerasuolo spin_lock(&tree->lock); 658f999f38bSDomenico Cerasuolo if (entry != zswap_rb_search(&tree->rbroot, swpoffset)) { 659f999f38bSDomenico Cerasuolo ret = -EAGAIN; 660f999f38bSDomenico Cerasuolo goto unlock; 661f999f38bSDomenico Cerasuolo } 662f999f38bSDomenico Cerasuolo /* Hold a reference to prevent a free during writeback */ 663f999f38bSDomenico Cerasuolo zswap_entry_get(entry); 664f999f38bSDomenico Cerasuolo spin_unlock(&tree->lock); 665f999f38bSDomenico Cerasuolo 6660bb48849SDomenico Cerasuolo ret = zswap_writeback_entry(entry, tree); 667f999f38bSDomenico Cerasuolo 668f999f38bSDomenico Cerasuolo spin_lock(&tree->lock); 669f999f38bSDomenico Cerasuolo if (ret) { 670f999f38bSDomenico Cerasuolo /* Writeback failed, put entry back on LRU */ 671f999f38bSDomenico Cerasuolo spin_lock(&pool->lru_lock); 672f999f38bSDomenico Cerasuolo list_move(&entry->lru, &pool->lru); 673f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 674ff9d5ba2SDomenico Cerasuolo goto put_unlock; 675f999f38bSDomenico Cerasuolo } 676f999f38bSDomenico Cerasuolo 677418fd29dSDomenico Cerasuolo /* 678418fd29dSDomenico Cerasuolo * Writeback started successfully, the page now belongs to the 679418fd29dSDomenico Cerasuolo * swapcache. Drop the entry from zswap - unless invalidate already 680418fd29dSDomenico Cerasuolo * took it out while we had the tree->lock released for IO. 681418fd29dSDomenico Cerasuolo */ 682418fd29dSDomenico Cerasuolo zswap_invalidate_entry(tree, entry); 683ff9d5ba2SDomenico Cerasuolo 684ff9d5ba2SDomenico Cerasuolo put_unlock: 685f999f38bSDomenico Cerasuolo /* Drop local reference */ 686f999f38bSDomenico Cerasuolo zswap_entry_put(tree, entry); 687f999f38bSDomenico Cerasuolo unlock: 688f999f38bSDomenico Cerasuolo spin_unlock(&tree->lock); 689f999f38bSDomenico Cerasuolo return ret ? -EAGAIN : 0; 690f999f38bSDomenico Cerasuolo } 691f999f38bSDomenico Cerasuolo 69245190f01SVitaly Wool static void shrink_worker(struct work_struct *w) 69345190f01SVitaly Wool { 69445190f01SVitaly Wool struct zswap_pool *pool = container_of(w, typeof(*pool), 69545190f01SVitaly Wool shrink_work); 696e0228d59SDomenico Cerasuolo int ret, failures = 0; 69745190f01SVitaly Wool 698e0228d59SDomenico Cerasuolo do { 699f999f38bSDomenico Cerasuolo ret = zswap_reclaim_entry(pool); 700e0228d59SDomenico Cerasuolo if (ret) { 70145190f01SVitaly Wool zswap_reject_reclaim_fail++; 702e0228d59SDomenico Cerasuolo if (ret != -EAGAIN) 703e0228d59SDomenico Cerasuolo break; 704e0228d59SDomenico Cerasuolo if (++failures == MAX_RECLAIM_RETRIES) 705e0228d59SDomenico Cerasuolo break; 706e0228d59SDomenico Cerasuolo } 707e0228d59SDomenico Cerasuolo cond_resched(); 708e0228d59SDomenico Cerasuolo } while (!zswap_can_accept()); 70945190f01SVitaly Wool zswap_pool_put(pool); 71045190f01SVitaly Wool } 71145190f01SVitaly Wool 712f1c54846SDan Streetman static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 713f1c54846SDan Streetman { 714b8cf32dcSYosry Ahmed int i; 715f1c54846SDan Streetman struct zswap_pool *pool; 71632a4e169SDan Streetman char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 717d0164adcSMel Gorman gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 718cab7a7e5SSebastian Andrzej Siewior int ret; 719f1c54846SDan Streetman 720bae21db8SDan Streetman if (!zswap_has_pool) { 721bae21db8SDan Streetman /* if either are unset, pool initialization failed, and we 722bae21db8SDan Streetman * need both params to be set correctly before trying to 723bae21db8SDan Streetman * create a pool. 724bae21db8SDan Streetman */ 725bae21db8SDan Streetman if (!strcmp(type, ZSWAP_PARAM_UNSET)) 726bae21db8SDan Streetman return NULL; 727bae21db8SDan Streetman if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 728bae21db8SDan Streetman return NULL; 729bae21db8SDan Streetman } 730bae21db8SDan Streetman 731f1c54846SDan Streetman pool = kzalloc(sizeof(*pool), GFP_KERNEL); 732f4ae0ce0SMarkus Elfring if (!pool) 733f1c54846SDan Streetman return NULL; 734f1c54846SDan Streetman 735b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) { 73632a4e169SDan Streetman /* unique name for each pool specifically required by zsmalloc */ 737b8cf32dcSYosry Ahmed snprintf(name, 38, "zswap%x", 738b8cf32dcSYosry Ahmed atomic_inc_return(&zswap_pools_count)); 73932a4e169SDan Streetman 740b8cf32dcSYosry Ahmed pool->zpools[i] = zpool_create_pool(type, name, gfp); 741b8cf32dcSYosry Ahmed if (!pool->zpools[i]) { 742f1c54846SDan Streetman pr_err("%s zpool not available\n", type); 743f1c54846SDan Streetman goto error; 744f1c54846SDan Streetman } 745b8cf32dcSYosry Ahmed } 746b8cf32dcSYosry Ahmed pr_debug("using %s zpool\n", zpool_get_type(pool->zpools[0])); 747f1c54846SDan Streetman 74879cd4202SZhiyuan Dai strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 7491ec3b5feSBarry Song 7501ec3b5feSBarry Song pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx); 7511ec3b5feSBarry Song if (!pool->acomp_ctx) { 752f1c54846SDan Streetman pr_err("percpu alloc failed\n"); 753f1c54846SDan Streetman goto error; 754f1c54846SDan Streetman } 755f1c54846SDan Streetman 756cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, 757cab7a7e5SSebastian Andrzej Siewior &pool->node); 758cab7a7e5SSebastian Andrzej Siewior if (ret) 759f1c54846SDan Streetman goto error; 760f1c54846SDan Streetman pr_debug("using %s compressor\n", pool->tfm_name); 761f1c54846SDan Streetman 762f1c54846SDan Streetman /* being the current pool takes 1 ref; this func expects the 763f1c54846SDan Streetman * caller to always add the new pool as the current pool 764f1c54846SDan Streetman */ 765f1c54846SDan Streetman kref_init(&pool->kref); 766f1c54846SDan Streetman INIT_LIST_HEAD(&pool->list); 767f999f38bSDomenico Cerasuolo INIT_LIST_HEAD(&pool->lru); 768f999f38bSDomenico Cerasuolo spin_lock_init(&pool->lru_lock); 76945190f01SVitaly Wool INIT_WORK(&pool->shrink_work, shrink_worker); 770f1c54846SDan Streetman 771f1c54846SDan Streetman zswap_pool_debug("created", pool); 772f1c54846SDan Streetman 773f1c54846SDan Streetman return pool; 774f1c54846SDan Streetman 775f1c54846SDan Streetman error: 7761ec3b5feSBarry Song if (pool->acomp_ctx) 7771ec3b5feSBarry Song free_percpu(pool->acomp_ctx); 778b8cf32dcSYosry Ahmed while (i--) 779b8cf32dcSYosry Ahmed zpool_destroy_pool(pool->zpools[i]); 780f1c54846SDan Streetman kfree(pool); 781f1c54846SDan Streetman return NULL; 782f1c54846SDan Streetman } 783f1c54846SDan Streetman 784141fdeecSLiu Shixin static struct zswap_pool *__zswap_pool_create_fallback(void) 785f1c54846SDan Streetman { 786bae21db8SDan Streetman bool has_comp, has_zpool; 787bae21db8SDan Streetman 7881ec3b5feSBarry Song has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 789bb8b93b5SMaciej S. Szmigiero if (!has_comp && strcmp(zswap_compressor, 790bb8b93b5SMaciej S. Szmigiero CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 791f1c54846SDan Streetman pr_err("compressor %s not available, using default %s\n", 792bb8b93b5SMaciej S. Szmigiero zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); 793c99b42c3SDan Streetman param_free_charp(&zswap_compressor); 794bb8b93b5SMaciej S. Szmigiero zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 7951ec3b5feSBarry Song has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 796f1c54846SDan Streetman } 797bae21db8SDan Streetman if (!has_comp) { 798bae21db8SDan Streetman pr_err("default compressor %s not available\n", 799bae21db8SDan Streetman zswap_compressor); 800bae21db8SDan Streetman param_free_charp(&zswap_compressor); 801bae21db8SDan Streetman zswap_compressor = ZSWAP_PARAM_UNSET; 802c99b42c3SDan Streetman } 803bae21db8SDan Streetman 804bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 805bb8b93b5SMaciej S. Szmigiero if (!has_zpool && strcmp(zswap_zpool_type, 806bb8b93b5SMaciej S. Szmigiero CONFIG_ZSWAP_ZPOOL_DEFAULT)) { 807f1c54846SDan Streetman pr_err("zpool %s not available, using default %s\n", 808bb8b93b5SMaciej S. Szmigiero zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); 809c99b42c3SDan Streetman param_free_charp(&zswap_zpool_type); 810bb8b93b5SMaciej S. Szmigiero zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 811bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 812f1c54846SDan Streetman } 813bae21db8SDan Streetman if (!has_zpool) { 814bae21db8SDan Streetman pr_err("default zpool %s not available\n", 815bae21db8SDan Streetman zswap_zpool_type); 816bae21db8SDan Streetman param_free_charp(&zswap_zpool_type); 817bae21db8SDan Streetman zswap_zpool_type = ZSWAP_PARAM_UNSET; 818bae21db8SDan Streetman } 819bae21db8SDan Streetman 820bae21db8SDan Streetman if (!has_comp || !has_zpool) 821bae21db8SDan Streetman return NULL; 822f1c54846SDan Streetman 823f1c54846SDan Streetman return zswap_pool_create(zswap_zpool_type, zswap_compressor); 824f1c54846SDan Streetman } 825f1c54846SDan Streetman 826f1c54846SDan Streetman static void zswap_pool_destroy(struct zswap_pool *pool) 827f1c54846SDan Streetman { 828b8cf32dcSYosry Ahmed int i; 829b8cf32dcSYosry Ahmed 830f1c54846SDan Streetman zswap_pool_debug("destroying", pool); 831f1c54846SDan Streetman 832cab7a7e5SSebastian Andrzej Siewior cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 8331ec3b5feSBarry Song free_percpu(pool->acomp_ctx); 834b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) 835b8cf32dcSYosry Ahmed zpool_destroy_pool(pool->zpools[i]); 836f1c54846SDan Streetman kfree(pool); 837f1c54846SDan Streetman } 838f1c54846SDan Streetman 839f1c54846SDan Streetman static int __must_check zswap_pool_get(struct zswap_pool *pool) 840f1c54846SDan Streetman { 841ae3d89a7SDan Streetman if (!pool) 842ae3d89a7SDan Streetman return 0; 843ae3d89a7SDan Streetman 844f1c54846SDan Streetman return kref_get_unless_zero(&pool->kref); 845f1c54846SDan Streetman } 846f1c54846SDan Streetman 847200867afSDan Streetman static void __zswap_pool_release(struct work_struct *work) 848f1c54846SDan Streetman { 84945190f01SVitaly Wool struct zswap_pool *pool = container_of(work, typeof(*pool), 85045190f01SVitaly Wool release_work); 851200867afSDan Streetman 852200867afSDan Streetman synchronize_rcu(); 853f1c54846SDan Streetman 854f1c54846SDan Streetman /* nobody should have been able to get a kref... */ 855f1c54846SDan Streetman WARN_ON(kref_get_unless_zero(&pool->kref)); 856f1c54846SDan Streetman 857f1c54846SDan Streetman /* pool is now off zswap_pools list and has no references. */ 858f1c54846SDan Streetman zswap_pool_destroy(pool); 859f1c54846SDan Streetman } 860f1c54846SDan Streetman 861f1c54846SDan Streetman static void __zswap_pool_empty(struct kref *kref) 862f1c54846SDan Streetman { 863f1c54846SDan Streetman struct zswap_pool *pool; 864f1c54846SDan Streetman 865f1c54846SDan Streetman pool = container_of(kref, typeof(*pool), kref); 866f1c54846SDan Streetman 867f1c54846SDan Streetman spin_lock(&zswap_pools_lock); 868f1c54846SDan Streetman 869f1c54846SDan Streetman WARN_ON(pool == zswap_pool_current()); 870f1c54846SDan Streetman 871f1c54846SDan Streetman list_del_rcu(&pool->list); 872200867afSDan Streetman 87345190f01SVitaly Wool INIT_WORK(&pool->release_work, __zswap_pool_release); 87445190f01SVitaly Wool schedule_work(&pool->release_work); 875f1c54846SDan Streetman 876f1c54846SDan Streetman spin_unlock(&zswap_pools_lock); 877f1c54846SDan Streetman } 878f1c54846SDan Streetman 879f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool) 880f1c54846SDan Streetman { 881f1c54846SDan Streetman kref_put(&pool->kref, __zswap_pool_empty); 8822b281117SSeth Jennings } 8832b281117SSeth Jennings 8842b281117SSeth Jennings /********************************* 88590b0fc26SDan Streetman * param callbacks 88690b0fc26SDan Streetman **********************************/ 88790b0fc26SDan Streetman 888141fdeecSLiu Shixin static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) 889141fdeecSLiu Shixin { 890141fdeecSLiu Shixin /* no change required */ 891141fdeecSLiu Shixin if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 892141fdeecSLiu Shixin return false; 893141fdeecSLiu Shixin return true; 894141fdeecSLiu Shixin } 895141fdeecSLiu Shixin 896c99b42c3SDan Streetman /* val must be a null-terminated string */ 89790b0fc26SDan Streetman static int __zswap_param_set(const char *val, const struct kernel_param *kp, 89890b0fc26SDan Streetman char *type, char *compressor) 89990b0fc26SDan Streetman { 90090b0fc26SDan Streetman struct zswap_pool *pool, *put_pool = NULL; 901c99b42c3SDan Streetman char *s = strstrip((char *)val); 902141fdeecSLiu Shixin int ret = 0; 903141fdeecSLiu Shixin bool new_pool = false; 90490b0fc26SDan Streetman 905141fdeecSLiu Shixin mutex_lock(&zswap_init_lock); 9069021ccecSLiu Shixin switch (zswap_init_state) { 9079021ccecSLiu Shixin case ZSWAP_UNINIT: 90890b0fc26SDan Streetman /* if this is load-time (pre-init) param setting, 90990b0fc26SDan Streetman * don't create a pool; that's done during init. 91090b0fc26SDan Streetman */ 911141fdeecSLiu Shixin ret = param_set_charp(s, kp); 912141fdeecSLiu Shixin break; 9139021ccecSLiu Shixin case ZSWAP_INIT_SUCCEED: 914141fdeecSLiu Shixin new_pool = zswap_pool_changed(s, kp); 9159021ccecSLiu Shixin break; 9169021ccecSLiu Shixin case ZSWAP_INIT_FAILED: 9179021ccecSLiu Shixin pr_err("can't set param, initialization failed\n"); 918141fdeecSLiu Shixin ret = -ENODEV; 9199021ccecSLiu Shixin } 920141fdeecSLiu Shixin mutex_unlock(&zswap_init_lock); 921141fdeecSLiu Shixin 922141fdeecSLiu Shixin /* no need to create a new pool, return directly */ 923141fdeecSLiu Shixin if (!new_pool) 924141fdeecSLiu Shixin return ret; 92590b0fc26SDan Streetman 92690b0fc26SDan Streetman if (!type) { 927c99b42c3SDan Streetman if (!zpool_has_pool(s)) { 928c99b42c3SDan Streetman pr_err("zpool %s not available\n", s); 929c99b42c3SDan Streetman return -ENOENT; 930c99b42c3SDan Streetman } 93190b0fc26SDan Streetman type = s; 93290b0fc26SDan Streetman } else if (!compressor) { 9331ec3b5feSBarry Song if (!crypto_has_acomp(s, 0, 0)) { 934c99b42c3SDan Streetman pr_err("compressor %s not available\n", s); 93590b0fc26SDan Streetman return -ENOENT; 93690b0fc26SDan Streetman } 937c99b42c3SDan Streetman compressor = s; 938c99b42c3SDan Streetman } else { 939c99b42c3SDan Streetman WARN_ON(1); 940c99b42c3SDan Streetman return -EINVAL; 94190b0fc26SDan Streetman } 94290b0fc26SDan Streetman 94390b0fc26SDan Streetman spin_lock(&zswap_pools_lock); 94490b0fc26SDan Streetman 94590b0fc26SDan Streetman pool = zswap_pool_find_get(type, compressor); 94690b0fc26SDan Streetman if (pool) { 94790b0fc26SDan Streetman zswap_pool_debug("using existing", pool); 948fd5bb66cSDan Streetman WARN_ON(pool == zswap_pool_current()); 94990b0fc26SDan Streetman list_del_rcu(&pool->list); 95090b0fc26SDan Streetman } 95190b0fc26SDan Streetman 952fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 953fd5bb66cSDan Streetman 954fd5bb66cSDan Streetman if (!pool) 955fd5bb66cSDan Streetman pool = zswap_pool_create(type, compressor); 956fd5bb66cSDan Streetman 95790b0fc26SDan Streetman if (pool) 958c99b42c3SDan Streetman ret = param_set_charp(s, kp); 95990b0fc26SDan Streetman else 96090b0fc26SDan Streetman ret = -EINVAL; 96190b0fc26SDan Streetman 962fd5bb66cSDan Streetman spin_lock(&zswap_pools_lock); 963fd5bb66cSDan Streetman 96490b0fc26SDan Streetman if (!ret) { 96590b0fc26SDan Streetman put_pool = zswap_pool_current(); 96690b0fc26SDan Streetman list_add_rcu(&pool->list, &zswap_pools); 967ae3d89a7SDan Streetman zswap_has_pool = true; 96890b0fc26SDan Streetman } else if (pool) { 96990b0fc26SDan Streetman /* add the possibly pre-existing pool to the end of the pools 97090b0fc26SDan Streetman * list; if it's new (and empty) then it'll be removed and 97190b0fc26SDan Streetman * destroyed by the put after we drop the lock 97290b0fc26SDan Streetman */ 97390b0fc26SDan Streetman list_add_tail_rcu(&pool->list, &zswap_pools); 97490b0fc26SDan Streetman put_pool = pool; 975fd5bb66cSDan Streetman } 976fd5bb66cSDan Streetman 977fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 978fd5bb66cSDan Streetman 979fd5bb66cSDan Streetman if (!zswap_has_pool && !pool) { 980ae3d89a7SDan Streetman /* if initial pool creation failed, and this pool creation also 981ae3d89a7SDan Streetman * failed, maybe both compressor and zpool params were bad. 982ae3d89a7SDan Streetman * Allow changing this param, so pool creation will succeed 983ae3d89a7SDan Streetman * when the other param is changed. We already verified this 9841ec3b5feSBarry Song * param is ok in the zpool_has_pool() or crypto_has_acomp() 985ae3d89a7SDan Streetman * checks above. 986ae3d89a7SDan Streetman */ 987ae3d89a7SDan Streetman ret = param_set_charp(s, kp); 98890b0fc26SDan Streetman } 98990b0fc26SDan Streetman 99090b0fc26SDan Streetman /* drop the ref from either the old current pool, 99190b0fc26SDan Streetman * or the new pool we failed to add 99290b0fc26SDan Streetman */ 99390b0fc26SDan Streetman if (put_pool) 99490b0fc26SDan Streetman zswap_pool_put(put_pool); 99590b0fc26SDan Streetman 99690b0fc26SDan Streetman return ret; 99790b0fc26SDan Streetman } 99890b0fc26SDan Streetman 99990b0fc26SDan Streetman static int zswap_compressor_param_set(const char *val, 100090b0fc26SDan Streetman const struct kernel_param *kp) 100190b0fc26SDan Streetman { 100290b0fc26SDan Streetman return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 100390b0fc26SDan Streetman } 100490b0fc26SDan Streetman 100590b0fc26SDan Streetman static int zswap_zpool_param_set(const char *val, 100690b0fc26SDan Streetman const struct kernel_param *kp) 100790b0fc26SDan Streetman { 100890b0fc26SDan Streetman return __zswap_param_set(val, kp, NULL, zswap_compressor); 100990b0fc26SDan Streetman } 101090b0fc26SDan Streetman 1011d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *val, 1012d7b028f5SDan Streetman const struct kernel_param *kp) 1013d7b028f5SDan Streetman { 1014141fdeecSLiu Shixin int ret = -ENODEV; 1015d7b028f5SDan Streetman 1016141fdeecSLiu Shixin /* if this is load-time (pre-init) param setting, only set param. */ 1017141fdeecSLiu Shixin if (system_state != SYSTEM_RUNNING) 1018d7b028f5SDan Streetman return param_set_bool(val, kp); 1019141fdeecSLiu Shixin 1020141fdeecSLiu Shixin mutex_lock(&zswap_init_lock); 10219021ccecSLiu Shixin switch (zswap_init_state) { 10229021ccecSLiu Shixin case ZSWAP_UNINIT: 1023141fdeecSLiu Shixin if (zswap_setup()) 1024141fdeecSLiu Shixin break; 1025141fdeecSLiu Shixin fallthrough; 10269021ccecSLiu Shixin case ZSWAP_INIT_SUCCEED: 1027141fdeecSLiu Shixin if (!zswap_has_pool) 10289021ccecSLiu Shixin pr_err("can't enable, no pool configured\n"); 1029141fdeecSLiu Shixin else 1030141fdeecSLiu Shixin ret = param_set_bool(val, kp); 1031141fdeecSLiu Shixin break; 10329021ccecSLiu Shixin case ZSWAP_INIT_FAILED: 1033d7b028f5SDan Streetman pr_err("can't enable, initialization failed\n"); 1034d7b028f5SDan Streetman } 1035141fdeecSLiu Shixin mutex_unlock(&zswap_init_lock); 1036141fdeecSLiu Shixin 1037141fdeecSLiu Shixin return ret; 1038d7b028f5SDan Streetman } 1039d7b028f5SDan Streetman 104090b0fc26SDan Streetman /********************************* 10412b281117SSeth Jennings * writeback code 10422b281117SSeth Jennings **********************************/ 10432b281117SSeth Jennings /* 10442b281117SSeth Jennings * Attempts to free an entry by adding a page to the swap cache, 10452b281117SSeth Jennings * decompressing the entry data into the page, and issuing a 10462b281117SSeth Jennings * bio write to write the page back to the swap device. 10472b281117SSeth Jennings * 10482b281117SSeth Jennings * This can be thought of as a "resumed writeback" of the page 10492b281117SSeth Jennings * to the swap device. We are basically resuming the same swap 105042c06a0eSJohannes Weiner * writeback path that was intercepted with the zswap_store() 10512b281117SSeth Jennings * in the first place. After the page has been decompressed into 10522b281117SSeth Jennings * the swap cache, the compressed version stored by zswap can be 10532b281117SSeth Jennings * freed. 10542b281117SSeth Jennings */ 10550bb48849SDomenico Cerasuolo static int zswap_writeback_entry(struct zswap_entry *entry, 1056ff9d5ba2SDomenico Cerasuolo struct zswap_tree *tree) 10572b281117SSeth Jennings { 10580bb48849SDomenico Cerasuolo swp_entry_t swpentry = entry->swpentry; 10592b281117SSeth Jennings struct page *page; 10601ec3b5feSBarry Song struct scatterlist input, output; 10611ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1062b8cf32dcSYosry Ahmed struct zpool *pool = zswap_find_zpool(entry); 106398804a94SJohannes Weiner bool page_was_allocated; 1064fc6697a8STian Tao u8 *src, *tmp = NULL; 10652b281117SSeth Jennings unsigned int dlen; 10660ab0abcfSWeijie Yang int ret; 10672b281117SSeth Jennings struct writeback_control wbc = { 10682b281117SSeth Jennings .sync_mode = WB_SYNC_NONE, 10692b281117SSeth Jennings }; 10702b281117SSeth Jennings 1071fc6697a8STian Tao if (!zpool_can_sleep_mapped(pool)) { 10728d9b6370SSergey Senozhatsky tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); 1073fc6697a8STian Tao if (!tmp) 1074fc6697a8STian Tao return -ENOMEM; 1075fc6697a8STian Tao } 1076fc6697a8STian Tao 10772b281117SSeth Jennings /* try to allocate swap cache page */ 107898804a94SJohannes Weiner page = __read_swap_cache_async(swpentry, GFP_KERNEL, NULL, 0, 107998804a94SJohannes Weiner &page_was_allocated); 108098804a94SJohannes Weiner if (!page) { 10812b281117SSeth Jennings ret = -ENOMEM; 10822b281117SSeth Jennings goto fail; 108398804a94SJohannes Weiner } 10842b281117SSeth Jennings 108598804a94SJohannes Weiner /* Found an existing page, we raced with load/swapin */ 108698804a94SJohannes Weiner if (!page_was_allocated) { 108709cbfeafSKirill A. Shutemov put_page(page); 10882b281117SSeth Jennings ret = -EEXIST; 10892b281117SSeth Jennings goto fail; 109098804a94SJohannes Weiner } 10912b281117SSeth Jennings 109204fc7816SDomenico Cerasuolo /* 109398804a94SJohannes Weiner * Page is locked, and the swapcache is now secured against 109498804a94SJohannes Weiner * concurrent swapping to and from the slot. Verify that the 109598804a94SJohannes Weiner * swap entry hasn't been invalidated and recycled behind our 109698804a94SJohannes Weiner * backs (our zswap_entry reference doesn't prevent that), to 109798804a94SJohannes Weiner * avoid overwriting a new swap page with old compressed data. 109804fc7816SDomenico Cerasuolo */ 109904fc7816SDomenico Cerasuolo spin_lock(&tree->lock); 11000bb48849SDomenico Cerasuolo if (zswap_rb_search(&tree->rbroot, swp_offset(entry->swpentry)) != entry) { 110104fc7816SDomenico Cerasuolo spin_unlock(&tree->lock); 110204fc7816SDomenico Cerasuolo delete_from_swap_cache(page_folio(page)); 110304fc7816SDomenico Cerasuolo ret = -ENOMEM; 110404fc7816SDomenico Cerasuolo goto fail; 110504fc7816SDomenico Cerasuolo } 110604fc7816SDomenico Cerasuolo spin_unlock(&tree->lock); 110704fc7816SDomenico Cerasuolo 11082b281117SSeth Jennings /* decompress */ 11091ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 11102b281117SSeth Jennings dlen = PAGE_SIZE; 1111fc6697a8STian Tao 11120bb48849SDomenico Cerasuolo src = zpool_map_handle(pool, entry->handle, ZPOOL_MM_RO); 11136b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) { 11146b3379e8SJohannes Weiner memcpy(tmp, src, entry->length); 11156b3379e8SJohannes Weiner src = tmp; 1116ff9d5ba2SDomenico Cerasuolo zpool_unmap_handle(pool, entry->handle); 11176b3379e8SJohannes Weiner } 11186b3379e8SJohannes Weiner 11191ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 11201ec3b5feSBarry Song sg_init_one(&input, src, entry->length); 11211ec3b5feSBarry Song sg_init_table(&output, 1); 11221ec3b5feSBarry Song sg_set_page(&output, page, PAGE_SIZE, 0); 11231ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen); 11241ec3b5feSBarry Song ret = crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait); 11251ec3b5feSBarry Song dlen = acomp_ctx->req->dlen; 11261ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 11271ec3b5feSBarry Song 11286b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) 11296b3379e8SJohannes Weiner kfree(tmp); 11306b3379e8SJohannes Weiner else 1131ff9d5ba2SDomenico Cerasuolo zpool_unmap_handle(pool, entry->handle); 11326b3379e8SJohannes Weiner 11332b281117SSeth Jennings BUG_ON(ret); 11342b281117SSeth Jennings BUG_ON(dlen != PAGE_SIZE); 11352b281117SSeth Jennings 11362b281117SSeth Jennings /* page is up to date */ 11372b281117SSeth Jennings SetPageUptodate(page); 11382b281117SSeth Jennings 1139b349acc7SWeijie Yang /* move it to the tail of the inactive list after end_writeback */ 1140b349acc7SWeijie Yang SetPageReclaim(page); 1141b349acc7SWeijie Yang 11422b281117SSeth Jennings /* start writeback */ 1143cf1e3fe4SChristoph Hellwig __swap_writepage(page, &wbc); 114409cbfeafSKirill A. Shutemov put_page(page); 11452b281117SSeth Jennings zswap_written_back_pages++; 11462b281117SSeth Jennings 11476b3379e8SJohannes Weiner return ret; 114898804a94SJohannes Weiner 11496b3379e8SJohannes Weiner fail: 11506b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) 11516b3379e8SJohannes Weiner kfree(tmp); 11520ab0abcfSWeijie Yang 11530ab0abcfSWeijie Yang /* 115498804a94SJohannes Weiner * If we get here because the page is already in swapcache, a 115598804a94SJohannes Weiner * load may be happening concurrently. It is safe and okay to 115698804a94SJohannes Weiner * not free the entry. It is also okay to return !0. 11570ab0abcfSWeijie Yang */ 11582b281117SSeth Jennings return ret; 11592b281117SSeth Jennings } 11602b281117SSeth Jennings 1161a85f878bSSrividya Desireddy static int zswap_is_page_same_filled(void *ptr, unsigned long *value) 1162a85f878bSSrividya Desireddy { 1163a85f878bSSrividya Desireddy unsigned long *page; 116462bf1258STaejoon Song unsigned long val; 116562bf1258STaejoon Song unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; 1166a85f878bSSrividya Desireddy 1167a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 116862bf1258STaejoon Song val = page[0]; 116962bf1258STaejoon Song 117062bf1258STaejoon Song if (val != page[last_pos]) 117162bf1258STaejoon Song return 0; 117262bf1258STaejoon Song 117362bf1258STaejoon Song for (pos = 1; pos < last_pos; pos++) { 117462bf1258STaejoon Song if (val != page[pos]) 1175a85f878bSSrividya Desireddy return 0; 1176a85f878bSSrividya Desireddy } 117762bf1258STaejoon Song 117862bf1258STaejoon Song *value = val; 117962bf1258STaejoon Song 1180a85f878bSSrividya Desireddy return 1; 1181a85f878bSSrividya Desireddy } 1182a85f878bSSrividya Desireddy 1183a85f878bSSrividya Desireddy static void zswap_fill_page(void *ptr, unsigned long value) 1184a85f878bSSrividya Desireddy { 1185a85f878bSSrividya Desireddy unsigned long *page; 1186a85f878bSSrividya Desireddy 1187a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 1188a85f878bSSrividya Desireddy memset_l(page, value, PAGE_SIZE / sizeof(unsigned long)); 1189a85f878bSSrividya Desireddy } 1190a85f878bSSrividya Desireddy 119134f4c198SMatthew Wilcox (Oracle) bool zswap_store(struct folio *folio) 11922b281117SSeth Jennings { 1193*3d2c9087SDavid Hildenbrand swp_entry_t swp = folio->swap; 119442c06a0eSJohannes Weiner int type = swp_type(swp); 119542c06a0eSJohannes Weiner pgoff_t offset = swp_offset(swp); 119634f4c198SMatthew Wilcox (Oracle) struct page *page = &folio->page; 11972b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 11982b281117SSeth Jennings struct zswap_entry *entry, *dupentry; 11991ec3b5feSBarry Song struct scatterlist input, output; 12001ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1201f4840ccfSJohannes Weiner struct obj_cgroup *objcg = NULL; 1202f4840ccfSJohannes Weiner struct zswap_pool *pool; 1203b8cf32dcSYosry Ahmed struct zpool *zpool; 12040bb48849SDomenico Cerasuolo unsigned int dlen = PAGE_SIZE; 1205a85f878bSSrividya Desireddy unsigned long handle, value; 12062b281117SSeth Jennings char *buf; 12072b281117SSeth Jennings u8 *src, *dst; 1208d2fcd82bSHui Zhu gfp_t gfp; 120942c06a0eSJohannes Weiner int ret; 121042c06a0eSJohannes Weiner 121134f4c198SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_locked(folio)); 121234f4c198SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); 12132b281117SSeth Jennings 121434f4c198SMatthew Wilcox (Oracle) /* Large folios aren't supported */ 121534f4c198SMatthew Wilcox (Oracle) if (folio_test_large(folio)) 121642c06a0eSJohannes Weiner return false; 12177ba71669SHuang Ying 121842c06a0eSJohannes Weiner if (!zswap_enabled || !tree) 121942c06a0eSJohannes Weiner return false; 12202b281117SSeth Jennings 12210bdf0efaSNhat Pham /* 12220bdf0efaSNhat Pham * XXX: zswap reclaim does not work with cgroups yet. Without a 12230bdf0efaSNhat Pham * cgroup-aware entry LRU, we will push out entries system-wide based on 12240bdf0efaSNhat Pham * local cgroup limits. 12250bdf0efaSNhat Pham */ 1226074e3e26SMatthew Wilcox (Oracle) objcg = get_obj_cgroup_from_folio(folio); 122742c06a0eSJohannes Weiner if (objcg && !obj_cgroup_may_zswap(objcg)) 12280bdf0efaSNhat Pham goto reject; 1229f4840ccfSJohannes Weiner 12302b281117SSeth Jennings /* reclaim space if needed */ 12312b281117SSeth Jennings if (zswap_is_full()) { 12322b281117SSeth Jennings zswap_pool_limit_hit++; 123345190f01SVitaly Wool zswap_pool_reached_full = true; 1234f4840ccfSJohannes Weiner goto shrink; 12352b281117SSeth Jennings } 123616e536efSLi Wang 123745190f01SVitaly Wool if (zswap_pool_reached_full) { 123842c06a0eSJohannes Weiner if (!zswap_can_accept()) 1239e0228d59SDomenico Cerasuolo goto shrink; 124042c06a0eSJohannes Weiner else 124145190f01SVitaly Wool zswap_pool_reached_full = false; 12422b281117SSeth Jennings } 12432b281117SSeth Jennings 12442b281117SSeth Jennings /* allocate entry */ 12452b281117SSeth Jennings entry = zswap_entry_cache_alloc(GFP_KERNEL); 12462b281117SSeth Jennings if (!entry) { 12472b281117SSeth Jennings zswap_reject_kmemcache_fail++; 12482b281117SSeth Jennings goto reject; 12492b281117SSeth Jennings } 12502b281117SSeth Jennings 1251a85f878bSSrividya Desireddy if (zswap_same_filled_pages_enabled) { 1252a85f878bSSrividya Desireddy src = kmap_atomic(page); 1253a85f878bSSrividya Desireddy if (zswap_is_page_same_filled(src, &value)) { 1254a85f878bSSrividya Desireddy kunmap_atomic(src); 12550bb48849SDomenico Cerasuolo entry->swpentry = swp_entry(type, offset); 1256a85f878bSSrividya Desireddy entry->length = 0; 1257a85f878bSSrividya Desireddy entry->value = value; 1258a85f878bSSrividya Desireddy atomic_inc(&zswap_same_filled_pages); 1259a85f878bSSrividya Desireddy goto insert_entry; 1260a85f878bSSrividya Desireddy } 1261a85f878bSSrividya Desireddy kunmap_atomic(src); 1262a85f878bSSrividya Desireddy } 1263a85f878bSSrividya Desireddy 126442c06a0eSJohannes Weiner if (!zswap_non_same_filled_pages_enabled) 1265cb325dddSMaciej S. Szmigiero goto freepage; 1266cb325dddSMaciej S. Szmigiero 1267f1c54846SDan Streetman /* if entry is successfully added, it keeps the reference */ 1268f1c54846SDan Streetman entry->pool = zswap_pool_current_get(); 126942c06a0eSJohannes Weiner if (!entry->pool) 12702b281117SSeth Jennings goto freepage; 12712b281117SSeth Jennings 1272f1c54846SDan Streetman /* compress */ 12731ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 12741ec3b5feSBarry Song 12751ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 12761ec3b5feSBarry Song 12771ec3b5feSBarry Song dst = acomp_ctx->dstmem; 12781ec3b5feSBarry Song sg_init_table(&input, 1); 12791ec3b5feSBarry Song sg_set_page(&input, page, PAGE_SIZE, 0); 12801ec3b5feSBarry Song 12811ec3b5feSBarry Song /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */ 12821ec3b5feSBarry Song sg_init_one(&output, dst, PAGE_SIZE * 2); 12831ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); 12841ec3b5feSBarry Song /* 12851ec3b5feSBarry Song * it maybe looks a little bit silly that we send an asynchronous request, 12861ec3b5feSBarry Song * then wait for its completion synchronously. This makes the process look 12871ec3b5feSBarry Song * synchronous in fact. 12881ec3b5feSBarry Song * Theoretically, acomp supports users send multiple acomp requests in one 12891ec3b5feSBarry Song * acomp instance, then get those requests done simultaneously. but in this 129042c06a0eSJohannes Weiner * case, zswap actually does store and load page by page, there is no 12911ec3b5feSBarry Song * existing method to send the second page before the first page is done 129242c06a0eSJohannes Weiner * in one thread doing zwap. 12931ec3b5feSBarry Song * but in different threads running on different cpu, we have different 12941ec3b5feSBarry Song * acomp instance, so multiple threads can do (de)compression in parallel. 12951ec3b5feSBarry Song */ 12961ec3b5feSBarry Song ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait); 12971ec3b5feSBarry Song dlen = acomp_ctx->req->dlen; 12981ec3b5feSBarry Song 129942c06a0eSJohannes Weiner if (ret) 1300f1c54846SDan Streetman goto put_dstmem; 1301f1c54846SDan Streetman 13022b281117SSeth Jennings /* store */ 1303b8cf32dcSYosry Ahmed zpool = zswap_find_zpool(entry); 1304d2fcd82bSHui Zhu gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 1305b8cf32dcSYosry Ahmed if (zpool_malloc_support_movable(zpool)) 1306d2fcd82bSHui Zhu gfp |= __GFP_HIGHMEM | __GFP_MOVABLE; 1307b8cf32dcSYosry Ahmed ret = zpool_malloc(zpool, dlen, gfp, &handle); 13082b281117SSeth Jennings if (ret == -ENOSPC) { 13092b281117SSeth Jennings zswap_reject_compress_poor++; 1310f1c54846SDan Streetman goto put_dstmem; 13112b281117SSeth Jennings } 13122b281117SSeth Jennings if (ret) { 13132b281117SSeth Jennings zswap_reject_alloc_fail++; 1314f1c54846SDan Streetman goto put_dstmem; 13152b281117SSeth Jennings } 1316b8cf32dcSYosry Ahmed buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO); 13170bb48849SDomenico Cerasuolo memcpy(buf, dst, dlen); 1318b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, handle); 13191ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 13202b281117SSeth Jennings 13212b281117SSeth Jennings /* populate entry */ 13220bb48849SDomenico Cerasuolo entry->swpentry = swp_entry(type, offset); 13232b281117SSeth Jennings entry->handle = handle; 13242b281117SSeth Jennings entry->length = dlen; 13252b281117SSeth Jennings 1326a85f878bSSrividya Desireddy insert_entry: 1327f4840ccfSJohannes Weiner entry->objcg = objcg; 1328f4840ccfSJohannes Weiner if (objcg) { 1329f4840ccfSJohannes Weiner obj_cgroup_charge_zswap(objcg, entry->length); 1330f4840ccfSJohannes Weiner /* Account before objcg ref is moved to tree */ 1331f4840ccfSJohannes Weiner count_objcg_event(objcg, ZSWPOUT); 1332f4840ccfSJohannes Weiner } 1333f4840ccfSJohannes Weiner 13342b281117SSeth Jennings /* map */ 13352b281117SSeth Jennings spin_lock(&tree->lock); 133642c06a0eSJohannes Weiner while (zswap_rb_insert(&tree->rbroot, entry, &dupentry) == -EEXIST) { 13372b281117SSeth Jennings zswap_duplicate_entry++; 133856c67049SJohannes Weiner zswap_invalidate_entry(tree, dupentry); 13392b281117SSeth Jennings } 134035499e2bSDomenico Cerasuolo if (entry->length) { 1341f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 1342f999f38bSDomenico Cerasuolo list_add(&entry->lru, &entry->pool->lru); 1343f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 1344f999f38bSDomenico Cerasuolo } 13452b281117SSeth Jennings spin_unlock(&tree->lock); 13462b281117SSeth Jennings 13472b281117SSeth Jennings /* update stats */ 13482b281117SSeth Jennings atomic_inc(&zswap_stored_pages); 1349f1c54846SDan Streetman zswap_update_total_size(); 1350f6498b77SJohannes Weiner count_vm_event(ZSWPOUT); 13512b281117SSeth Jennings 135242c06a0eSJohannes Weiner return true; 13532b281117SSeth Jennings 1354f1c54846SDan Streetman put_dstmem: 13551ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 1356f1c54846SDan Streetman zswap_pool_put(entry->pool); 1357f1c54846SDan Streetman freepage: 13582b281117SSeth Jennings zswap_entry_cache_free(entry); 13592b281117SSeth Jennings reject: 1360f4840ccfSJohannes Weiner if (objcg) 1361f4840ccfSJohannes Weiner obj_cgroup_put(objcg); 136242c06a0eSJohannes Weiner return false; 1363f4840ccfSJohannes Weiner 1364f4840ccfSJohannes Weiner shrink: 1365f4840ccfSJohannes Weiner pool = zswap_pool_last_get(); 1366f4840ccfSJohannes Weiner if (pool) 1367f4840ccfSJohannes Weiner queue_work(shrink_wq, &pool->shrink_work); 1368f4840ccfSJohannes Weiner goto reject; 13692b281117SSeth Jennings } 13702b281117SSeth Jennings 1371ca54f6d8SMatthew Wilcox (Oracle) bool zswap_load(struct folio *folio) 13722b281117SSeth Jennings { 1373*3d2c9087SDavid Hildenbrand swp_entry_t swp = folio->swap; 137442c06a0eSJohannes Weiner int type = swp_type(swp); 137542c06a0eSJohannes Weiner pgoff_t offset = swp_offset(swp); 1376ca54f6d8SMatthew Wilcox (Oracle) struct page *page = &folio->page; 13772b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 13782b281117SSeth Jennings struct zswap_entry *entry; 13791ec3b5feSBarry Song struct scatterlist input, output; 13801ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1381fc6697a8STian Tao u8 *src, *dst, *tmp; 1382b8cf32dcSYosry Ahmed struct zpool *zpool; 13832b281117SSeth Jennings unsigned int dlen; 138442c06a0eSJohannes Weiner bool ret; 138542c06a0eSJohannes Weiner 1386ca54f6d8SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_locked(folio)); 13872b281117SSeth Jennings 13882b281117SSeth Jennings /* find */ 13892b281117SSeth Jennings spin_lock(&tree->lock); 13900ab0abcfSWeijie Yang entry = zswap_entry_find_get(&tree->rbroot, offset); 13912b281117SSeth Jennings if (!entry) { 13922b281117SSeth Jennings spin_unlock(&tree->lock); 139342c06a0eSJohannes Weiner return false; 13942b281117SSeth Jennings } 13952b281117SSeth Jennings spin_unlock(&tree->lock); 13962b281117SSeth Jennings 1397a85f878bSSrividya Desireddy if (!entry->length) { 1398a85f878bSSrividya Desireddy dst = kmap_atomic(page); 1399a85f878bSSrividya Desireddy zswap_fill_page(dst, entry->value); 1400a85f878bSSrividya Desireddy kunmap_atomic(dst); 140142c06a0eSJohannes Weiner ret = true; 1402f6498b77SJohannes Weiner goto stats; 1403a85f878bSSrividya Desireddy } 1404a85f878bSSrividya Desireddy 1405b8cf32dcSYosry Ahmed zpool = zswap_find_zpool(entry); 1406b8cf32dcSYosry Ahmed if (!zpool_can_sleep_mapped(zpool)) { 14078d9b6370SSergey Senozhatsky tmp = kmalloc(entry->length, GFP_KERNEL); 1408fc6697a8STian Tao if (!tmp) { 140942c06a0eSJohannes Weiner ret = false; 1410fc6697a8STian Tao goto freeentry; 1411fc6697a8STian Tao } 1412fc6697a8STian Tao } 1413fc6697a8STian Tao 14142b281117SSeth Jennings /* decompress */ 14152b281117SSeth Jennings dlen = PAGE_SIZE; 1416b8cf32dcSYosry Ahmed src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO); 14171ec3b5feSBarry Song 1418b8cf32dcSYosry Ahmed if (!zpool_can_sleep_mapped(zpool)) { 1419fc6697a8STian Tao memcpy(tmp, src, entry->length); 1420fc6697a8STian Tao src = tmp; 1421b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, entry->handle); 1422fc6697a8STian Tao } 1423fc6697a8STian Tao 14241ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 14251ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 14261ec3b5feSBarry Song sg_init_one(&input, src, entry->length); 14271ec3b5feSBarry Song sg_init_table(&output, 1); 14281ec3b5feSBarry Song sg_set_page(&output, page, PAGE_SIZE, 0); 14291ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen); 143042c06a0eSJohannes Weiner if (crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait)) 143142c06a0eSJohannes Weiner WARN_ON(1); 14321ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 14331ec3b5feSBarry Song 1434b8cf32dcSYosry Ahmed if (zpool_can_sleep_mapped(zpool)) 1435b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, entry->handle); 1436fc6697a8STian Tao else 1437fc6697a8STian Tao kfree(tmp); 1438fc6697a8STian Tao 143942c06a0eSJohannes Weiner ret = true; 1440f6498b77SJohannes Weiner stats: 1441f6498b77SJohannes Weiner count_vm_event(ZSWPIN); 1442f4840ccfSJohannes Weiner if (entry->objcg) 1443f4840ccfSJohannes Weiner count_objcg_event(entry->objcg, ZSWPIN); 1444a85f878bSSrividya Desireddy freeentry: 14452b281117SSeth Jennings spin_lock(&tree->lock); 144642c06a0eSJohannes Weiner if (ret && zswap_exclusive_loads_enabled) { 1447b9c91c43SYosry Ahmed zswap_invalidate_entry(tree, entry); 1448ca54f6d8SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 144935499e2bSDomenico Cerasuolo } else if (entry->length) { 1450f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 1451f999f38bSDomenico Cerasuolo list_move(&entry->lru, &entry->pool->lru); 1452f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 1453b9c91c43SYosry Ahmed } 145418a93707SYosry Ahmed zswap_entry_put(tree, entry); 14552b281117SSeth Jennings spin_unlock(&tree->lock); 14562b281117SSeth Jennings 1457fc6697a8STian Tao return ret; 14582b281117SSeth Jennings } 14592b281117SSeth Jennings 146042c06a0eSJohannes Weiner void zswap_invalidate(int type, pgoff_t offset) 14612b281117SSeth Jennings { 14622b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 14632b281117SSeth Jennings struct zswap_entry *entry; 14642b281117SSeth Jennings 14652b281117SSeth Jennings /* find */ 14662b281117SSeth Jennings spin_lock(&tree->lock); 14672b281117SSeth Jennings entry = zswap_rb_search(&tree->rbroot, offset); 14682b281117SSeth Jennings if (!entry) { 14692b281117SSeth Jennings /* entry was written back */ 14702b281117SSeth Jennings spin_unlock(&tree->lock); 14712b281117SSeth Jennings return; 14722b281117SSeth Jennings } 1473b9c91c43SYosry Ahmed zswap_invalidate_entry(tree, entry); 14742b281117SSeth Jennings spin_unlock(&tree->lock); 14752b281117SSeth Jennings } 14762b281117SSeth Jennings 147742c06a0eSJohannes Weiner void zswap_swapon(int type) 147842c06a0eSJohannes Weiner { 147942c06a0eSJohannes Weiner struct zswap_tree *tree; 148042c06a0eSJohannes Weiner 148142c06a0eSJohannes Weiner tree = kzalloc(sizeof(*tree), GFP_KERNEL); 148242c06a0eSJohannes Weiner if (!tree) { 148342c06a0eSJohannes Weiner pr_err("alloc failed, zswap disabled for swap type %d\n", type); 148442c06a0eSJohannes Weiner return; 148542c06a0eSJohannes Weiner } 148642c06a0eSJohannes Weiner 148742c06a0eSJohannes Weiner tree->rbroot = RB_ROOT; 148842c06a0eSJohannes Weiner spin_lock_init(&tree->lock); 148942c06a0eSJohannes Weiner zswap_trees[type] = tree; 149042c06a0eSJohannes Weiner } 149142c06a0eSJohannes Weiner 149242c06a0eSJohannes Weiner void zswap_swapoff(int type) 14932b281117SSeth Jennings { 14942b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 14950bd42136SCody P Schafer struct zswap_entry *entry, *n; 14962b281117SSeth Jennings 14972b281117SSeth Jennings if (!tree) 14982b281117SSeth Jennings return; 14992b281117SSeth Jennings 15002b281117SSeth Jennings /* walk the tree and free everything */ 15012b281117SSeth Jennings spin_lock(&tree->lock); 15020ab0abcfSWeijie Yang rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) 150360105e12SMinchan Kim zswap_free_entry(entry); 15042b281117SSeth Jennings tree->rbroot = RB_ROOT; 15052b281117SSeth Jennings spin_unlock(&tree->lock); 1506aa9bca05SWeijie Yang kfree(tree); 1507aa9bca05SWeijie Yang zswap_trees[type] = NULL; 15082b281117SSeth Jennings } 15092b281117SSeth Jennings 15102b281117SSeth Jennings /********************************* 15112b281117SSeth Jennings * debugfs functions 15122b281117SSeth Jennings **********************************/ 15132b281117SSeth Jennings #ifdef CONFIG_DEBUG_FS 15142b281117SSeth Jennings #include <linux/debugfs.h> 15152b281117SSeth Jennings 15162b281117SSeth Jennings static struct dentry *zswap_debugfs_root; 15172b281117SSeth Jennings 1518141fdeecSLiu Shixin static int zswap_debugfs_init(void) 15192b281117SSeth Jennings { 15202b281117SSeth Jennings if (!debugfs_initialized()) 15212b281117SSeth Jennings return -ENODEV; 15222b281117SSeth Jennings 15232b281117SSeth Jennings zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 15242b281117SSeth Jennings 15250825a6f9SJoe Perches debugfs_create_u64("pool_limit_hit", 0444, 15262b281117SSeth Jennings zswap_debugfs_root, &zswap_pool_limit_hit); 15270825a6f9SJoe Perches debugfs_create_u64("reject_reclaim_fail", 0444, 15282b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_reclaim_fail); 15290825a6f9SJoe Perches debugfs_create_u64("reject_alloc_fail", 0444, 15302b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_alloc_fail); 15310825a6f9SJoe Perches debugfs_create_u64("reject_kmemcache_fail", 0444, 15322b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_kmemcache_fail); 15330825a6f9SJoe Perches debugfs_create_u64("reject_compress_poor", 0444, 15342b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_compress_poor); 15350825a6f9SJoe Perches debugfs_create_u64("written_back_pages", 0444, 15362b281117SSeth Jennings zswap_debugfs_root, &zswap_written_back_pages); 15370825a6f9SJoe Perches debugfs_create_u64("duplicate_entry", 0444, 15382b281117SSeth Jennings zswap_debugfs_root, &zswap_duplicate_entry); 15390825a6f9SJoe Perches debugfs_create_u64("pool_total_size", 0444, 154012d79d64SDan Streetman zswap_debugfs_root, &zswap_pool_total_size); 15410825a6f9SJoe Perches debugfs_create_atomic_t("stored_pages", 0444, 15422b281117SSeth Jennings zswap_debugfs_root, &zswap_stored_pages); 1543a85f878bSSrividya Desireddy debugfs_create_atomic_t("same_filled_pages", 0444, 1544a85f878bSSrividya Desireddy zswap_debugfs_root, &zswap_same_filled_pages); 15452b281117SSeth Jennings 15462b281117SSeth Jennings return 0; 15472b281117SSeth Jennings } 15482b281117SSeth Jennings #else 1549141fdeecSLiu Shixin static int zswap_debugfs_init(void) 15502b281117SSeth Jennings { 15512b281117SSeth Jennings return 0; 15522b281117SSeth Jennings } 15532b281117SSeth Jennings #endif 15542b281117SSeth Jennings 15552b281117SSeth Jennings /********************************* 15562b281117SSeth Jennings * module init and exit 15572b281117SSeth Jennings **********************************/ 1558141fdeecSLiu Shixin static int zswap_setup(void) 15592b281117SSeth Jennings { 1560f1c54846SDan Streetman struct zswap_pool *pool; 1561ad7ed770SSebastian Andrzej Siewior int ret; 156260105e12SMinchan Kim 1563b7919122SLiu Shixin zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 1564b7919122SLiu Shixin if (!zswap_entry_cache) { 15652b281117SSeth Jennings pr_err("entry cache creation failed\n"); 1566f1c54846SDan Streetman goto cache_fail; 15672b281117SSeth Jennings } 1568f1c54846SDan Streetman 1569ad7ed770SSebastian Andrzej Siewior ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare", 1570ad7ed770SSebastian Andrzej Siewior zswap_dstmem_prepare, zswap_dstmem_dead); 1571ad7ed770SSebastian Andrzej Siewior if (ret) { 1572f1c54846SDan Streetman pr_err("dstmem alloc failed\n"); 1573f1c54846SDan Streetman goto dstmem_fail; 15742b281117SSeth Jennings } 1575f1c54846SDan Streetman 1576cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, 1577cab7a7e5SSebastian Andrzej Siewior "mm/zswap_pool:prepare", 1578cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_prepare, 1579cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_dead); 1580cab7a7e5SSebastian Andrzej Siewior if (ret) 1581cab7a7e5SSebastian Andrzej Siewior goto hp_fail; 1582cab7a7e5SSebastian Andrzej Siewior 1583f1c54846SDan Streetman pool = __zswap_pool_create_fallback(); 1584ae3d89a7SDan Streetman if (pool) { 1585f1c54846SDan Streetman pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1586b8cf32dcSYosry Ahmed zpool_get_type(pool->zpools[0])); 1587f1c54846SDan Streetman list_add(&pool->list, &zswap_pools); 1588ae3d89a7SDan Streetman zswap_has_pool = true; 1589ae3d89a7SDan Streetman } else { 1590ae3d89a7SDan Streetman pr_err("pool creation failed\n"); 1591ae3d89a7SDan Streetman zswap_enabled = false; 1592ae3d89a7SDan Streetman } 159360105e12SMinchan Kim 159445190f01SVitaly Wool shrink_wq = create_workqueue("zswap-shrink"); 159545190f01SVitaly Wool if (!shrink_wq) 159645190f01SVitaly Wool goto fallback_fail; 159745190f01SVitaly Wool 15982b281117SSeth Jennings if (zswap_debugfs_init()) 15992b281117SSeth Jennings pr_warn("debugfs initialization failed\n"); 16009021ccecSLiu Shixin zswap_init_state = ZSWAP_INIT_SUCCEED; 16012b281117SSeth Jennings return 0; 1602f1c54846SDan Streetman 160345190f01SVitaly Wool fallback_fail: 160438aeb071SDan Carpenter if (pool) 160545190f01SVitaly Wool zswap_pool_destroy(pool); 1606cab7a7e5SSebastian Andrzej Siewior hp_fail: 1607ad7ed770SSebastian Andrzej Siewior cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); 1608f1c54846SDan Streetman dstmem_fail: 1609b7919122SLiu Shixin kmem_cache_destroy(zswap_entry_cache); 1610f1c54846SDan Streetman cache_fail: 1611d7b028f5SDan Streetman /* if built-in, we aren't unloaded on failure; don't allow use */ 16129021ccecSLiu Shixin zswap_init_state = ZSWAP_INIT_FAILED; 1613d7b028f5SDan Streetman zswap_enabled = false; 16142b281117SSeth Jennings return -ENOMEM; 16152b281117SSeth Jennings } 1616141fdeecSLiu Shixin 1617141fdeecSLiu Shixin static int __init zswap_init(void) 1618141fdeecSLiu Shixin { 1619141fdeecSLiu Shixin if (!zswap_enabled) 1620141fdeecSLiu Shixin return 0; 1621141fdeecSLiu Shixin return zswap_setup(); 1622141fdeecSLiu Shixin } 16232b281117SSeth Jennings /* must be late so crypto has time to come up */ 1624141fdeecSLiu Shixin late_initcall(zswap_init); 16252b281117SSeth Jennings 162668386da8SSeth Jennings MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 16272b281117SSeth Jennings MODULE_DESCRIPTION("Compressed cache for swap pages"); 1628