1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 22b281117SSeth Jennings /* 32b281117SSeth Jennings * zswap.c - zswap driver file 42b281117SSeth Jennings * 542c06a0eSJohannes Weiner * zswap is a cache that takes pages that are in the process 62b281117SSeth Jennings * of being swapped out and attempts to compress and store them in a 72b281117SSeth Jennings * RAM-based memory pool. This can result in a significant I/O reduction on 82b281117SSeth Jennings * the swap device and, in the case where decompressing from RAM is faster 92b281117SSeth Jennings * than reading from the swap device, can also improve workload performance. 102b281117SSeth Jennings * 112b281117SSeth Jennings * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 122b281117SSeth Jennings */ 132b281117SSeth Jennings 142b281117SSeth Jennings #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 152b281117SSeth Jennings 162b281117SSeth Jennings #include <linux/module.h> 172b281117SSeth Jennings #include <linux/cpu.h> 182b281117SSeth Jennings #include <linux/highmem.h> 192b281117SSeth Jennings #include <linux/slab.h> 202b281117SSeth Jennings #include <linux/spinlock.h> 212b281117SSeth Jennings #include <linux/types.h> 222b281117SSeth Jennings #include <linux/atomic.h> 232b281117SSeth Jennings #include <linux/rbtree.h> 242b281117SSeth Jennings #include <linux/swap.h> 252b281117SSeth Jennings #include <linux/crypto.h> 261ec3b5feSBarry Song #include <linux/scatterlist.h> 27*ddc1a5cbSHugh Dickins #include <linux/mempolicy.h> 282b281117SSeth Jennings #include <linux/mempool.h> 2912d79d64SDan Streetman #include <linux/zpool.h> 301ec3b5feSBarry Song #include <crypto/acompress.h> 3142c06a0eSJohannes Weiner #include <linux/zswap.h> 322b281117SSeth Jennings #include <linux/mm_types.h> 332b281117SSeth Jennings #include <linux/page-flags.h> 342b281117SSeth Jennings #include <linux/swapops.h> 352b281117SSeth Jennings #include <linux/writeback.h> 362b281117SSeth Jennings #include <linux/pagemap.h> 3745190f01SVitaly Wool #include <linux/workqueue.h> 382b281117SSeth Jennings 39014bb1deSNeilBrown #include "swap.h" 40e0228d59SDomenico Cerasuolo #include "internal.h" 41014bb1deSNeilBrown 422b281117SSeth Jennings /********************************* 432b281117SSeth Jennings * statistics 442b281117SSeth Jennings **********************************/ 4512d79d64SDan Streetman /* Total bytes used by the compressed storage */ 46f6498b77SJohannes Weiner u64 zswap_pool_total_size; 472b281117SSeth Jennings /* The number of compressed pages currently stored in zswap */ 48f6498b77SJohannes Weiner atomic_t zswap_stored_pages = ATOMIC_INIT(0); 49a85f878bSSrividya Desireddy /* The number of same-value filled pages currently stored in zswap */ 50a85f878bSSrividya Desireddy static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0); 512b281117SSeth Jennings 522b281117SSeth Jennings /* 532b281117SSeth Jennings * The statistics below are not protected from concurrent access for 542b281117SSeth Jennings * performance reasons so they may not be a 100% accurate. However, 552b281117SSeth Jennings * they do provide useful information on roughly how many times a 562b281117SSeth Jennings * certain event is occurring. 572b281117SSeth Jennings */ 582b281117SSeth Jennings 592b281117SSeth Jennings /* Pool limit was hit (see zswap_max_pool_percent) */ 602b281117SSeth Jennings static u64 zswap_pool_limit_hit; 612b281117SSeth Jennings /* Pages written back when pool limit was reached */ 622b281117SSeth Jennings static u64 zswap_written_back_pages; 632b281117SSeth Jennings /* Store failed due to a reclaim failure after pool limit was reached */ 642b281117SSeth Jennings static u64 zswap_reject_reclaim_fail; 652b281117SSeth Jennings /* Compressed page was too big for the allocator to (optimally) store */ 662b281117SSeth Jennings static u64 zswap_reject_compress_poor; 672b281117SSeth Jennings /* Store failed because underlying allocator could not get memory */ 682b281117SSeth Jennings static u64 zswap_reject_alloc_fail; 692b281117SSeth Jennings /* Store failed because the entry metadata could not be allocated (rare) */ 702b281117SSeth Jennings static u64 zswap_reject_kmemcache_fail; 712b281117SSeth Jennings /* Duplicate store was encountered (rare) */ 722b281117SSeth Jennings static u64 zswap_duplicate_entry; 732b281117SSeth Jennings 7445190f01SVitaly Wool /* Shrinker work queue */ 7545190f01SVitaly Wool static struct workqueue_struct *shrink_wq; 7645190f01SVitaly Wool /* Pool limit was hit, we need to calm down */ 7745190f01SVitaly Wool static bool zswap_pool_reached_full; 7845190f01SVitaly Wool 792b281117SSeth Jennings /********************************* 802b281117SSeth Jennings * tunables 812b281117SSeth Jennings **********************************/ 82c00ed16aSDan Streetman 83bae21db8SDan Streetman #define ZSWAP_PARAM_UNSET "" 84bae21db8SDan Streetman 85141fdeecSLiu Shixin static int zswap_setup(void); 86141fdeecSLiu Shixin 87bb8b93b5SMaciej S. Szmigiero /* Enable/disable zswap */ 88bb8b93b5SMaciej S. Szmigiero static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); 89d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *, 90d7b028f5SDan Streetman const struct kernel_param *); 9183aed6cdSJoe Perches static const struct kernel_param_ops zswap_enabled_param_ops = { 92d7b028f5SDan Streetman .set = zswap_enabled_param_set, 93d7b028f5SDan Streetman .get = param_get_bool, 94d7b028f5SDan Streetman }; 95d7b028f5SDan Streetman module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); 962b281117SSeth Jennings 9790b0fc26SDan Streetman /* Crypto compressor to use */ 98bb8b93b5SMaciej S. Szmigiero static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 9990b0fc26SDan Streetman static int zswap_compressor_param_set(const char *, 10090b0fc26SDan Streetman const struct kernel_param *); 10183aed6cdSJoe Perches static const struct kernel_param_ops zswap_compressor_param_ops = { 10290b0fc26SDan Streetman .set = zswap_compressor_param_set, 103c99b42c3SDan Streetman .get = param_get_charp, 104c99b42c3SDan Streetman .free = param_free_charp, 10590b0fc26SDan Streetman }; 10690b0fc26SDan Streetman module_param_cb(compressor, &zswap_compressor_param_ops, 107c99b42c3SDan Streetman &zswap_compressor, 0644); 10890b0fc26SDan Streetman 10990b0fc26SDan Streetman /* Compressed storage zpool to use */ 110bb8b93b5SMaciej S. Szmigiero static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 11190b0fc26SDan Streetman static int zswap_zpool_param_set(const char *, const struct kernel_param *); 11283aed6cdSJoe Perches static const struct kernel_param_ops zswap_zpool_param_ops = { 11390b0fc26SDan Streetman .set = zswap_zpool_param_set, 114c99b42c3SDan Streetman .get = param_get_charp, 115c99b42c3SDan Streetman .free = param_free_charp, 11690b0fc26SDan Streetman }; 117c99b42c3SDan Streetman module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 1182b281117SSeth Jennings 1192b281117SSeth Jennings /* The maximum percentage of memory that the compressed pool can occupy */ 1202b281117SSeth Jennings static unsigned int zswap_max_pool_percent = 20; 12190b0fc26SDan Streetman module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 12260105e12SMinchan Kim 12345190f01SVitaly Wool /* The threshold for accepting new pages after the max_pool_percent was hit */ 12445190f01SVitaly Wool static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ 12545190f01SVitaly Wool module_param_named(accept_threshold_percent, zswap_accept_thr_percent, 12645190f01SVitaly Wool uint, 0644); 12745190f01SVitaly Wool 128cb325dddSMaciej S. Szmigiero /* 129cb325dddSMaciej S. Szmigiero * Enable/disable handling same-value filled pages (enabled by default). 130cb325dddSMaciej S. Szmigiero * If disabled every page is considered non-same-value filled. 131cb325dddSMaciej S. Szmigiero */ 132a85f878bSSrividya Desireddy static bool zswap_same_filled_pages_enabled = true; 133a85f878bSSrividya Desireddy module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled, 134a85f878bSSrividya Desireddy bool, 0644); 135a85f878bSSrividya Desireddy 136cb325dddSMaciej S. Szmigiero /* Enable/disable handling non-same-value filled pages (enabled by default) */ 137cb325dddSMaciej S. Szmigiero static bool zswap_non_same_filled_pages_enabled = true; 138cb325dddSMaciej S. Szmigiero module_param_named(non_same_filled_pages_enabled, zswap_non_same_filled_pages_enabled, 139cb325dddSMaciej S. Szmigiero bool, 0644); 140cb325dddSMaciej S. Szmigiero 141b9c91c43SYosry Ahmed static bool zswap_exclusive_loads_enabled = IS_ENABLED( 142b9c91c43SYosry Ahmed CONFIG_ZSWAP_EXCLUSIVE_LOADS_DEFAULT_ON); 143b9c91c43SYosry Ahmed module_param_named(exclusive_loads, zswap_exclusive_loads_enabled, bool, 0644); 144b9c91c43SYosry Ahmed 145b8cf32dcSYosry Ahmed /* Number of zpools in zswap_pool (empirically determined for scalability) */ 146b8cf32dcSYosry Ahmed #define ZSWAP_NR_ZPOOLS 32 147b8cf32dcSYosry Ahmed 1482b281117SSeth Jennings /********************************* 1492b281117SSeth Jennings * data structures 1502b281117SSeth Jennings **********************************/ 151f1c54846SDan Streetman 1521ec3b5feSBarry Song struct crypto_acomp_ctx { 1531ec3b5feSBarry Song struct crypto_acomp *acomp; 1541ec3b5feSBarry Song struct acomp_req *req; 1551ec3b5feSBarry Song struct crypto_wait wait; 1561ec3b5feSBarry Song u8 *dstmem; 1571ec3b5feSBarry Song struct mutex *mutex; 1581ec3b5feSBarry Song }; 1591ec3b5feSBarry Song 160f999f38bSDomenico Cerasuolo /* 161f999f38bSDomenico Cerasuolo * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock. 162f999f38bSDomenico Cerasuolo * The only case where lru_lock is not acquired while holding tree.lock is 163f999f38bSDomenico Cerasuolo * when a zswap_entry is taken off the lru for writeback, in that case it 164f999f38bSDomenico Cerasuolo * needs to be verified that it's still valid in the tree. 165f999f38bSDomenico Cerasuolo */ 166f1c54846SDan Streetman struct zswap_pool { 167b8cf32dcSYosry Ahmed struct zpool *zpools[ZSWAP_NR_ZPOOLS]; 1681ec3b5feSBarry Song struct crypto_acomp_ctx __percpu *acomp_ctx; 169f1c54846SDan Streetman struct kref kref; 170f1c54846SDan Streetman struct list_head list; 17145190f01SVitaly Wool struct work_struct release_work; 17245190f01SVitaly Wool struct work_struct shrink_work; 173cab7a7e5SSebastian Andrzej Siewior struct hlist_node node; 174f1c54846SDan Streetman char tfm_name[CRYPTO_MAX_ALG_NAME]; 175f999f38bSDomenico Cerasuolo struct list_head lru; 176f999f38bSDomenico Cerasuolo spinlock_t lru_lock; 177f1c54846SDan Streetman }; 178f1c54846SDan Streetman 1792b281117SSeth Jennings /* 1802b281117SSeth Jennings * struct zswap_entry 1812b281117SSeth Jennings * 1822b281117SSeth Jennings * This structure contains the metadata for tracking a single compressed 1832b281117SSeth Jennings * page within zswap. 1842b281117SSeth Jennings * 1852b281117SSeth Jennings * rbnode - links the entry into red-black tree for the appropriate swap type 18697157d89SXiu Jianfeng * swpentry - associated swap entry, the offset indexes into the red-black tree 1872b281117SSeth Jennings * refcount - the number of outstanding reference to the entry. This is needed 1882b281117SSeth Jennings * to protect against premature freeing of the entry by code 1896b452516SSeongJae Park * concurrent calls to load, invalidate, and writeback. The lock 1902b281117SSeth Jennings * for the zswap_tree structure that contains the entry must 1912b281117SSeth Jennings * be held while changing the refcount. Since the lock must 1922b281117SSeth Jennings * be held, there is no reason to also make refcount atomic. 1932b281117SSeth Jennings * length - the length in bytes of the compressed page data. Needed during 194f999f38bSDomenico Cerasuolo * decompression. For a same value filled page length is 0, and both 195f999f38bSDomenico Cerasuolo * pool and lru are invalid and must be ignored. 196f1c54846SDan Streetman * pool - the zswap_pool the entry's data is in 197f1c54846SDan Streetman * handle - zpool allocation handle that stores the compressed page data 198a85f878bSSrividya Desireddy * value - value of the same-value filled pages which have same content 19997157d89SXiu Jianfeng * objcg - the obj_cgroup that the compressed memory is charged to 200f999f38bSDomenico Cerasuolo * lru - handle to the pool's lru used to evict pages. 2012b281117SSeth Jennings */ 2022b281117SSeth Jennings struct zswap_entry { 2032b281117SSeth Jennings struct rb_node rbnode; 2040bb48849SDomenico Cerasuolo swp_entry_t swpentry; 2052b281117SSeth Jennings int refcount; 2062b281117SSeth Jennings unsigned int length; 207f1c54846SDan Streetman struct zswap_pool *pool; 208a85f878bSSrividya Desireddy union { 2092b281117SSeth Jennings unsigned long handle; 210a85f878bSSrividya Desireddy unsigned long value; 211a85f878bSSrividya Desireddy }; 212f4840ccfSJohannes Weiner struct obj_cgroup *objcg; 213f999f38bSDomenico Cerasuolo struct list_head lru; 2142b281117SSeth Jennings }; 2152b281117SSeth Jennings 2162b281117SSeth Jennings /* 2172b281117SSeth Jennings * The tree lock in the zswap_tree struct protects a few things: 2182b281117SSeth Jennings * - the rbtree 2192b281117SSeth Jennings * - the refcount field of each entry in the tree 2202b281117SSeth Jennings */ 2212b281117SSeth Jennings struct zswap_tree { 2222b281117SSeth Jennings struct rb_root rbroot; 2232b281117SSeth Jennings spinlock_t lock; 2242b281117SSeth Jennings }; 2252b281117SSeth Jennings 2262b281117SSeth Jennings static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; 2272b281117SSeth Jennings 228f1c54846SDan Streetman /* RCU-protected iteration */ 229f1c54846SDan Streetman static LIST_HEAD(zswap_pools); 230f1c54846SDan Streetman /* protects zswap_pools list modification */ 231f1c54846SDan Streetman static DEFINE_SPINLOCK(zswap_pools_lock); 23232a4e169SDan Streetman /* pool counter to provide unique names to zpool */ 23332a4e169SDan Streetman static atomic_t zswap_pools_count = ATOMIC_INIT(0); 234f1c54846SDan Streetman 2359021ccecSLiu Shixin enum zswap_init_type { 2369021ccecSLiu Shixin ZSWAP_UNINIT, 2379021ccecSLiu Shixin ZSWAP_INIT_SUCCEED, 2389021ccecSLiu Shixin ZSWAP_INIT_FAILED 2399021ccecSLiu Shixin }; 24090b0fc26SDan Streetman 2419021ccecSLiu Shixin static enum zswap_init_type zswap_init_state; 242d7b028f5SDan Streetman 243141fdeecSLiu Shixin /* used to ensure the integrity of initialization */ 244141fdeecSLiu Shixin static DEFINE_MUTEX(zswap_init_lock); 245f1c54846SDan Streetman 246ae3d89a7SDan Streetman /* init completed, but couldn't create the initial pool */ 247ae3d89a7SDan Streetman static bool zswap_has_pool; 248ae3d89a7SDan Streetman 249f1c54846SDan Streetman /********************************* 250f1c54846SDan Streetman * helpers and fwd declarations 251f1c54846SDan Streetman **********************************/ 252f1c54846SDan Streetman 253f1c54846SDan Streetman #define zswap_pool_debug(msg, p) \ 254f1c54846SDan Streetman pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 255b8cf32dcSYosry Ahmed zpool_get_type((p)->zpools[0])) 256f1c54846SDan Streetman 2570bb48849SDomenico Cerasuolo static int zswap_writeback_entry(struct zswap_entry *entry, 258ff9d5ba2SDomenico Cerasuolo struct zswap_tree *tree); 259f1c54846SDan Streetman static int zswap_pool_get(struct zswap_pool *pool); 260f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool); 261f1c54846SDan Streetman 262f1c54846SDan Streetman static bool zswap_is_full(void) 263f1c54846SDan Streetman { 264ca79b0c2SArun KS return totalram_pages() * zswap_max_pool_percent / 100 < 265f1c54846SDan Streetman DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 266f1c54846SDan Streetman } 267f1c54846SDan Streetman 26845190f01SVitaly Wool static bool zswap_can_accept(void) 26945190f01SVitaly Wool { 27045190f01SVitaly Wool return totalram_pages() * zswap_accept_thr_percent / 100 * 27145190f01SVitaly Wool zswap_max_pool_percent / 100 > 27245190f01SVitaly Wool DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 27345190f01SVitaly Wool } 27445190f01SVitaly Wool 275f1c54846SDan Streetman static void zswap_update_total_size(void) 276f1c54846SDan Streetman { 277f1c54846SDan Streetman struct zswap_pool *pool; 278f1c54846SDan Streetman u64 total = 0; 279b8cf32dcSYosry Ahmed int i; 280f1c54846SDan Streetman 281f1c54846SDan Streetman rcu_read_lock(); 282f1c54846SDan Streetman 283f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 284b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) 285b8cf32dcSYosry Ahmed total += zpool_get_total_size(pool->zpools[i]); 286f1c54846SDan Streetman 287f1c54846SDan Streetman rcu_read_unlock(); 288f1c54846SDan Streetman 289f1c54846SDan Streetman zswap_pool_total_size = total; 290f1c54846SDan Streetman } 291f1c54846SDan Streetman 2922b281117SSeth Jennings /********************************* 2932b281117SSeth Jennings * zswap entry functions 2942b281117SSeth Jennings **********************************/ 2952b281117SSeth Jennings static struct kmem_cache *zswap_entry_cache; 2962b281117SSeth Jennings 2972b281117SSeth Jennings static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) 2982b281117SSeth Jennings { 2992b281117SSeth Jennings struct zswap_entry *entry; 3002b281117SSeth Jennings entry = kmem_cache_alloc(zswap_entry_cache, gfp); 3012b281117SSeth Jennings if (!entry) 3022b281117SSeth Jennings return NULL; 3032b281117SSeth Jennings entry->refcount = 1; 3040ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 3052b281117SSeth Jennings return entry; 3062b281117SSeth Jennings } 3072b281117SSeth Jennings 3082b281117SSeth Jennings static void zswap_entry_cache_free(struct zswap_entry *entry) 3092b281117SSeth Jennings { 3102b281117SSeth Jennings kmem_cache_free(zswap_entry_cache, entry); 3112b281117SSeth Jennings } 3122b281117SSeth Jennings 3132b281117SSeth Jennings /********************************* 3142b281117SSeth Jennings * rbtree functions 3152b281117SSeth Jennings **********************************/ 3162b281117SSeth Jennings static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) 3172b281117SSeth Jennings { 3182b281117SSeth Jennings struct rb_node *node = root->rb_node; 3192b281117SSeth Jennings struct zswap_entry *entry; 3200bb48849SDomenico Cerasuolo pgoff_t entry_offset; 3212b281117SSeth Jennings 3222b281117SSeth Jennings while (node) { 3232b281117SSeth Jennings entry = rb_entry(node, struct zswap_entry, rbnode); 3240bb48849SDomenico Cerasuolo entry_offset = swp_offset(entry->swpentry); 3250bb48849SDomenico Cerasuolo if (entry_offset > offset) 3262b281117SSeth Jennings node = node->rb_left; 3270bb48849SDomenico Cerasuolo else if (entry_offset < offset) 3282b281117SSeth Jennings node = node->rb_right; 3292b281117SSeth Jennings else 3302b281117SSeth Jennings return entry; 3312b281117SSeth Jennings } 3322b281117SSeth Jennings return NULL; 3332b281117SSeth Jennings } 3342b281117SSeth Jennings 3352b281117SSeth Jennings /* 3362b281117SSeth Jennings * In the case that a entry with the same offset is found, a pointer to 3372b281117SSeth Jennings * the existing entry is stored in dupentry and the function returns -EEXIST 3382b281117SSeth Jennings */ 3392b281117SSeth Jennings static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, 3402b281117SSeth Jennings struct zswap_entry **dupentry) 3412b281117SSeth Jennings { 3422b281117SSeth Jennings struct rb_node **link = &root->rb_node, *parent = NULL; 3432b281117SSeth Jennings struct zswap_entry *myentry; 3440bb48849SDomenico Cerasuolo pgoff_t myentry_offset, entry_offset = swp_offset(entry->swpentry); 3452b281117SSeth Jennings 3462b281117SSeth Jennings while (*link) { 3472b281117SSeth Jennings parent = *link; 3482b281117SSeth Jennings myentry = rb_entry(parent, struct zswap_entry, rbnode); 3490bb48849SDomenico Cerasuolo myentry_offset = swp_offset(myentry->swpentry); 3500bb48849SDomenico Cerasuolo if (myentry_offset > entry_offset) 3512b281117SSeth Jennings link = &(*link)->rb_left; 3520bb48849SDomenico Cerasuolo else if (myentry_offset < entry_offset) 3532b281117SSeth Jennings link = &(*link)->rb_right; 3542b281117SSeth Jennings else { 3552b281117SSeth Jennings *dupentry = myentry; 3562b281117SSeth Jennings return -EEXIST; 3572b281117SSeth Jennings } 3582b281117SSeth Jennings } 3592b281117SSeth Jennings rb_link_node(&entry->rbnode, parent, link); 3602b281117SSeth Jennings rb_insert_color(&entry->rbnode, root); 3612b281117SSeth Jennings return 0; 3622b281117SSeth Jennings } 3632b281117SSeth Jennings 36418a93707SYosry Ahmed static bool zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) 3650ab0abcfSWeijie Yang { 3660ab0abcfSWeijie Yang if (!RB_EMPTY_NODE(&entry->rbnode)) { 3670ab0abcfSWeijie Yang rb_erase(&entry->rbnode, root); 3680ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 36918a93707SYosry Ahmed return true; 3700ab0abcfSWeijie Yang } 37118a93707SYosry Ahmed return false; 3720ab0abcfSWeijie Yang } 3730ab0abcfSWeijie Yang 374b8cf32dcSYosry Ahmed static struct zpool *zswap_find_zpool(struct zswap_entry *entry) 375b8cf32dcSYosry Ahmed { 376b8cf32dcSYosry Ahmed int i = 0; 377b8cf32dcSYosry Ahmed 378b8cf32dcSYosry Ahmed if (ZSWAP_NR_ZPOOLS > 1) 379b8cf32dcSYosry Ahmed i = hash_ptr(entry, ilog2(ZSWAP_NR_ZPOOLS)); 380b8cf32dcSYosry Ahmed 381b8cf32dcSYosry Ahmed return entry->pool->zpools[i]; 382b8cf32dcSYosry Ahmed } 383b8cf32dcSYosry Ahmed 3840ab0abcfSWeijie Yang /* 38512d79d64SDan Streetman * Carries out the common pattern of freeing and entry's zpool allocation, 3860ab0abcfSWeijie Yang * freeing the entry itself, and decrementing the number of stored pages. 3870ab0abcfSWeijie Yang */ 38860105e12SMinchan Kim static void zswap_free_entry(struct zswap_entry *entry) 3890ab0abcfSWeijie Yang { 390f4840ccfSJohannes Weiner if (entry->objcg) { 391f4840ccfSJohannes Weiner obj_cgroup_uncharge_zswap(entry->objcg, entry->length); 392f4840ccfSJohannes Weiner obj_cgroup_put(entry->objcg); 393f4840ccfSJohannes Weiner } 394a85f878bSSrividya Desireddy if (!entry->length) 395a85f878bSSrividya Desireddy atomic_dec(&zswap_same_filled_pages); 396a85f878bSSrividya Desireddy else { 397f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 398f999f38bSDomenico Cerasuolo list_del(&entry->lru); 399f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 400b8cf32dcSYosry Ahmed zpool_free(zswap_find_zpool(entry), entry->handle); 401f1c54846SDan Streetman zswap_pool_put(entry->pool); 402a85f878bSSrividya Desireddy } 4030ab0abcfSWeijie Yang zswap_entry_cache_free(entry); 4040ab0abcfSWeijie Yang atomic_dec(&zswap_stored_pages); 405f1c54846SDan Streetman zswap_update_total_size(); 4060ab0abcfSWeijie Yang } 4070ab0abcfSWeijie Yang 4080ab0abcfSWeijie Yang /* caller must hold the tree lock */ 4090ab0abcfSWeijie Yang static void zswap_entry_get(struct zswap_entry *entry) 4100ab0abcfSWeijie Yang { 4110ab0abcfSWeijie Yang entry->refcount++; 4120ab0abcfSWeijie Yang } 4130ab0abcfSWeijie Yang 4140ab0abcfSWeijie Yang /* caller must hold the tree lock 4150ab0abcfSWeijie Yang * remove from the tree and free it, if nobody reference the entry 4160ab0abcfSWeijie Yang */ 4170ab0abcfSWeijie Yang static void zswap_entry_put(struct zswap_tree *tree, 4180ab0abcfSWeijie Yang struct zswap_entry *entry) 4190ab0abcfSWeijie Yang { 4200ab0abcfSWeijie Yang int refcount = --entry->refcount; 4210ab0abcfSWeijie Yang 42273108957SJohannes Weiner WARN_ON_ONCE(refcount < 0); 4230ab0abcfSWeijie Yang if (refcount == 0) { 42473108957SJohannes Weiner WARN_ON_ONCE(!RB_EMPTY_NODE(&entry->rbnode)); 42560105e12SMinchan Kim zswap_free_entry(entry); 4260ab0abcfSWeijie Yang } 4270ab0abcfSWeijie Yang } 4280ab0abcfSWeijie Yang 4290ab0abcfSWeijie Yang /* caller must hold the tree lock */ 4300ab0abcfSWeijie Yang static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, 4310ab0abcfSWeijie Yang pgoff_t offset) 4320ab0abcfSWeijie Yang { 433b0c9865fSAlexey Klimov struct zswap_entry *entry; 4340ab0abcfSWeijie Yang 4350ab0abcfSWeijie Yang entry = zswap_rb_search(root, offset); 4360ab0abcfSWeijie Yang if (entry) 4370ab0abcfSWeijie Yang zswap_entry_get(entry); 4380ab0abcfSWeijie Yang 4390ab0abcfSWeijie Yang return entry; 4400ab0abcfSWeijie Yang } 4410ab0abcfSWeijie Yang 4422b281117SSeth Jennings /********************************* 4432b281117SSeth Jennings * per-cpu code 4442b281117SSeth Jennings **********************************/ 4452b281117SSeth Jennings static DEFINE_PER_CPU(u8 *, zswap_dstmem); 4461ec3b5feSBarry Song /* 4471ec3b5feSBarry Song * If users dynamically change the zpool type and compressor at runtime, i.e. 4481ec3b5feSBarry Song * zswap is running, zswap can have more than one zpool on one cpu, but they 4491ec3b5feSBarry Song * are sharing dtsmem. So we need this mutex to be per-cpu. 4501ec3b5feSBarry Song */ 4511ec3b5feSBarry Song static DEFINE_PER_CPU(struct mutex *, zswap_mutex); 4522b281117SSeth Jennings 453ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_prepare(unsigned int cpu) 4542b281117SSeth Jennings { 4551ec3b5feSBarry Song struct mutex *mutex; 4562b281117SSeth Jennings u8 *dst; 4572b281117SSeth Jennings 45872d09633SEric Dumazet dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); 4592b2695f5SMarkus Elfring if (!dst) 460ad7ed770SSebastian Andrzej Siewior return -ENOMEM; 4612b2695f5SMarkus Elfring 4621ec3b5feSBarry Song mutex = kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu)); 4631ec3b5feSBarry Song if (!mutex) { 4641ec3b5feSBarry Song kfree(dst); 4651ec3b5feSBarry Song return -ENOMEM; 4661ec3b5feSBarry Song } 4671ec3b5feSBarry Song 4681ec3b5feSBarry Song mutex_init(mutex); 4692b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = dst; 4701ec3b5feSBarry Song per_cpu(zswap_mutex, cpu) = mutex; 471ad7ed770SSebastian Andrzej Siewior return 0; 472ad7ed770SSebastian Andrzej Siewior } 473ad7ed770SSebastian Andrzej Siewior 474ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_dead(unsigned int cpu) 475ad7ed770SSebastian Andrzej Siewior { 4761ec3b5feSBarry Song struct mutex *mutex; 477ad7ed770SSebastian Andrzej Siewior u8 *dst; 478ad7ed770SSebastian Andrzej Siewior 4791ec3b5feSBarry Song mutex = per_cpu(zswap_mutex, cpu); 4801ec3b5feSBarry Song kfree(mutex); 4811ec3b5feSBarry Song per_cpu(zswap_mutex, cpu) = NULL; 4821ec3b5feSBarry Song 4832b281117SSeth Jennings dst = per_cpu(zswap_dstmem, cpu); 4842b281117SSeth Jennings kfree(dst); 4852b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = NULL; 4862b281117SSeth Jennings 4872b281117SSeth Jennings return 0; 488f1c54846SDan Streetman } 489f1c54846SDan Streetman 490cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 491f1c54846SDan Streetman { 492cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 4931ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 4941ec3b5feSBarry Song struct crypto_acomp *acomp; 4951ec3b5feSBarry Song struct acomp_req *req; 496f1c54846SDan Streetman 4971ec3b5feSBarry Song acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu)); 4981ec3b5feSBarry Song if (IS_ERR(acomp)) { 4991ec3b5feSBarry Song pr_err("could not alloc crypto acomp %s : %ld\n", 5001ec3b5feSBarry Song pool->tfm_name, PTR_ERR(acomp)); 5011ec3b5feSBarry Song return PTR_ERR(acomp); 5021ec3b5feSBarry Song } 5031ec3b5feSBarry Song acomp_ctx->acomp = acomp; 504cab7a7e5SSebastian Andrzej Siewior 5051ec3b5feSBarry Song req = acomp_request_alloc(acomp_ctx->acomp); 5061ec3b5feSBarry Song if (!req) { 5071ec3b5feSBarry Song pr_err("could not alloc crypto acomp_request %s\n", 5081ec3b5feSBarry Song pool->tfm_name); 5091ec3b5feSBarry Song crypto_free_acomp(acomp_ctx->acomp); 510cab7a7e5SSebastian Andrzej Siewior return -ENOMEM; 511f1c54846SDan Streetman } 5121ec3b5feSBarry Song acomp_ctx->req = req; 5131ec3b5feSBarry Song 5141ec3b5feSBarry Song crypto_init_wait(&acomp_ctx->wait); 5151ec3b5feSBarry Song /* 5161ec3b5feSBarry Song * if the backend of acomp is async zip, crypto_req_done() will wakeup 5171ec3b5feSBarry Song * crypto_wait_req(); if the backend of acomp is scomp, the callback 5181ec3b5feSBarry Song * won't be called, crypto_wait_req() will return without blocking. 5191ec3b5feSBarry Song */ 5201ec3b5feSBarry Song acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 5211ec3b5feSBarry Song crypto_req_done, &acomp_ctx->wait); 5221ec3b5feSBarry Song 5231ec3b5feSBarry Song acomp_ctx->mutex = per_cpu(zswap_mutex, cpu); 5241ec3b5feSBarry Song acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu); 5251ec3b5feSBarry Song 526cab7a7e5SSebastian Andrzej Siewior return 0; 527cab7a7e5SSebastian Andrzej Siewior } 528cab7a7e5SSebastian Andrzej Siewior 529cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 530cab7a7e5SSebastian Andrzej Siewior { 531cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 5321ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 533cab7a7e5SSebastian Andrzej Siewior 5341ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx)) { 5351ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx->req)) 5361ec3b5feSBarry Song acomp_request_free(acomp_ctx->req); 5371ec3b5feSBarry Song if (!IS_ERR_OR_NULL(acomp_ctx->acomp)) 5381ec3b5feSBarry Song crypto_free_acomp(acomp_ctx->acomp); 5391ec3b5feSBarry Song } 5401ec3b5feSBarry Song 541f1c54846SDan Streetman return 0; 542f1c54846SDan Streetman } 543f1c54846SDan Streetman 544f1c54846SDan Streetman /********************************* 545f1c54846SDan Streetman * pool functions 546f1c54846SDan Streetman **********************************/ 547f1c54846SDan Streetman 548f1c54846SDan Streetman static struct zswap_pool *__zswap_pool_current(void) 549f1c54846SDan Streetman { 550f1c54846SDan Streetman struct zswap_pool *pool; 551f1c54846SDan Streetman 552f1c54846SDan Streetman pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 553ae3d89a7SDan Streetman WARN_ONCE(!pool && zswap_has_pool, 554ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 555f1c54846SDan Streetman 556f1c54846SDan Streetman return pool; 557f1c54846SDan Streetman } 558f1c54846SDan Streetman 559f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current(void) 560f1c54846SDan Streetman { 561f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 562f1c54846SDan Streetman 563f1c54846SDan Streetman return __zswap_pool_current(); 564f1c54846SDan Streetman } 565f1c54846SDan Streetman 566f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current_get(void) 567f1c54846SDan Streetman { 568f1c54846SDan Streetman struct zswap_pool *pool; 569f1c54846SDan Streetman 570f1c54846SDan Streetman rcu_read_lock(); 571f1c54846SDan Streetman 572f1c54846SDan Streetman pool = __zswap_pool_current(); 573ae3d89a7SDan Streetman if (!zswap_pool_get(pool)) 574f1c54846SDan Streetman pool = NULL; 575f1c54846SDan Streetman 576f1c54846SDan Streetman rcu_read_unlock(); 577f1c54846SDan Streetman 578f1c54846SDan Streetman return pool; 579f1c54846SDan Streetman } 580f1c54846SDan Streetman 581f1c54846SDan Streetman static struct zswap_pool *zswap_pool_last_get(void) 582f1c54846SDan Streetman { 583f1c54846SDan Streetman struct zswap_pool *pool, *last = NULL; 584f1c54846SDan Streetman 585f1c54846SDan Streetman rcu_read_lock(); 586f1c54846SDan Streetman 587f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 588f1c54846SDan Streetman last = pool; 589ae3d89a7SDan Streetman WARN_ONCE(!last && zswap_has_pool, 590ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 591ae3d89a7SDan Streetman if (!zswap_pool_get(last)) 592f1c54846SDan Streetman last = NULL; 593f1c54846SDan Streetman 594f1c54846SDan Streetman rcu_read_unlock(); 595f1c54846SDan Streetman 596f1c54846SDan Streetman return last; 597f1c54846SDan Streetman } 598f1c54846SDan Streetman 5998bc8b228SDan Streetman /* type and compressor must be null-terminated */ 600f1c54846SDan Streetman static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 601f1c54846SDan Streetman { 602f1c54846SDan Streetman struct zswap_pool *pool; 603f1c54846SDan Streetman 604f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 605f1c54846SDan Streetman 606f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) { 6078bc8b228SDan Streetman if (strcmp(pool->tfm_name, compressor)) 608f1c54846SDan Streetman continue; 609b8cf32dcSYosry Ahmed /* all zpools share the same type */ 610b8cf32dcSYosry Ahmed if (strcmp(zpool_get_type(pool->zpools[0]), type)) 611f1c54846SDan Streetman continue; 612f1c54846SDan Streetman /* if we can't get it, it's about to be destroyed */ 613f1c54846SDan Streetman if (!zswap_pool_get(pool)) 614f1c54846SDan Streetman continue; 615f1c54846SDan Streetman return pool; 616f1c54846SDan Streetman } 617f1c54846SDan Streetman 618f1c54846SDan Streetman return NULL; 619f1c54846SDan Streetman } 620f1c54846SDan Streetman 62118a93707SYosry Ahmed /* 62218a93707SYosry Ahmed * If the entry is still valid in the tree, drop the initial ref and remove it 62318a93707SYosry Ahmed * from the tree. This function must be called with an additional ref held, 62418a93707SYosry Ahmed * otherwise it may race with another invalidation freeing the entry. 62518a93707SYosry Ahmed */ 626418fd29dSDomenico Cerasuolo static void zswap_invalidate_entry(struct zswap_tree *tree, 627418fd29dSDomenico Cerasuolo struct zswap_entry *entry) 628418fd29dSDomenico Cerasuolo { 62918a93707SYosry Ahmed if (zswap_rb_erase(&tree->rbroot, entry)) 630418fd29dSDomenico Cerasuolo zswap_entry_put(tree, entry); 631418fd29dSDomenico Cerasuolo } 632418fd29dSDomenico Cerasuolo 633f999f38bSDomenico Cerasuolo static int zswap_reclaim_entry(struct zswap_pool *pool) 634f999f38bSDomenico Cerasuolo { 635f999f38bSDomenico Cerasuolo struct zswap_entry *entry; 636f999f38bSDomenico Cerasuolo struct zswap_tree *tree; 637f999f38bSDomenico Cerasuolo pgoff_t swpoffset; 638f999f38bSDomenico Cerasuolo int ret; 639f999f38bSDomenico Cerasuolo 640f999f38bSDomenico Cerasuolo /* Get an entry off the LRU */ 641f999f38bSDomenico Cerasuolo spin_lock(&pool->lru_lock); 642f999f38bSDomenico Cerasuolo if (list_empty(&pool->lru)) { 643f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 644f999f38bSDomenico Cerasuolo return -EINVAL; 645f999f38bSDomenico Cerasuolo } 646f999f38bSDomenico Cerasuolo entry = list_last_entry(&pool->lru, struct zswap_entry, lru); 647f999f38bSDomenico Cerasuolo list_del_init(&entry->lru); 648f999f38bSDomenico Cerasuolo /* 649f999f38bSDomenico Cerasuolo * Once the lru lock is dropped, the entry might get freed. The 650f999f38bSDomenico Cerasuolo * swpoffset is copied to the stack, and entry isn't deref'd again 651f999f38bSDomenico Cerasuolo * until the entry is verified to still be alive in the tree. 652f999f38bSDomenico Cerasuolo */ 6530bb48849SDomenico Cerasuolo swpoffset = swp_offset(entry->swpentry); 6540bb48849SDomenico Cerasuolo tree = zswap_trees[swp_type(entry->swpentry)]; 655f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 656f999f38bSDomenico Cerasuolo 657f999f38bSDomenico Cerasuolo /* Check for invalidate() race */ 658f999f38bSDomenico Cerasuolo spin_lock(&tree->lock); 659f999f38bSDomenico Cerasuolo if (entry != zswap_rb_search(&tree->rbroot, swpoffset)) { 660f999f38bSDomenico Cerasuolo ret = -EAGAIN; 661f999f38bSDomenico Cerasuolo goto unlock; 662f999f38bSDomenico Cerasuolo } 663f999f38bSDomenico Cerasuolo /* Hold a reference to prevent a free during writeback */ 664f999f38bSDomenico Cerasuolo zswap_entry_get(entry); 665f999f38bSDomenico Cerasuolo spin_unlock(&tree->lock); 666f999f38bSDomenico Cerasuolo 6670bb48849SDomenico Cerasuolo ret = zswap_writeback_entry(entry, tree); 668f999f38bSDomenico Cerasuolo 669f999f38bSDomenico Cerasuolo spin_lock(&tree->lock); 670f999f38bSDomenico Cerasuolo if (ret) { 671f999f38bSDomenico Cerasuolo /* Writeback failed, put entry back on LRU */ 672f999f38bSDomenico Cerasuolo spin_lock(&pool->lru_lock); 673f999f38bSDomenico Cerasuolo list_move(&entry->lru, &pool->lru); 674f999f38bSDomenico Cerasuolo spin_unlock(&pool->lru_lock); 675ff9d5ba2SDomenico Cerasuolo goto put_unlock; 676f999f38bSDomenico Cerasuolo } 677f999f38bSDomenico Cerasuolo 678418fd29dSDomenico Cerasuolo /* 679418fd29dSDomenico Cerasuolo * Writeback started successfully, the page now belongs to the 680418fd29dSDomenico Cerasuolo * swapcache. Drop the entry from zswap - unless invalidate already 681418fd29dSDomenico Cerasuolo * took it out while we had the tree->lock released for IO. 682418fd29dSDomenico Cerasuolo */ 683418fd29dSDomenico Cerasuolo zswap_invalidate_entry(tree, entry); 684ff9d5ba2SDomenico Cerasuolo 685ff9d5ba2SDomenico Cerasuolo put_unlock: 686f999f38bSDomenico Cerasuolo /* Drop local reference */ 687f999f38bSDomenico Cerasuolo zswap_entry_put(tree, entry); 688f999f38bSDomenico Cerasuolo unlock: 689f999f38bSDomenico Cerasuolo spin_unlock(&tree->lock); 690f999f38bSDomenico Cerasuolo return ret ? -EAGAIN : 0; 691f999f38bSDomenico Cerasuolo } 692f999f38bSDomenico Cerasuolo 69345190f01SVitaly Wool static void shrink_worker(struct work_struct *w) 69445190f01SVitaly Wool { 69545190f01SVitaly Wool struct zswap_pool *pool = container_of(w, typeof(*pool), 69645190f01SVitaly Wool shrink_work); 697e0228d59SDomenico Cerasuolo int ret, failures = 0; 69845190f01SVitaly Wool 699e0228d59SDomenico Cerasuolo do { 700f999f38bSDomenico Cerasuolo ret = zswap_reclaim_entry(pool); 701e0228d59SDomenico Cerasuolo if (ret) { 70245190f01SVitaly Wool zswap_reject_reclaim_fail++; 703e0228d59SDomenico Cerasuolo if (ret != -EAGAIN) 704e0228d59SDomenico Cerasuolo break; 705e0228d59SDomenico Cerasuolo if (++failures == MAX_RECLAIM_RETRIES) 706e0228d59SDomenico Cerasuolo break; 707e0228d59SDomenico Cerasuolo } 708e0228d59SDomenico Cerasuolo cond_resched(); 709e0228d59SDomenico Cerasuolo } while (!zswap_can_accept()); 71045190f01SVitaly Wool zswap_pool_put(pool); 71145190f01SVitaly Wool } 71245190f01SVitaly Wool 713f1c54846SDan Streetman static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 714f1c54846SDan Streetman { 715b8cf32dcSYosry Ahmed int i; 716f1c54846SDan Streetman struct zswap_pool *pool; 71732a4e169SDan Streetman char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 718d0164adcSMel Gorman gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 719cab7a7e5SSebastian Andrzej Siewior int ret; 720f1c54846SDan Streetman 721bae21db8SDan Streetman if (!zswap_has_pool) { 722bae21db8SDan Streetman /* if either are unset, pool initialization failed, and we 723bae21db8SDan Streetman * need both params to be set correctly before trying to 724bae21db8SDan Streetman * create a pool. 725bae21db8SDan Streetman */ 726bae21db8SDan Streetman if (!strcmp(type, ZSWAP_PARAM_UNSET)) 727bae21db8SDan Streetman return NULL; 728bae21db8SDan Streetman if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 729bae21db8SDan Streetman return NULL; 730bae21db8SDan Streetman } 731bae21db8SDan Streetman 732f1c54846SDan Streetman pool = kzalloc(sizeof(*pool), GFP_KERNEL); 733f4ae0ce0SMarkus Elfring if (!pool) 734f1c54846SDan Streetman return NULL; 735f1c54846SDan Streetman 736b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) { 73732a4e169SDan Streetman /* unique name for each pool specifically required by zsmalloc */ 738b8cf32dcSYosry Ahmed snprintf(name, 38, "zswap%x", 739b8cf32dcSYosry Ahmed atomic_inc_return(&zswap_pools_count)); 74032a4e169SDan Streetman 741b8cf32dcSYosry Ahmed pool->zpools[i] = zpool_create_pool(type, name, gfp); 742b8cf32dcSYosry Ahmed if (!pool->zpools[i]) { 743f1c54846SDan Streetman pr_err("%s zpool not available\n", type); 744f1c54846SDan Streetman goto error; 745f1c54846SDan Streetman } 746b8cf32dcSYosry Ahmed } 747b8cf32dcSYosry Ahmed pr_debug("using %s zpool\n", zpool_get_type(pool->zpools[0])); 748f1c54846SDan Streetman 74979cd4202SZhiyuan Dai strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 7501ec3b5feSBarry Song 7511ec3b5feSBarry Song pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx); 7521ec3b5feSBarry Song if (!pool->acomp_ctx) { 753f1c54846SDan Streetman pr_err("percpu alloc failed\n"); 754f1c54846SDan Streetman goto error; 755f1c54846SDan Streetman } 756f1c54846SDan Streetman 757cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, 758cab7a7e5SSebastian Andrzej Siewior &pool->node); 759cab7a7e5SSebastian Andrzej Siewior if (ret) 760f1c54846SDan Streetman goto error; 761f1c54846SDan Streetman pr_debug("using %s compressor\n", pool->tfm_name); 762f1c54846SDan Streetman 763f1c54846SDan Streetman /* being the current pool takes 1 ref; this func expects the 764f1c54846SDan Streetman * caller to always add the new pool as the current pool 765f1c54846SDan Streetman */ 766f1c54846SDan Streetman kref_init(&pool->kref); 767f1c54846SDan Streetman INIT_LIST_HEAD(&pool->list); 768f999f38bSDomenico Cerasuolo INIT_LIST_HEAD(&pool->lru); 769f999f38bSDomenico Cerasuolo spin_lock_init(&pool->lru_lock); 77045190f01SVitaly Wool INIT_WORK(&pool->shrink_work, shrink_worker); 771f1c54846SDan Streetman 772f1c54846SDan Streetman zswap_pool_debug("created", pool); 773f1c54846SDan Streetman 774f1c54846SDan Streetman return pool; 775f1c54846SDan Streetman 776f1c54846SDan Streetman error: 7771ec3b5feSBarry Song if (pool->acomp_ctx) 7781ec3b5feSBarry Song free_percpu(pool->acomp_ctx); 779b8cf32dcSYosry Ahmed while (i--) 780b8cf32dcSYosry Ahmed zpool_destroy_pool(pool->zpools[i]); 781f1c54846SDan Streetman kfree(pool); 782f1c54846SDan Streetman return NULL; 783f1c54846SDan Streetman } 784f1c54846SDan Streetman 785141fdeecSLiu Shixin static struct zswap_pool *__zswap_pool_create_fallback(void) 786f1c54846SDan Streetman { 787bae21db8SDan Streetman bool has_comp, has_zpool; 788bae21db8SDan Streetman 7891ec3b5feSBarry Song has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 790bb8b93b5SMaciej S. Szmigiero if (!has_comp && strcmp(zswap_compressor, 791bb8b93b5SMaciej S. Szmigiero CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 792f1c54846SDan Streetman pr_err("compressor %s not available, using default %s\n", 793bb8b93b5SMaciej S. Szmigiero zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); 794c99b42c3SDan Streetman param_free_charp(&zswap_compressor); 795bb8b93b5SMaciej S. Szmigiero zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 7961ec3b5feSBarry Song has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 797f1c54846SDan Streetman } 798bae21db8SDan Streetman if (!has_comp) { 799bae21db8SDan Streetman pr_err("default compressor %s not available\n", 800bae21db8SDan Streetman zswap_compressor); 801bae21db8SDan Streetman param_free_charp(&zswap_compressor); 802bae21db8SDan Streetman zswap_compressor = ZSWAP_PARAM_UNSET; 803c99b42c3SDan Streetman } 804bae21db8SDan Streetman 805bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 806bb8b93b5SMaciej S. Szmigiero if (!has_zpool && strcmp(zswap_zpool_type, 807bb8b93b5SMaciej S. Szmigiero CONFIG_ZSWAP_ZPOOL_DEFAULT)) { 808f1c54846SDan Streetman pr_err("zpool %s not available, using default %s\n", 809bb8b93b5SMaciej S. Szmigiero zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); 810c99b42c3SDan Streetman param_free_charp(&zswap_zpool_type); 811bb8b93b5SMaciej S. Szmigiero zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 812bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 813f1c54846SDan Streetman } 814bae21db8SDan Streetman if (!has_zpool) { 815bae21db8SDan Streetman pr_err("default zpool %s not available\n", 816bae21db8SDan Streetman zswap_zpool_type); 817bae21db8SDan Streetman param_free_charp(&zswap_zpool_type); 818bae21db8SDan Streetman zswap_zpool_type = ZSWAP_PARAM_UNSET; 819bae21db8SDan Streetman } 820bae21db8SDan Streetman 821bae21db8SDan Streetman if (!has_comp || !has_zpool) 822bae21db8SDan Streetman return NULL; 823f1c54846SDan Streetman 824f1c54846SDan Streetman return zswap_pool_create(zswap_zpool_type, zswap_compressor); 825f1c54846SDan Streetman } 826f1c54846SDan Streetman 827f1c54846SDan Streetman static void zswap_pool_destroy(struct zswap_pool *pool) 828f1c54846SDan Streetman { 829b8cf32dcSYosry Ahmed int i; 830b8cf32dcSYosry Ahmed 831f1c54846SDan Streetman zswap_pool_debug("destroying", pool); 832f1c54846SDan Streetman 833cab7a7e5SSebastian Andrzej Siewior cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 8341ec3b5feSBarry Song free_percpu(pool->acomp_ctx); 835b8cf32dcSYosry Ahmed for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) 836b8cf32dcSYosry Ahmed zpool_destroy_pool(pool->zpools[i]); 837f1c54846SDan Streetman kfree(pool); 838f1c54846SDan Streetman } 839f1c54846SDan Streetman 840f1c54846SDan Streetman static int __must_check zswap_pool_get(struct zswap_pool *pool) 841f1c54846SDan Streetman { 842ae3d89a7SDan Streetman if (!pool) 843ae3d89a7SDan Streetman return 0; 844ae3d89a7SDan Streetman 845f1c54846SDan Streetman return kref_get_unless_zero(&pool->kref); 846f1c54846SDan Streetman } 847f1c54846SDan Streetman 848200867afSDan Streetman static void __zswap_pool_release(struct work_struct *work) 849f1c54846SDan Streetman { 85045190f01SVitaly Wool struct zswap_pool *pool = container_of(work, typeof(*pool), 85145190f01SVitaly Wool release_work); 852200867afSDan Streetman 853200867afSDan Streetman synchronize_rcu(); 854f1c54846SDan Streetman 855f1c54846SDan Streetman /* nobody should have been able to get a kref... */ 856f1c54846SDan Streetman WARN_ON(kref_get_unless_zero(&pool->kref)); 857f1c54846SDan Streetman 858f1c54846SDan Streetman /* pool is now off zswap_pools list and has no references. */ 859f1c54846SDan Streetman zswap_pool_destroy(pool); 860f1c54846SDan Streetman } 861f1c54846SDan Streetman 862f1c54846SDan Streetman static void __zswap_pool_empty(struct kref *kref) 863f1c54846SDan Streetman { 864f1c54846SDan Streetman struct zswap_pool *pool; 865f1c54846SDan Streetman 866f1c54846SDan Streetman pool = container_of(kref, typeof(*pool), kref); 867f1c54846SDan Streetman 868f1c54846SDan Streetman spin_lock(&zswap_pools_lock); 869f1c54846SDan Streetman 870f1c54846SDan Streetman WARN_ON(pool == zswap_pool_current()); 871f1c54846SDan Streetman 872f1c54846SDan Streetman list_del_rcu(&pool->list); 873200867afSDan Streetman 87445190f01SVitaly Wool INIT_WORK(&pool->release_work, __zswap_pool_release); 87545190f01SVitaly Wool schedule_work(&pool->release_work); 876f1c54846SDan Streetman 877f1c54846SDan Streetman spin_unlock(&zswap_pools_lock); 878f1c54846SDan Streetman } 879f1c54846SDan Streetman 880f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool) 881f1c54846SDan Streetman { 882f1c54846SDan Streetman kref_put(&pool->kref, __zswap_pool_empty); 8832b281117SSeth Jennings } 8842b281117SSeth Jennings 8852b281117SSeth Jennings /********************************* 88690b0fc26SDan Streetman * param callbacks 88790b0fc26SDan Streetman **********************************/ 88890b0fc26SDan Streetman 889141fdeecSLiu Shixin static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) 890141fdeecSLiu Shixin { 891141fdeecSLiu Shixin /* no change required */ 892141fdeecSLiu Shixin if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 893141fdeecSLiu Shixin return false; 894141fdeecSLiu Shixin return true; 895141fdeecSLiu Shixin } 896141fdeecSLiu Shixin 897c99b42c3SDan Streetman /* val must be a null-terminated string */ 89890b0fc26SDan Streetman static int __zswap_param_set(const char *val, const struct kernel_param *kp, 89990b0fc26SDan Streetman char *type, char *compressor) 90090b0fc26SDan Streetman { 90190b0fc26SDan Streetman struct zswap_pool *pool, *put_pool = NULL; 902c99b42c3SDan Streetman char *s = strstrip((char *)val); 903141fdeecSLiu Shixin int ret = 0; 904141fdeecSLiu Shixin bool new_pool = false; 90590b0fc26SDan Streetman 906141fdeecSLiu Shixin mutex_lock(&zswap_init_lock); 9079021ccecSLiu Shixin switch (zswap_init_state) { 9089021ccecSLiu Shixin case ZSWAP_UNINIT: 90990b0fc26SDan Streetman /* if this is load-time (pre-init) param setting, 91090b0fc26SDan Streetman * don't create a pool; that's done during init. 91190b0fc26SDan Streetman */ 912141fdeecSLiu Shixin ret = param_set_charp(s, kp); 913141fdeecSLiu Shixin break; 9149021ccecSLiu Shixin case ZSWAP_INIT_SUCCEED: 915141fdeecSLiu Shixin new_pool = zswap_pool_changed(s, kp); 9169021ccecSLiu Shixin break; 9179021ccecSLiu Shixin case ZSWAP_INIT_FAILED: 9189021ccecSLiu Shixin pr_err("can't set param, initialization failed\n"); 919141fdeecSLiu Shixin ret = -ENODEV; 9209021ccecSLiu Shixin } 921141fdeecSLiu Shixin mutex_unlock(&zswap_init_lock); 922141fdeecSLiu Shixin 923141fdeecSLiu Shixin /* no need to create a new pool, return directly */ 924141fdeecSLiu Shixin if (!new_pool) 925141fdeecSLiu Shixin return ret; 92690b0fc26SDan Streetman 92790b0fc26SDan Streetman if (!type) { 928c99b42c3SDan Streetman if (!zpool_has_pool(s)) { 929c99b42c3SDan Streetman pr_err("zpool %s not available\n", s); 930c99b42c3SDan Streetman return -ENOENT; 931c99b42c3SDan Streetman } 93290b0fc26SDan Streetman type = s; 93390b0fc26SDan Streetman } else if (!compressor) { 9341ec3b5feSBarry Song if (!crypto_has_acomp(s, 0, 0)) { 935c99b42c3SDan Streetman pr_err("compressor %s not available\n", s); 93690b0fc26SDan Streetman return -ENOENT; 93790b0fc26SDan Streetman } 938c99b42c3SDan Streetman compressor = s; 939c99b42c3SDan Streetman } else { 940c99b42c3SDan Streetman WARN_ON(1); 941c99b42c3SDan Streetman return -EINVAL; 94290b0fc26SDan Streetman } 94390b0fc26SDan Streetman 94490b0fc26SDan Streetman spin_lock(&zswap_pools_lock); 94590b0fc26SDan Streetman 94690b0fc26SDan Streetman pool = zswap_pool_find_get(type, compressor); 94790b0fc26SDan Streetman if (pool) { 94890b0fc26SDan Streetman zswap_pool_debug("using existing", pool); 949fd5bb66cSDan Streetman WARN_ON(pool == zswap_pool_current()); 95090b0fc26SDan Streetman list_del_rcu(&pool->list); 95190b0fc26SDan Streetman } 95290b0fc26SDan Streetman 953fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 954fd5bb66cSDan Streetman 955fd5bb66cSDan Streetman if (!pool) 956fd5bb66cSDan Streetman pool = zswap_pool_create(type, compressor); 957fd5bb66cSDan Streetman 95890b0fc26SDan Streetman if (pool) 959c99b42c3SDan Streetman ret = param_set_charp(s, kp); 96090b0fc26SDan Streetman else 96190b0fc26SDan Streetman ret = -EINVAL; 96290b0fc26SDan Streetman 963fd5bb66cSDan Streetman spin_lock(&zswap_pools_lock); 964fd5bb66cSDan Streetman 96590b0fc26SDan Streetman if (!ret) { 96690b0fc26SDan Streetman put_pool = zswap_pool_current(); 96790b0fc26SDan Streetman list_add_rcu(&pool->list, &zswap_pools); 968ae3d89a7SDan Streetman zswap_has_pool = true; 96990b0fc26SDan Streetman } else if (pool) { 97090b0fc26SDan Streetman /* add the possibly pre-existing pool to the end of the pools 97190b0fc26SDan Streetman * list; if it's new (and empty) then it'll be removed and 97290b0fc26SDan Streetman * destroyed by the put after we drop the lock 97390b0fc26SDan Streetman */ 97490b0fc26SDan Streetman list_add_tail_rcu(&pool->list, &zswap_pools); 97590b0fc26SDan Streetman put_pool = pool; 976fd5bb66cSDan Streetman } 977fd5bb66cSDan Streetman 978fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 979fd5bb66cSDan Streetman 980fd5bb66cSDan Streetman if (!zswap_has_pool && !pool) { 981ae3d89a7SDan Streetman /* if initial pool creation failed, and this pool creation also 982ae3d89a7SDan Streetman * failed, maybe both compressor and zpool params were bad. 983ae3d89a7SDan Streetman * Allow changing this param, so pool creation will succeed 984ae3d89a7SDan Streetman * when the other param is changed. We already verified this 9851ec3b5feSBarry Song * param is ok in the zpool_has_pool() or crypto_has_acomp() 986ae3d89a7SDan Streetman * checks above. 987ae3d89a7SDan Streetman */ 988ae3d89a7SDan Streetman ret = param_set_charp(s, kp); 98990b0fc26SDan Streetman } 99090b0fc26SDan Streetman 99190b0fc26SDan Streetman /* drop the ref from either the old current pool, 99290b0fc26SDan Streetman * or the new pool we failed to add 99390b0fc26SDan Streetman */ 99490b0fc26SDan Streetman if (put_pool) 99590b0fc26SDan Streetman zswap_pool_put(put_pool); 99690b0fc26SDan Streetman 99790b0fc26SDan Streetman return ret; 99890b0fc26SDan Streetman } 99990b0fc26SDan Streetman 100090b0fc26SDan Streetman static int zswap_compressor_param_set(const char *val, 100190b0fc26SDan Streetman const struct kernel_param *kp) 100290b0fc26SDan Streetman { 100390b0fc26SDan Streetman return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 100490b0fc26SDan Streetman } 100590b0fc26SDan Streetman 100690b0fc26SDan Streetman static int zswap_zpool_param_set(const char *val, 100790b0fc26SDan Streetman const struct kernel_param *kp) 100890b0fc26SDan Streetman { 100990b0fc26SDan Streetman return __zswap_param_set(val, kp, NULL, zswap_compressor); 101090b0fc26SDan Streetman } 101190b0fc26SDan Streetman 1012d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *val, 1013d7b028f5SDan Streetman const struct kernel_param *kp) 1014d7b028f5SDan Streetman { 1015141fdeecSLiu Shixin int ret = -ENODEV; 1016d7b028f5SDan Streetman 1017141fdeecSLiu Shixin /* if this is load-time (pre-init) param setting, only set param. */ 1018141fdeecSLiu Shixin if (system_state != SYSTEM_RUNNING) 1019d7b028f5SDan Streetman return param_set_bool(val, kp); 1020141fdeecSLiu Shixin 1021141fdeecSLiu Shixin mutex_lock(&zswap_init_lock); 10229021ccecSLiu Shixin switch (zswap_init_state) { 10239021ccecSLiu Shixin case ZSWAP_UNINIT: 1024141fdeecSLiu Shixin if (zswap_setup()) 1025141fdeecSLiu Shixin break; 1026141fdeecSLiu Shixin fallthrough; 10279021ccecSLiu Shixin case ZSWAP_INIT_SUCCEED: 1028141fdeecSLiu Shixin if (!zswap_has_pool) 10299021ccecSLiu Shixin pr_err("can't enable, no pool configured\n"); 1030141fdeecSLiu Shixin else 1031141fdeecSLiu Shixin ret = param_set_bool(val, kp); 1032141fdeecSLiu Shixin break; 10339021ccecSLiu Shixin case ZSWAP_INIT_FAILED: 1034d7b028f5SDan Streetman pr_err("can't enable, initialization failed\n"); 1035d7b028f5SDan Streetman } 1036141fdeecSLiu Shixin mutex_unlock(&zswap_init_lock); 1037141fdeecSLiu Shixin 1038141fdeecSLiu Shixin return ret; 1039d7b028f5SDan Streetman } 1040d7b028f5SDan Streetman 104190b0fc26SDan Streetman /********************************* 10422b281117SSeth Jennings * writeback code 10432b281117SSeth Jennings **********************************/ 10442b281117SSeth Jennings /* 10452b281117SSeth Jennings * Attempts to free an entry by adding a page to the swap cache, 10462b281117SSeth Jennings * decompressing the entry data into the page, and issuing a 10472b281117SSeth Jennings * bio write to write the page back to the swap device. 10482b281117SSeth Jennings * 10492b281117SSeth Jennings * This can be thought of as a "resumed writeback" of the page 10502b281117SSeth Jennings * to the swap device. We are basically resuming the same swap 105142c06a0eSJohannes Weiner * writeback path that was intercepted with the zswap_store() 10522b281117SSeth Jennings * in the first place. After the page has been decompressed into 10532b281117SSeth Jennings * the swap cache, the compressed version stored by zswap can be 10542b281117SSeth Jennings * freed. 10552b281117SSeth Jennings */ 10560bb48849SDomenico Cerasuolo static int zswap_writeback_entry(struct zswap_entry *entry, 1057ff9d5ba2SDomenico Cerasuolo struct zswap_tree *tree) 10582b281117SSeth Jennings { 10590bb48849SDomenico Cerasuolo swp_entry_t swpentry = entry->swpentry; 10602b281117SSeth Jennings struct page *page; 1061*ddc1a5cbSHugh Dickins struct mempolicy *mpol; 10621ec3b5feSBarry Song struct scatterlist input, output; 10631ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1064b8cf32dcSYosry Ahmed struct zpool *pool = zswap_find_zpool(entry); 106598804a94SJohannes Weiner bool page_was_allocated; 1066fc6697a8STian Tao u8 *src, *tmp = NULL; 10672b281117SSeth Jennings unsigned int dlen; 10680ab0abcfSWeijie Yang int ret; 10692b281117SSeth Jennings struct writeback_control wbc = { 10702b281117SSeth Jennings .sync_mode = WB_SYNC_NONE, 10712b281117SSeth Jennings }; 10722b281117SSeth Jennings 1073fc6697a8STian Tao if (!zpool_can_sleep_mapped(pool)) { 10748d9b6370SSergey Senozhatsky tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); 1075fc6697a8STian Tao if (!tmp) 1076fc6697a8STian Tao return -ENOMEM; 1077fc6697a8STian Tao } 1078fc6697a8STian Tao 10792b281117SSeth Jennings /* try to allocate swap cache page */ 1080*ddc1a5cbSHugh Dickins mpol = get_task_policy(current); 1081*ddc1a5cbSHugh Dickins page = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol, 1082*ddc1a5cbSHugh Dickins NO_INTERLEAVE_INDEX, &page_was_allocated); 108398804a94SJohannes Weiner if (!page) { 10842b281117SSeth Jennings ret = -ENOMEM; 10852b281117SSeth Jennings goto fail; 108698804a94SJohannes Weiner } 10872b281117SSeth Jennings 108898804a94SJohannes Weiner /* Found an existing page, we raced with load/swapin */ 108998804a94SJohannes Weiner if (!page_was_allocated) { 109009cbfeafSKirill A. Shutemov put_page(page); 10912b281117SSeth Jennings ret = -EEXIST; 10922b281117SSeth Jennings goto fail; 109398804a94SJohannes Weiner } 10942b281117SSeth Jennings 109504fc7816SDomenico Cerasuolo /* 109698804a94SJohannes Weiner * Page is locked, and the swapcache is now secured against 109798804a94SJohannes Weiner * concurrent swapping to and from the slot. Verify that the 109898804a94SJohannes Weiner * swap entry hasn't been invalidated and recycled behind our 109998804a94SJohannes Weiner * backs (our zswap_entry reference doesn't prevent that), to 110098804a94SJohannes Weiner * avoid overwriting a new swap page with old compressed data. 110104fc7816SDomenico Cerasuolo */ 110204fc7816SDomenico Cerasuolo spin_lock(&tree->lock); 11030bb48849SDomenico Cerasuolo if (zswap_rb_search(&tree->rbroot, swp_offset(entry->swpentry)) != entry) { 110404fc7816SDomenico Cerasuolo spin_unlock(&tree->lock); 110504fc7816SDomenico Cerasuolo delete_from_swap_cache(page_folio(page)); 110604fc7816SDomenico Cerasuolo ret = -ENOMEM; 110704fc7816SDomenico Cerasuolo goto fail; 110804fc7816SDomenico Cerasuolo } 110904fc7816SDomenico Cerasuolo spin_unlock(&tree->lock); 111004fc7816SDomenico Cerasuolo 11112b281117SSeth Jennings /* decompress */ 11121ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 11132b281117SSeth Jennings dlen = PAGE_SIZE; 1114fc6697a8STian Tao 11150bb48849SDomenico Cerasuolo src = zpool_map_handle(pool, entry->handle, ZPOOL_MM_RO); 11166b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) { 11176b3379e8SJohannes Weiner memcpy(tmp, src, entry->length); 11186b3379e8SJohannes Weiner src = tmp; 1119ff9d5ba2SDomenico Cerasuolo zpool_unmap_handle(pool, entry->handle); 11206b3379e8SJohannes Weiner } 11216b3379e8SJohannes Weiner 11221ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 11231ec3b5feSBarry Song sg_init_one(&input, src, entry->length); 11241ec3b5feSBarry Song sg_init_table(&output, 1); 11251ec3b5feSBarry Song sg_set_page(&output, page, PAGE_SIZE, 0); 11261ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen); 11271ec3b5feSBarry Song ret = crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait); 11281ec3b5feSBarry Song dlen = acomp_ctx->req->dlen; 11291ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 11301ec3b5feSBarry Song 11316b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) 11326b3379e8SJohannes Weiner kfree(tmp); 11336b3379e8SJohannes Weiner else 1134ff9d5ba2SDomenico Cerasuolo zpool_unmap_handle(pool, entry->handle); 11356b3379e8SJohannes Weiner 11362b281117SSeth Jennings BUG_ON(ret); 11372b281117SSeth Jennings BUG_ON(dlen != PAGE_SIZE); 11382b281117SSeth Jennings 11392b281117SSeth Jennings /* page is up to date */ 11402b281117SSeth Jennings SetPageUptodate(page); 11412b281117SSeth Jennings 1142b349acc7SWeijie Yang /* move it to the tail of the inactive list after end_writeback */ 1143b349acc7SWeijie Yang SetPageReclaim(page); 1144b349acc7SWeijie Yang 11452b281117SSeth Jennings /* start writeback */ 1146cf1e3fe4SChristoph Hellwig __swap_writepage(page, &wbc); 114709cbfeafSKirill A. Shutemov put_page(page); 11482b281117SSeth Jennings zswap_written_back_pages++; 11492b281117SSeth Jennings 11506b3379e8SJohannes Weiner return ret; 115198804a94SJohannes Weiner 11526b3379e8SJohannes Weiner fail: 11536b3379e8SJohannes Weiner if (!zpool_can_sleep_mapped(pool)) 11546b3379e8SJohannes Weiner kfree(tmp); 11550ab0abcfSWeijie Yang 11560ab0abcfSWeijie Yang /* 115798804a94SJohannes Weiner * If we get here because the page is already in swapcache, a 115898804a94SJohannes Weiner * load may be happening concurrently. It is safe and okay to 115998804a94SJohannes Weiner * not free the entry. It is also okay to return !0. 11600ab0abcfSWeijie Yang */ 11612b281117SSeth Jennings return ret; 11622b281117SSeth Jennings } 11632b281117SSeth Jennings 1164a85f878bSSrividya Desireddy static int zswap_is_page_same_filled(void *ptr, unsigned long *value) 1165a85f878bSSrividya Desireddy { 1166a85f878bSSrividya Desireddy unsigned long *page; 116762bf1258STaejoon Song unsigned long val; 116862bf1258STaejoon Song unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; 1169a85f878bSSrividya Desireddy 1170a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 117162bf1258STaejoon Song val = page[0]; 117262bf1258STaejoon Song 117362bf1258STaejoon Song if (val != page[last_pos]) 117462bf1258STaejoon Song return 0; 117562bf1258STaejoon Song 117662bf1258STaejoon Song for (pos = 1; pos < last_pos; pos++) { 117762bf1258STaejoon Song if (val != page[pos]) 1178a85f878bSSrividya Desireddy return 0; 1179a85f878bSSrividya Desireddy } 118062bf1258STaejoon Song 118162bf1258STaejoon Song *value = val; 118262bf1258STaejoon Song 1183a85f878bSSrividya Desireddy return 1; 1184a85f878bSSrividya Desireddy } 1185a85f878bSSrividya Desireddy 1186a85f878bSSrividya Desireddy static void zswap_fill_page(void *ptr, unsigned long value) 1187a85f878bSSrividya Desireddy { 1188a85f878bSSrividya Desireddy unsigned long *page; 1189a85f878bSSrividya Desireddy 1190a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 1191a85f878bSSrividya Desireddy memset_l(page, value, PAGE_SIZE / sizeof(unsigned long)); 1192a85f878bSSrividya Desireddy } 1193a85f878bSSrividya Desireddy 119434f4c198SMatthew Wilcox (Oracle) bool zswap_store(struct folio *folio) 11952b281117SSeth Jennings { 11963d2c9087SDavid Hildenbrand swp_entry_t swp = folio->swap; 119742c06a0eSJohannes Weiner int type = swp_type(swp); 119842c06a0eSJohannes Weiner pgoff_t offset = swp_offset(swp); 119934f4c198SMatthew Wilcox (Oracle) struct page *page = &folio->page; 12002b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 12012b281117SSeth Jennings struct zswap_entry *entry, *dupentry; 12021ec3b5feSBarry Song struct scatterlist input, output; 12031ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1204f4840ccfSJohannes Weiner struct obj_cgroup *objcg = NULL; 1205f4840ccfSJohannes Weiner struct zswap_pool *pool; 1206b8cf32dcSYosry Ahmed struct zpool *zpool; 12070bb48849SDomenico Cerasuolo unsigned int dlen = PAGE_SIZE; 1208a85f878bSSrividya Desireddy unsigned long handle, value; 12092b281117SSeth Jennings char *buf; 12102b281117SSeth Jennings u8 *src, *dst; 1211d2fcd82bSHui Zhu gfp_t gfp; 121242c06a0eSJohannes Weiner int ret; 121342c06a0eSJohannes Weiner 121434f4c198SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_locked(folio)); 121534f4c198SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); 12162b281117SSeth Jennings 121734f4c198SMatthew Wilcox (Oracle) /* Large folios aren't supported */ 121834f4c198SMatthew Wilcox (Oracle) if (folio_test_large(folio)) 121942c06a0eSJohannes Weiner return false; 12207ba71669SHuang Ying 122142c06a0eSJohannes Weiner if (!zswap_enabled || !tree) 122242c06a0eSJohannes Weiner return false; 12232b281117SSeth Jennings 12240bdf0efaSNhat Pham /* 1225ca56489cSDomenico Cerasuolo * If this is a duplicate, it must be removed before attempting to store 1226ca56489cSDomenico Cerasuolo * it, otherwise, if the store fails the old page won't be removed from 1227ca56489cSDomenico Cerasuolo * the tree, and it might be written back overriding the new data. 1228ca56489cSDomenico Cerasuolo */ 1229ca56489cSDomenico Cerasuolo spin_lock(&tree->lock); 1230ca56489cSDomenico Cerasuolo dupentry = zswap_rb_search(&tree->rbroot, offset); 1231ca56489cSDomenico Cerasuolo if (dupentry) { 1232ca56489cSDomenico Cerasuolo zswap_duplicate_entry++; 1233ca56489cSDomenico Cerasuolo zswap_invalidate_entry(tree, dupentry); 1234ca56489cSDomenico Cerasuolo } 1235ca56489cSDomenico Cerasuolo spin_unlock(&tree->lock); 1236ca56489cSDomenico Cerasuolo 1237ca56489cSDomenico Cerasuolo /* 12380bdf0efaSNhat Pham * XXX: zswap reclaim does not work with cgroups yet. Without a 12390bdf0efaSNhat Pham * cgroup-aware entry LRU, we will push out entries system-wide based on 12400bdf0efaSNhat Pham * local cgroup limits. 12410bdf0efaSNhat Pham */ 1242074e3e26SMatthew Wilcox (Oracle) objcg = get_obj_cgroup_from_folio(folio); 124342c06a0eSJohannes Weiner if (objcg && !obj_cgroup_may_zswap(objcg)) 12440bdf0efaSNhat Pham goto reject; 1245f4840ccfSJohannes Weiner 12462b281117SSeth Jennings /* reclaim space if needed */ 12472b281117SSeth Jennings if (zswap_is_full()) { 12482b281117SSeth Jennings zswap_pool_limit_hit++; 124945190f01SVitaly Wool zswap_pool_reached_full = true; 1250f4840ccfSJohannes Weiner goto shrink; 12512b281117SSeth Jennings } 125216e536efSLi Wang 125345190f01SVitaly Wool if (zswap_pool_reached_full) { 125442c06a0eSJohannes Weiner if (!zswap_can_accept()) 1255e0228d59SDomenico Cerasuolo goto shrink; 125642c06a0eSJohannes Weiner else 125745190f01SVitaly Wool zswap_pool_reached_full = false; 12582b281117SSeth Jennings } 12592b281117SSeth Jennings 12602b281117SSeth Jennings /* allocate entry */ 12612b281117SSeth Jennings entry = zswap_entry_cache_alloc(GFP_KERNEL); 12622b281117SSeth Jennings if (!entry) { 12632b281117SSeth Jennings zswap_reject_kmemcache_fail++; 12642b281117SSeth Jennings goto reject; 12652b281117SSeth Jennings } 12662b281117SSeth Jennings 1267a85f878bSSrividya Desireddy if (zswap_same_filled_pages_enabled) { 1268a85f878bSSrividya Desireddy src = kmap_atomic(page); 1269a85f878bSSrividya Desireddy if (zswap_is_page_same_filled(src, &value)) { 1270a85f878bSSrividya Desireddy kunmap_atomic(src); 12710bb48849SDomenico Cerasuolo entry->swpentry = swp_entry(type, offset); 1272a85f878bSSrividya Desireddy entry->length = 0; 1273a85f878bSSrividya Desireddy entry->value = value; 1274a85f878bSSrividya Desireddy atomic_inc(&zswap_same_filled_pages); 1275a85f878bSSrividya Desireddy goto insert_entry; 1276a85f878bSSrividya Desireddy } 1277a85f878bSSrividya Desireddy kunmap_atomic(src); 1278a85f878bSSrividya Desireddy } 1279a85f878bSSrividya Desireddy 128042c06a0eSJohannes Weiner if (!zswap_non_same_filled_pages_enabled) 1281cb325dddSMaciej S. Szmigiero goto freepage; 1282cb325dddSMaciej S. Szmigiero 1283f1c54846SDan Streetman /* if entry is successfully added, it keeps the reference */ 1284f1c54846SDan Streetman entry->pool = zswap_pool_current_get(); 128542c06a0eSJohannes Weiner if (!entry->pool) 12862b281117SSeth Jennings goto freepage; 12872b281117SSeth Jennings 1288f1c54846SDan Streetman /* compress */ 12891ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 12901ec3b5feSBarry Song 12911ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 12921ec3b5feSBarry Song 12931ec3b5feSBarry Song dst = acomp_ctx->dstmem; 12941ec3b5feSBarry Song sg_init_table(&input, 1); 12951ec3b5feSBarry Song sg_set_page(&input, page, PAGE_SIZE, 0); 12961ec3b5feSBarry Song 12971ec3b5feSBarry Song /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */ 12981ec3b5feSBarry Song sg_init_one(&output, dst, PAGE_SIZE * 2); 12991ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); 13001ec3b5feSBarry Song /* 13011ec3b5feSBarry Song * it maybe looks a little bit silly that we send an asynchronous request, 13021ec3b5feSBarry Song * then wait for its completion synchronously. This makes the process look 13031ec3b5feSBarry Song * synchronous in fact. 13041ec3b5feSBarry Song * Theoretically, acomp supports users send multiple acomp requests in one 13051ec3b5feSBarry Song * acomp instance, then get those requests done simultaneously. but in this 130642c06a0eSJohannes Weiner * case, zswap actually does store and load page by page, there is no 13071ec3b5feSBarry Song * existing method to send the second page before the first page is done 130842c06a0eSJohannes Weiner * in one thread doing zwap. 13091ec3b5feSBarry Song * but in different threads running on different cpu, we have different 13101ec3b5feSBarry Song * acomp instance, so multiple threads can do (de)compression in parallel. 13111ec3b5feSBarry Song */ 13121ec3b5feSBarry Song ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait); 13131ec3b5feSBarry Song dlen = acomp_ctx->req->dlen; 13141ec3b5feSBarry Song 131542c06a0eSJohannes Weiner if (ret) 1316f1c54846SDan Streetman goto put_dstmem; 1317f1c54846SDan Streetman 13182b281117SSeth Jennings /* store */ 1319b8cf32dcSYosry Ahmed zpool = zswap_find_zpool(entry); 1320d2fcd82bSHui Zhu gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 1321b8cf32dcSYosry Ahmed if (zpool_malloc_support_movable(zpool)) 1322d2fcd82bSHui Zhu gfp |= __GFP_HIGHMEM | __GFP_MOVABLE; 1323b8cf32dcSYosry Ahmed ret = zpool_malloc(zpool, dlen, gfp, &handle); 13242b281117SSeth Jennings if (ret == -ENOSPC) { 13252b281117SSeth Jennings zswap_reject_compress_poor++; 1326f1c54846SDan Streetman goto put_dstmem; 13272b281117SSeth Jennings } 13282b281117SSeth Jennings if (ret) { 13292b281117SSeth Jennings zswap_reject_alloc_fail++; 1330f1c54846SDan Streetman goto put_dstmem; 13312b281117SSeth Jennings } 1332b8cf32dcSYosry Ahmed buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO); 13330bb48849SDomenico Cerasuolo memcpy(buf, dst, dlen); 1334b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, handle); 13351ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 13362b281117SSeth Jennings 13372b281117SSeth Jennings /* populate entry */ 13380bb48849SDomenico Cerasuolo entry->swpentry = swp_entry(type, offset); 13392b281117SSeth Jennings entry->handle = handle; 13402b281117SSeth Jennings entry->length = dlen; 13412b281117SSeth Jennings 1342a85f878bSSrividya Desireddy insert_entry: 1343f4840ccfSJohannes Weiner entry->objcg = objcg; 1344f4840ccfSJohannes Weiner if (objcg) { 1345f4840ccfSJohannes Weiner obj_cgroup_charge_zswap(objcg, entry->length); 1346f4840ccfSJohannes Weiner /* Account before objcg ref is moved to tree */ 1347f4840ccfSJohannes Weiner count_objcg_event(objcg, ZSWPOUT); 1348f4840ccfSJohannes Weiner } 1349f4840ccfSJohannes Weiner 13502b281117SSeth Jennings /* map */ 13512b281117SSeth Jennings spin_lock(&tree->lock); 1352ca56489cSDomenico Cerasuolo /* 1353ca56489cSDomenico Cerasuolo * A duplicate entry should have been removed at the beginning of this 1354ca56489cSDomenico Cerasuolo * function. Since the swap entry should be pinned, if a duplicate is 1355ca56489cSDomenico Cerasuolo * found again here it means that something went wrong in the swap 1356ca56489cSDomenico Cerasuolo * cache. 1357ca56489cSDomenico Cerasuolo */ 135842c06a0eSJohannes Weiner while (zswap_rb_insert(&tree->rbroot, entry, &dupentry) == -EEXIST) { 1359ca56489cSDomenico Cerasuolo WARN_ON(1); 13602b281117SSeth Jennings zswap_duplicate_entry++; 136156c67049SJohannes Weiner zswap_invalidate_entry(tree, dupentry); 13622b281117SSeth Jennings } 136335499e2bSDomenico Cerasuolo if (entry->length) { 1364f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 1365f999f38bSDomenico Cerasuolo list_add(&entry->lru, &entry->pool->lru); 1366f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 1367f999f38bSDomenico Cerasuolo } 13682b281117SSeth Jennings spin_unlock(&tree->lock); 13692b281117SSeth Jennings 13702b281117SSeth Jennings /* update stats */ 13712b281117SSeth Jennings atomic_inc(&zswap_stored_pages); 1372f1c54846SDan Streetman zswap_update_total_size(); 1373f6498b77SJohannes Weiner count_vm_event(ZSWPOUT); 13742b281117SSeth Jennings 137542c06a0eSJohannes Weiner return true; 13762b281117SSeth Jennings 1377f1c54846SDan Streetman put_dstmem: 13781ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 1379f1c54846SDan Streetman zswap_pool_put(entry->pool); 1380f1c54846SDan Streetman freepage: 13812b281117SSeth Jennings zswap_entry_cache_free(entry); 13822b281117SSeth Jennings reject: 1383f4840ccfSJohannes Weiner if (objcg) 1384f4840ccfSJohannes Weiner obj_cgroup_put(objcg); 138542c06a0eSJohannes Weiner return false; 1386f4840ccfSJohannes Weiner 1387f4840ccfSJohannes Weiner shrink: 1388f4840ccfSJohannes Weiner pool = zswap_pool_last_get(); 1389969d63e1SJohannes Weiner if (pool && !queue_work(shrink_wq, &pool->shrink_work)) 1390969d63e1SJohannes Weiner zswap_pool_put(pool); 1391f4840ccfSJohannes Weiner goto reject; 13922b281117SSeth Jennings } 13932b281117SSeth Jennings 1394ca54f6d8SMatthew Wilcox (Oracle) bool zswap_load(struct folio *folio) 13952b281117SSeth Jennings { 13963d2c9087SDavid Hildenbrand swp_entry_t swp = folio->swap; 139742c06a0eSJohannes Weiner int type = swp_type(swp); 139842c06a0eSJohannes Weiner pgoff_t offset = swp_offset(swp); 1399ca54f6d8SMatthew Wilcox (Oracle) struct page *page = &folio->page; 14002b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 14012b281117SSeth Jennings struct zswap_entry *entry; 14021ec3b5feSBarry Song struct scatterlist input, output; 14031ec3b5feSBarry Song struct crypto_acomp_ctx *acomp_ctx; 1404fc6697a8STian Tao u8 *src, *dst, *tmp; 1405b8cf32dcSYosry Ahmed struct zpool *zpool; 14062b281117SSeth Jennings unsigned int dlen; 140742c06a0eSJohannes Weiner bool ret; 140842c06a0eSJohannes Weiner 1409ca54f6d8SMatthew Wilcox (Oracle) VM_WARN_ON_ONCE(!folio_test_locked(folio)); 14102b281117SSeth Jennings 14112b281117SSeth Jennings /* find */ 14122b281117SSeth Jennings spin_lock(&tree->lock); 14130ab0abcfSWeijie Yang entry = zswap_entry_find_get(&tree->rbroot, offset); 14142b281117SSeth Jennings if (!entry) { 14152b281117SSeth Jennings spin_unlock(&tree->lock); 141642c06a0eSJohannes Weiner return false; 14172b281117SSeth Jennings } 14182b281117SSeth Jennings spin_unlock(&tree->lock); 14192b281117SSeth Jennings 1420a85f878bSSrividya Desireddy if (!entry->length) { 1421a85f878bSSrividya Desireddy dst = kmap_atomic(page); 1422a85f878bSSrividya Desireddy zswap_fill_page(dst, entry->value); 1423a85f878bSSrividya Desireddy kunmap_atomic(dst); 142442c06a0eSJohannes Weiner ret = true; 1425f6498b77SJohannes Weiner goto stats; 1426a85f878bSSrividya Desireddy } 1427a85f878bSSrividya Desireddy 1428b8cf32dcSYosry Ahmed zpool = zswap_find_zpool(entry); 1429b8cf32dcSYosry Ahmed if (!zpool_can_sleep_mapped(zpool)) { 14308d9b6370SSergey Senozhatsky tmp = kmalloc(entry->length, GFP_KERNEL); 1431fc6697a8STian Tao if (!tmp) { 143242c06a0eSJohannes Weiner ret = false; 1433fc6697a8STian Tao goto freeentry; 1434fc6697a8STian Tao } 1435fc6697a8STian Tao } 1436fc6697a8STian Tao 14372b281117SSeth Jennings /* decompress */ 14382b281117SSeth Jennings dlen = PAGE_SIZE; 1439b8cf32dcSYosry Ahmed src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO); 14401ec3b5feSBarry Song 1441b8cf32dcSYosry Ahmed if (!zpool_can_sleep_mapped(zpool)) { 1442fc6697a8STian Tao memcpy(tmp, src, entry->length); 1443fc6697a8STian Tao src = tmp; 1444b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, entry->handle); 1445fc6697a8STian Tao } 1446fc6697a8STian Tao 14471ec3b5feSBarry Song acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); 14481ec3b5feSBarry Song mutex_lock(acomp_ctx->mutex); 14491ec3b5feSBarry Song sg_init_one(&input, src, entry->length); 14501ec3b5feSBarry Song sg_init_table(&output, 1); 14511ec3b5feSBarry Song sg_set_page(&output, page, PAGE_SIZE, 0); 14521ec3b5feSBarry Song acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen); 145342c06a0eSJohannes Weiner if (crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait)) 145442c06a0eSJohannes Weiner WARN_ON(1); 14551ec3b5feSBarry Song mutex_unlock(acomp_ctx->mutex); 14561ec3b5feSBarry Song 1457b8cf32dcSYosry Ahmed if (zpool_can_sleep_mapped(zpool)) 1458b8cf32dcSYosry Ahmed zpool_unmap_handle(zpool, entry->handle); 1459fc6697a8STian Tao else 1460fc6697a8STian Tao kfree(tmp); 1461fc6697a8STian Tao 146242c06a0eSJohannes Weiner ret = true; 1463f6498b77SJohannes Weiner stats: 1464f6498b77SJohannes Weiner count_vm_event(ZSWPIN); 1465f4840ccfSJohannes Weiner if (entry->objcg) 1466f4840ccfSJohannes Weiner count_objcg_event(entry->objcg, ZSWPIN); 1467a85f878bSSrividya Desireddy freeentry: 14682b281117SSeth Jennings spin_lock(&tree->lock); 146942c06a0eSJohannes Weiner if (ret && zswap_exclusive_loads_enabled) { 1470b9c91c43SYosry Ahmed zswap_invalidate_entry(tree, entry); 1471ca54f6d8SMatthew Wilcox (Oracle) folio_mark_dirty(folio); 147235499e2bSDomenico Cerasuolo } else if (entry->length) { 1473f999f38bSDomenico Cerasuolo spin_lock(&entry->pool->lru_lock); 1474f999f38bSDomenico Cerasuolo list_move(&entry->lru, &entry->pool->lru); 1475f999f38bSDomenico Cerasuolo spin_unlock(&entry->pool->lru_lock); 1476b9c91c43SYosry Ahmed } 147718a93707SYosry Ahmed zswap_entry_put(tree, entry); 14782b281117SSeth Jennings spin_unlock(&tree->lock); 14792b281117SSeth Jennings 1480fc6697a8STian Tao return ret; 14812b281117SSeth Jennings } 14822b281117SSeth Jennings 148342c06a0eSJohannes Weiner void zswap_invalidate(int type, pgoff_t offset) 14842b281117SSeth Jennings { 14852b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 14862b281117SSeth Jennings struct zswap_entry *entry; 14872b281117SSeth Jennings 14882b281117SSeth Jennings /* find */ 14892b281117SSeth Jennings spin_lock(&tree->lock); 14902b281117SSeth Jennings entry = zswap_rb_search(&tree->rbroot, offset); 14912b281117SSeth Jennings if (!entry) { 14922b281117SSeth Jennings /* entry was written back */ 14932b281117SSeth Jennings spin_unlock(&tree->lock); 14942b281117SSeth Jennings return; 14952b281117SSeth Jennings } 1496b9c91c43SYosry Ahmed zswap_invalidate_entry(tree, entry); 14972b281117SSeth Jennings spin_unlock(&tree->lock); 14982b281117SSeth Jennings } 14992b281117SSeth Jennings 150042c06a0eSJohannes Weiner void zswap_swapon(int type) 150142c06a0eSJohannes Weiner { 150242c06a0eSJohannes Weiner struct zswap_tree *tree; 150342c06a0eSJohannes Weiner 150442c06a0eSJohannes Weiner tree = kzalloc(sizeof(*tree), GFP_KERNEL); 150542c06a0eSJohannes Weiner if (!tree) { 150642c06a0eSJohannes Weiner pr_err("alloc failed, zswap disabled for swap type %d\n", type); 150742c06a0eSJohannes Weiner return; 150842c06a0eSJohannes Weiner } 150942c06a0eSJohannes Weiner 151042c06a0eSJohannes Weiner tree->rbroot = RB_ROOT; 151142c06a0eSJohannes Weiner spin_lock_init(&tree->lock); 151242c06a0eSJohannes Weiner zswap_trees[type] = tree; 151342c06a0eSJohannes Weiner } 151442c06a0eSJohannes Weiner 151542c06a0eSJohannes Weiner void zswap_swapoff(int type) 15162b281117SSeth Jennings { 15172b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 15180bd42136SCody P Schafer struct zswap_entry *entry, *n; 15192b281117SSeth Jennings 15202b281117SSeth Jennings if (!tree) 15212b281117SSeth Jennings return; 15222b281117SSeth Jennings 15232b281117SSeth Jennings /* walk the tree and free everything */ 15242b281117SSeth Jennings spin_lock(&tree->lock); 15250ab0abcfSWeijie Yang rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) 152660105e12SMinchan Kim zswap_free_entry(entry); 15272b281117SSeth Jennings tree->rbroot = RB_ROOT; 15282b281117SSeth Jennings spin_unlock(&tree->lock); 1529aa9bca05SWeijie Yang kfree(tree); 1530aa9bca05SWeijie Yang zswap_trees[type] = NULL; 15312b281117SSeth Jennings } 15322b281117SSeth Jennings 15332b281117SSeth Jennings /********************************* 15342b281117SSeth Jennings * debugfs functions 15352b281117SSeth Jennings **********************************/ 15362b281117SSeth Jennings #ifdef CONFIG_DEBUG_FS 15372b281117SSeth Jennings #include <linux/debugfs.h> 15382b281117SSeth Jennings 15392b281117SSeth Jennings static struct dentry *zswap_debugfs_root; 15402b281117SSeth Jennings 1541141fdeecSLiu Shixin static int zswap_debugfs_init(void) 15422b281117SSeth Jennings { 15432b281117SSeth Jennings if (!debugfs_initialized()) 15442b281117SSeth Jennings return -ENODEV; 15452b281117SSeth Jennings 15462b281117SSeth Jennings zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 15472b281117SSeth Jennings 15480825a6f9SJoe Perches debugfs_create_u64("pool_limit_hit", 0444, 15492b281117SSeth Jennings zswap_debugfs_root, &zswap_pool_limit_hit); 15500825a6f9SJoe Perches debugfs_create_u64("reject_reclaim_fail", 0444, 15512b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_reclaim_fail); 15520825a6f9SJoe Perches debugfs_create_u64("reject_alloc_fail", 0444, 15532b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_alloc_fail); 15540825a6f9SJoe Perches debugfs_create_u64("reject_kmemcache_fail", 0444, 15552b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_kmemcache_fail); 15560825a6f9SJoe Perches debugfs_create_u64("reject_compress_poor", 0444, 15572b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_compress_poor); 15580825a6f9SJoe Perches debugfs_create_u64("written_back_pages", 0444, 15592b281117SSeth Jennings zswap_debugfs_root, &zswap_written_back_pages); 15600825a6f9SJoe Perches debugfs_create_u64("duplicate_entry", 0444, 15612b281117SSeth Jennings zswap_debugfs_root, &zswap_duplicate_entry); 15620825a6f9SJoe Perches debugfs_create_u64("pool_total_size", 0444, 156312d79d64SDan Streetman zswap_debugfs_root, &zswap_pool_total_size); 15640825a6f9SJoe Perches debugfs_create_atomic_t("stored_pages", 0444, 15652b281117SSeth Jennings zswap_debugfs_root, &zswap_stored_pages); 1566a85f878bSSrividya Desireddy debugfs_create_atomic_t("same_filled_pages", 0444, 1567a85f878bSSrividya Desireddy zswap_debugfs_root, &zswap_same_filled_pages); 15682b281117SSeth Jennings 15692b281117SSeth Jennings return 0; 15702b281117SSeth Jennings } 15712b281117SSeth Jennings #else 1572141fdeecSLiu Shixin static int zswap_debugfs_init(void) 15732b281117SSeth Jennings { 15742b281117SSeth Jennings return 0; 15752b281117SSeth Jennings } 15762b281117SSeth Jennings #endif 15772b281117SSeth Jennings 15782b281117SSeth Jennings /********************************* 15792b281117SSeth Jennings * module init and exit 15802b281117SSeth Jennings **********************************/ 1581141fdeecSLiu Shixin static int zswap_setup(void) 15822b281117SSeth Jennings { 1583f1c54846SDan Streetman struct zswap_pool *pool; 1584ad7ed770SSebastian Andrzej Siewior int ret; 158560105e12SMinchan Kim 1586b7919122SLiu Shixin zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 1587b7919122SLiu Shixin if (!zswap_entry_cache) { 15882b281117SSeth Jennings pr_err("entry cache creation failed\n"); 1589f1c54846SDan Streetman goto cache_fail; 15902b281117SSeth Jennings } 1591f1c54846SDan Streetman 1592ad7ed770SSebastian Andrzej Siewior ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare", 1593ad7ed770SSebastian Andrzej Siewior zswap_dstmem_prepare, zswap_dstmem_dead); 1594ad7ed770SSebastian Andrzej Siewior if (ret) { 1595f1c54846SDan Streetman pr_err("dstmem alloc failed\n"); 1596f1c54846SDan Streetman goto dstmem_fail; 15972b281117SSeth Jennings } 1598f1c54846SDan Streetman 1599cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, 1600cab7a7e5SSebastian Andrzej Siewior "mm/zswap_pool:prepare", 1601cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_prepare, 1602cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_dead); 1603cab7a7e5SSebastian Andrzej Siewior if (ret) 1604cab7a7e5SSebastian Andrzej Siewior goto hp_fail; 1605cab7a7e5SSebastian Andrzej Siewior 1606f1c54846SDan Streetman pool = __zswap_pool_create_fallback(); 1607ae3d89a7SDan Streetman if (pool) { 1608f1c54846SDan Streetman pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1609b8cf32dcSYosry Ahmed zpool_get_type(pool->zpools[0])); 1610f1c54846SDan Streetman list_add(&pool->list, &zswap_pools); 1611ae3d89a7SDan Streetman zswap_has_pool = true; 1612ae3d89a7SDan Streetman } else { 1613ae3d89a7SDan Streetman pr_err("pool creation failed\n"); 1614ae3d89a7SDan Streetman zswap_enabled = false; 1615ae3d89a7SDan Streetman } 161660105e12SMinchan Kim 161745190f01SVitaly Wool shrink_wq = create_workqueue("zswap-shrink"); 161845190f01SVitaly Wool if (!shrink_wq) 161945190f01SVitaly Wool goto fallback_fail; 162045190f01SVitaly Wool 16212b281117SSeth Jennings if (zswap_debugfs_init()) 16222b281117SSeth Jennings pr_warn("debugfs initialization failed\n"); 16239021ccecSLiu Shixin zswap_init_state = ZSWAP_INIT_SUCCEED; 16242b281117SSeth Jennings return 0; 1625f1c54846SDan Streetman 162645190f01SVitaly Wool fallback_fail: 162738aeb071SDan Carpenter if (pool) 162845190f01SVitaly Wool zswap_pool_destroy(pool); 1629cab7a7e5SSebastian Andrzej Siewior hp_fail: 1630ad7ed770SSebastian Andrzej Siewior cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); 1631f1c54846SDan Streetman dstmem_fail: 1632b7919122SLiu Shixin kmem_cache_destroy(zswap_entry_cache); 1633f1c54846SDan Streetman cache_fail: 1634d7b028f5SDan Streetman /* if built-in, we aren't unloaded on failure; don't allow use */ 16359021ccecSLiu Shixin zswap_init_state = ZSWAP_INIT_FAILED; 1636d7b028f5SDan Streetman zswap_enabled = false; 16372b281117SSeth Jennings return -ENOMEM; 16382b281117SSeth Jennings } 1639141fdeecSLiu Shixin 1640141fdeecSLiu Shixin static int __init zswap_init(void) 1641141fdeecSLiu Shixin { 1642141fdeecSLiu Shixin if (!zswap_enabled) 1643141fdeecSLiu Shixin return 0; 1644141fdeecSLiu Shixin return zswap_setup(); 1645141fdeecSLiu Shixin } 16462b281117SSeth Jennings /* must be late so crypto has time to come up */ 1647141fdeecSLiu Shixin late_initcall(zswap_init); 16482b281117SSeth Jennings 164968386da8SSeth Jennings MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 16502b281117SSeth Jennings MODULE_DESCRIPTION("Compressed cache for swap pages"); 1651