1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 22b281117SSeth Jennings /* 32b281117SSeth Jennings * zswap.c - zswap driver file 42b281117SSeth Jennings * 52b281117SSeth Jennings * zswap is a backend for frontswap that takes pages that are in the process 62b281117SSeth Jennings * of being swapped out and attempts to compress and store them in a 72b281117SSeth Jennings * RAM-based memory pool. This can result in a significant I/O reduction on 82b281117SSeth Jennings * the swap device and, in the case where decompressing from RAM is faster 92b281117SSeth Jennings * than reading from the swap device, can also improve workload performance. 102b281117SSeth Jennings * 112b281117SSeth Jennings * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 122b281117SSeth Jennings */ 132b281117SSeth Jennings 142b281117SSeth Jennings #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 152b281117SSeth Jennings 162b281117SSeth Jennings #include <linux/module.h> 172b281117SSeth Jennings #include <linux/cpu.h> 182b281117SSeth Jennings #include <linux/highmem.h> 192b281117SSeth Jennings #include <linux/slab.h> 202b281117SSeth Jennings #include <linux/spinlock.h> 212b281117SSeth Jennings #include <linux/types.h> 222b281117SSeth Jennings #include <linux/atomic.h> 232b281117SSeth Jennings #include <linux/frontswap.h> 242b281117SSeth Jennings #include <linux/rbtree.h> 252b281117SSeth Jennings #include <linux/swap.h> 262b281117SSeth Jennings #include <linux/crypto.h> 272b281117SSeth Jennings #include <linux/mempool.h> 2812d79d64SDan Streetman #include <linux/zpool.h> 292b281117SSeth Jennings 302b281117SSeth Jennings #include <linux/mm_types.h> 312b281117SSeth Jennings #include <linux/page-flags.h> 322b281117SSeth Jennings #include <linux/swapops.h> 332b281117SSeth Jennings #include <linux/writeback.h> 342b281117SSeth Jennings #include <linux/pagemap.h> 35*45190f01SVitaly Wool #include <linux/workqueue.h> 362b281117SSeth Jennings 372b281117SSeth Jennings /********************************* 382b281117SSeth Jennings * statistics 392b281117SSeth Jennings **********************************/ 4012d79d64SDan Streetman /* Total bytes used by the compressed storage */ 4112d79d64SDan Streetman static u64 zswap_pool_total_size; 422b281117SSeth Jennings /* The number of compressed pages currently stored in zswap */ 432b281117SSeth Jennings static atomic_t zswap_stored_pages = ATOMIC_INIT(0); 44a85f878bSSrividya Desireddy /* The number of same-value filled pages currently stored in zswap */ 45a85f878bSSrividya Desireddy static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0); 462b281117SSeth Jennings 472b281117SSeth Jennings /* 482b281117SSeth Jennings * The statistics below are not protected from concurrent access for 492b281117SSeth Jennings * performance reasons so they may not be a 100% accurate. However, 502b281117SSeth Jennings * they do provide useful information on roughly how many times a 512b281117SSeth Jennings * certain event is occurring. 522b281117SSeth Jennings */ 532b281117SSeth Jennings 542b281117SSeth Jennings /* Pool limit was hit (see zswap_max_pool_percent) */ 552b281117SSeth Jennings static u64 zswap_pool_limit_hit; 562b281117SSeth Jennings /* Pages written back when pool limit was reached */ 572b281117SSeth Jennings static u64 zswap_written_back_pages; 582b281117SSeth Jennings /* Store failed due to a reclaim failure after pool limit was reached */ 592b281117SSeth Jennings static u64 zswap_reject_reclaim_fail; 602b281117SSeth Jennings /* Compressed page was too big for the allocator to (optimally) store */ 612b281117SSeth Jennings static u64 zswap_reject_compress_poor; 622b281117SSeth Jennings /* Store failed because underlying allocator could not get memory */ 632b281117SSeth Jennings static u64 zswap_reject_alloc_fail; 642b281117SSeth Jennings /* Store failed because the entry metadata could not be allocated (rare) */ 652b281117SSeth Jennings static u64 zswap_reject_kmemcache_fail; 662b281117SSeth Jennings /* Duplicate store was encountered (rare) */ 672b281117SSeth Jennings static u64 zswap_duplicate_entry; 682b281117SSeth Jennings 69*45190f01SVitaly Wool /* Shrinker work queue */ 70*45190f01SVitaly Wool static struct workqueue_struct *shrink_wq; 71*45190f01SVitaly Wool /* Pool limit was hit, we need to calm down */ 72*45190f01SVitaly Wool static bool zswap_pool_reached_full; 73*45190f01SVitaly Wool 742b281117SSeth Jennings /********************************* 752b281117SSeth Jennings * tunables 762b281117SSeth Jennings **********************************/ 77c00ed16aSDan Streetman 78bae21db8SDan Streetman #define ZSWAP_PARAM_UNSET "" 79bae21db8SDan Streetman 80c00ed16aSDan Streetman /* Enable/disable zswap (disabled by default) */ 81c00ed16aSDan Streetman static bool zswap_enabled; 82d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *, 83d7b028f5SDan Streetman const struct kernel_param *); 84d7b028f5SDan Streetman static struct kernel_param_ops zswap_enabled_param_ops = { 85d7b028f5SDan Streetman .set = zswap_enabled_param_set, 86d7b028f5SDan Streetman .get = param_get_bool, 87d7b028f5SDan Streetman }; 88d7b028f5SDan Streetman module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); 892b281117SSeth Jennings 9090b0fc26SDan Streetman /* Crypto compressor to use */ 912b281117SSeth Jennings #define ZSWAP_COMPRESSOR_DEFAULT "lzo" 92c99b42c3SDan Streetman static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; 9390b0fc26SDan Streetman static int zswap_compressor_param_set(const char *, 9490b0fc26SDan Streetman const struct kernel_param *); 9590b0fc26SDan Streetman static struct kernel_param_ops zswap_compressor_param_ops = { 9690b0fc26SDan Streetman .set = zswap_compressor_param_set, 97c99b42c3SDan Streetman .get = param_get_charp, 98c99b42c3SDan Streetman .free = param_free_charp, 9990b0fc26SDan Streetman }; 10090b0fc26SDan Streetman module_param_cb(compressor, &zswap_compressor_param_ops, 101c99b42c3SDan Streetman &zswap_compressor, 0644); 10290b0fc26SDan Streetman 10390b0fc26SDan Streetman /* Compressed storage zpool to use */ 10490b0fc26SDan Streetman #define ZSWAP_ZPOOL_DEFAULT "zbud" 105c99b42c3SDan Streetman static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; 10690b0fc26SDan Streetman static int zswap_zpool_param_set(const char *, const struct kernel_param *); 10790b0fc26SDan Streetman static struct kernel_param_ops zswap_zpool_param_ops = { 10890b0fc26SDan Streetman .set = zswap_zpool_param_set, 109c99b42c3SDan Streetman .get = param_get_charp, 110c99b42c3SDan Streetman .free = param_free_charp, 11190b0fc26SDan Streetman }; 112c99b42c3SDan Streetman module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 1132b281117SSeth Jennings 1142b281117SSeth Jennings /* The maximum percentage of memory that the compressed pool can occupy */ 1152b281117SSeth Jennings static unsigned int zswap_max_pool_percent = 20; 11690b0fc26SDan Streetman module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 11760105e12SMinchan Kim 118*45190f01SVitaly Wool /* The threshold for accepting new pages after the max_pool_percent was hit */ 119*45190f01SVitaly Wool static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ 120*45190f01SVitaly Wool module_param_named(accept_threshold_percent, zswap_accept_thr_percent, 121*45190f01SVitaly Wool uint, 0644); 122*45190f01SVitaly Wool 123a85f878bSSrividya Desireddy /* Enable/disable handling same-value filled pages (enabled by default) */ 124a85f878bSSrividya Desireddy static bool zswap_same_filled_pages_enabled = true; 125a85f878bSSrividya Desireddy module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled, 126a85f878bSSrividya Desireddy bool, 0644); 127a85f878bSSrividya Desireddy 1282b281117SSeth Jennings /********************************* 1292b281117SSeth Jennings * data structures 1302b281117SSeth Jennings **********************************/ 131f1c54846SDan Streetman 132f1c54846SDan Streetman struct zswap_pool { 133f1c54846SDan Streetman struct zpool *zpool; 134f1c54846SDan Streetman struct crypto_comp * __percpu *tfm; 135f1c54846SDan Streetman struct kref kref; 136f1c54846SDan Streetman struct list_head list; 137*45190f01SVitaly Wool struct work_struct release_work; 138*45190f01SVitaly Wool struct work_struct shrink_work; 139cab7a7e5SSebastian Andrzej Siewior struct hlist_node node; 140f1c54846SDan Streetman char tfm_name[CRYPTO_MAX_ALG_NAME]; 141f1c54846SDan Streetman }; 142f1c54846SDan Streetman 1432b281117SSeth Jennings /* 1442b281117SSeth Jennings * struct zswap_entry 1452b281117SSeth Jennings * 1462b281117SSeth Jennings * This structure contains the metadata for tracking a single compressed 1472b281117SSeth Jennings * page within zswap. 1482b281117SSeth Jennings * 1492b281117SSeth Jennings * rbnode - links the entry into red-black tree for the appropriate swap type 150f1c54846SDan Streetman * offset - the swap offset for the entry. Index into the red-black tree. 1512b281117SSeth Jennings * refcount - the number of outstanding reference to the entry. This is needed 1522b281117SSeth Jennings * to protect against premature freeing of the entry by code 1536b452516SSeongJae Park * concurrent calls to load, invalidate, and writeback. The lock 1542b281117SSeth Jennings * for the zswap_tree structure that contains the entry must 1552b281117SSeth Jennings * be held while changing the refcount. Since the lock must 1562b281117SSeth Jennings * be held, there is no reason to also make refcount atomic. 1572b281117SSeth Jennings * length - the length in bytes of the compressed page data. Needed during 158a85f878bSSrividya Desireddy * decompression. For a same value filled page length is 0. 159f1c54846SDan Streetman * pool - the zswap_pool the entry's data is in 160f1c54846SDan Streetman * handle - zpool allocation handle that stores the compressed page data 161a85f878bSSrividya Desireddy * value - value of the same-value filled pages which have same content 1622b281117SSeth Jennings */ 1632b281117SSeth Jennings struct zswap_entry { 1642b281117SSeth Jennings struct rb_node rbnode; 1652b281117SSeth Jennings pgoff_t offset; 1662b281117SSeth Jennings int refcount; 1672b281117SSeth Jennings unsigned int length; 168f1c54846SDan Streetman struct zswap_pool *pool; 169a85f878bSSrividya Desireddy union { 1702b281117SSeth Jennings unsigned long handle; 171a85f878bSSrividya Desireddy unsigned long value; 172a85f878bSSrividya Desireddy }; 1732b281117SSeth Jennings }; 1742b281117SSeth Jennings 1752b281117SSeth Jennings struct zswap_header { 1762b281117SSeth Jennings swp_entry_t swpentry; 1772b281117SSeth Jennings }; 1782b281117SSeth Jennings 1792b281117SSeth Jennings /* 1802b281117SSeth Jennings * The tree lock in the zswap_tree struct protects a few things: 1812b281117SSeth Jennings * - the rbtree 1822b281117SSeth Jennings * - the refcount field of each entry in the tree 1832b281117SSeth Jennings */ 1842b281117SSeth Jennings struct zswap_tree { 1852b281117SSeth Jennings struct rb_root rbroot; 1862b281117SSeth Jennings spinlock_t lock; 1872b281117SSeth Jennings }; 1882b281117SSeth Jennings 1892b281117SSeth Jennings static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; 1902b281117SSeth Jennings 191f1c54846SDan Streetman /* RCU-protected iteration */ 192f1c54846SDan Streetman static LIST_HEAD(zswap_pools); 193f1c54846SDan Streetman /* protects zswap_pools list modification */ 194f1c54846SDan Streetman static DEFINE_SPINLOCK(zswap_pools_lock); 19532a4e169SDan Streetman /* pool counter to provide unique names to zpool */ 19632a4e169SDan Streetman static atomic_t zswap_pools_count = ATOMIC_INIT(0); 197f1c54846SDan Streetman 19890b0fc26SDan Streetman /* used by param callback function */ 19990b0fc26SDan Streetman static bool zswap_init_started; 20090b0fc26SDan Streetman 201d7b028f5SDan Streetman /* fatal error during init */ 202d7b028f5SDan Streetman static bool zswap_init_failed; 203d7b028f5SDan Streetman 204ae3d89a7SDan Streetman /* init completed, but couldn't create the initial pool */ 205ae3d89a7SDan Streetman static bool zswap_has_pool; 206ae3d89a7SDan Streetman 207f1c54846SDan Streetman /********************************* 208f1c54846SDan Streetman * helpers and fwd declarations 209f1c54846SDan Streetman **********************************/ 210f1c54846SDan Streetman 211f1c54846SDan Streetman #define zswap_pool_debug(msg, p) \ 212f1c54846SDan Streetman pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 213f1c54846SDan Streetman zpool_get_type((p)->zpool)) 214f1c54846SDan Streetman 215f1c54846SDan Streetman static int zswap_writeback_entry(struct zpool *pool, unsigned long handle); 216f1c54846SDan Streetman static int zswap_pool_get(struct zswap_pool *pool); 217f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool); 218f1c54846SDan Streetman 219f1c54846SDan Streetman static const struct zpool_ops zswap_zpool_ops = { 220f1c54846SDan Streetman .evict = zswap_writeback_entry 221f1c54846SDan Streetman }; 222f1c54846SDan Streetman 223f1c54846SDan Streetman static bool zswap_is_full(void) 224f1c54846SDan Streetman { 225ca79b0c2SArun KS return totalram_pages() * zswap_max_pool_percent / 100 < 226f1c54846SDan Streetman DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 227f1c54846SDan Streetman } 228f1c54846SDan Streetman 229*45190f01SVitaly Wool static bool zswap_can_accept(void) 230*45190f01SVitaly Wool { 231*45190f01SVitaly Wool return totalram_pages() * zswap_accept_thr_percent / 100 * 232*45190f01SVitaly Wool zswap_max_pool_percent / 100 > 233*45190f01SVitaly Wool DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 234*45190f01SVitaly Wool } 235*45190f01SVitaly Wool 236f1c54846SDan Streetman static void zswap_update_total_size(void) 237f1c54846SDan Streetman { 238f1c54846SDan Streetman struct zswap_pool *pool; 239f1c54846SDan Streetman u64 total = 0; 240f1c54846SDan Streetman 241f1c54846SDan Streetman rcu_read_lock(); 242f1c54846SDan Streetman 243f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 244f1c54846SDan Streetman total += zpool_get_total_size(pool->zpool); 245f1c54846SDan Streetman 246f1c54846SDan Streetman rcu_read_unlock(); 247f1c54846SDan Streetman 248f1c54846SDan Streetman zswap_pool_total_size = total; 249f1c54846SDan Streetman } 250f1c54846SDan Streetman 2512b281117SSeth Jennings /********************************* 2522b281117SSeth Jennings * zswap entry functions 2532b281117SSeth Jennings **********************************/ 2542b281117SSeth Jennings static struct kmem_cache *zswap_entry_cache; 2552b281117SSeth Jennings 256dd01d7d8SMahendran Ganesh static int __init zswap_entry_cache_create(void) 2572b281117SSeth Jennings { 2582b281117SSeth Jennings zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 2595d2d42deSSeongJae Park return zswap_entry_cache == NULL; 2602b281117SSeth Jennings } 2612b281117SSeth Jennings 262c119239bSFabian Frederick static void __init zswap_entry_cache_destroy(void) 2632b281117SSeth Jennings { 2642b281117SSeth Jennings kmem_cache_destroy(zswap_entry_cache); 2652b281117SSeth Jennings } 2662b281117SSeth Jennings 2672b281117SSeth Jennings static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) 2682b281117SSeth Jennings { 2692b281117SSeth Jennings struct zswap_entry *entry; 2702b281117SSeth Jennings entry = kmem_cache_alloc(zswap_entry_cache, gfp); 2712b281117SSeth Jennings if (!entry) 2722b281117SSeth Jennings return NULL; 2732b281117SSeth Jennings entry->refcount = 1; 2740ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 2752b281117SSeth Jennings return entry; 2762b281117SSeth Jennings } 2772b281117SSeth Jennings 2782b281117SSeth Jennings static void zswap_entry_cache_free(struct zswap_entry *entry) 2792b281117SSeth Jennings { 2802b281117SSeth Jennings kmem_cache_free(zswap_entry_cache, entry); 2812b281117SSeth Jennings } 2822b281117SSeth Jennings 2832b281117SSeth Jennings /********************************* 2842b281117SSeth Jennings * rbtree functions 2852b281117SSeth Jennings **********************************/ 2862b281117SSeth Jennings static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) 2872b281117SSeth Jennings { 2882b281117SSeth Jennings struct rb_node *node = root->rb_node; 2892b281117SSeth Jennings struct zswap_entry *entry; 2902b281117SSeth Jennings 2912b281117SSeth Jennings while (node) { 2922b281117SSeth Jennings entry = rb_entry(node, struct zswap_entry, rbnode); 2932b281117SSeth Jennings if (entry->offset > offset) 2942b281117SSeth Jennings node = node->rb_left; 2952b281117SSeth Jennings else if (entry->offset < offset) 2962b281117SSeth Jennings node = node->rb_right; 2972b281117SSeth Jennings else 2982b281117SSeth Jennings return entry; 2992b281117SSeth Jennings } 3002b281117SSeth Jennings return NULL; 3012b281117SSeth Jennings } 3022b281117SSeth Jennings 3032b281117SSeth Jennings /* 3042b281117SSeth Jennings * In the case that a entry with the same offset is found, a pointer to 3052b281117SSeth Jennings * the existing entry is stored in dupentry and the function returns -EEXIST 3062b281117SSeth Jennings */ 3072b281117SSeth Jennings static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, 3082b281117SSeth Jennings struct zswap_entry **dupentry) 3092b281117SSeth Jennings { 3102b281117SSeth Jennings struct rb_node **link = &root->rb_node, *parent = NULL; 3112b281117SSeth Jennings struct zswap_entry *myentry; 3122b281117SSeth Jennings 3132b281117SSeth Jennings while (*link) { 3142b281117SSeth Jennings parent = *link; 3152b281117SSeth Jennings myentry = rb_entry(parent, struct zswap_entry, rbnode); 3162b281117SSeth Jennings if (myentry->offset > entry->offset) 3172b281117SSeth Jennings link = &(*link)->rb_left; 3182b281117SSeth Jennings else if (myentry->offset < entry->offset) 3192b281117SSeth Jennings link = &(*link)->rb_right; 3202b281117SSeth Jennings else { 3212b281117SSeth Jennings *dupentry = myentry; 3222b281117SSeth Jennings return -EEXIST; 3232b281117SSeth Jennings } 3242b281117SSeth Jennings } 3252b281117SSeth Jennings rb_link_node(&entry->rbnode, parent, link); 3262b281117SSeth Jennings rb_insert_color(&entry->rbnode, root); 3272b281117SSeth Jennings return 0; 3282b281117SSeth Jennings } 3292b281117SSeth Jennings 3300ab0abcfSWeijie Yang static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) 3310ab0abcfSWeijie Yang { 3320ab0abcfSWeijie Yang if (!RB_EMPTY_NODE(&entry->rbnode)) { 3330ab0abcfSWeijie Yang rb_erase(&entry->rbnode, root); 3340ab0abcfSWeijie Yang RB_CLEAR_NODE(&entry->rbnode); 3350ab0abcfSWeijie Yang } 3360ab0abcfSWeijie Yang } 3370ab0abcfSWeijie Yang 3380ab0abcfSWeijie Yang /* 33912d79d64SDan Streetman * Carries out the common pattern of freeing and entry's zpool allocation, 3400ab0abcfSWeijie Yang * freeing the entry itself, and decrementing the number of stored pages. 3410ab0abcfSWeijie Yang */ 34260105e12SMinchan Kim static void zswap_free_entry(struct zswap_entry *entry) 3430ab0abcfSWeijie Yang { 344a85f878bSSrividya Desireddy if (!entry->length) 345a85f878bSSrividya Desireddy atomic_dec(&zswap_same_filled_pages); 346a85f878bSSrividya Desireddy else { 347f1c54846SDan Streetman zpool_free(entry->pool->zpool, entry->handle); 348f1c54846SDan Streetman zswap_pool_put(entry->pool); 349a85f878bSSrividya Desireddy } 3500ab0abcfSWeijie Yang zswap_entry_cache_free(entry); 3510ab0abcfSWeijie Yang atomic_dec(&zswap_stored_pages); 352f1c54846SDan Streetman zswap_update_total_size(); 3530ab0abcfSWeijie Yang } 3540ab0abcfSWeijie Yang 3550ab0abcfSWeijie Yang /* caller must hold the tree lock */ 3560ab0abcfSWeijie Yang static void zswap_entry_get(struct zswap_entry *entry) 3570ab0abcfSWeijie Yang { 3580ab0abcfSWeijie Yang entry->refcount++; 3590ab0abcfSWeijie Yang } 3600ab0abcfSWeijie Yang 3610ab0abcfSWeijie Yang /* caller must hold the tree lock 3620ab0abcfSWeijie Yang * remove from the tree and free it, if nobody reference the entry 3630ab0abcfSWeijie Yang */ 3640ab0abcfSWeijie Yang static void zswap_entry_put(struct zswap_tree *tree, 3650ab0abcfSWeijie Yang struct zswap_entry *entry) 3660ab0abcfSWeijie Yang { 3670ab0abcfSWeijie Yang int refcount = --entry->refcount; 3680ab0abcfSWeijie Yang 3690ab0abcfSWeijie Yang BUG_ON(refcount < 0); 3700ab0abcfSWeijie Yang if (refcount == 0) { 3710ab0abcfSWeijie Yang zswap_rb_erase(&tree->rbroot, entry); 37260105e12SMinchan Kim zswap_free_entry(entry); 3730ab0abcfSWeijie Yang } 3740ab0abcfSWeijie Yang } 3750ab0abcfSWeijie Yang 3760ab0abcfSWeijie Yang /* caller must hold the tree lock */ 3770ab0abcfSWeijie Yang static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, 3780ab0abcfSWeijie Yang pgoff_t offset) 3790ab0abcfSWeijie Yang { 380b0c9865fSAlexey Klimov struct zswap_entry *entry; 3810ab0abcfSWeijie Yang 3820ab0abcfSWeijie Yang entry = zswap_rb_search(root, offset); 3830ab0abcfSWeijie Yang if (entry) 3840ab0abcfSWeijie Yang zswap_entry_get(entry); 3850ab0abcfSWeijie Yang 3860ab0abcfSWeijie Yang return entry; 3870ab0abcfSWeijie Yang } 3880ab0abcfSWeijie Yang 3892b281117SSeth Jennings /********************************* 3902b281117SSeth Jennings * per-cpu code 3912b281117SSeth Jennings **********************************/ 3922b281117SSeth Jennings static DEFINE_PER_CPU(u8 *, zswap_dstmem); 3932b281117SSeth Jennings 394ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_prepare(unsigned int cpu) 3952b281117SSeth Jennings { 3962b281117SSeth Jennings u8 *dst; 3972b281117SSeth Jennings 39872d09633SEric Dumazet dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); 3992b2695f5SMarkus Elfring if (!dst) 400ad7ed770SSebastian Andrzej Siewior return -ENOMEM; 4012b2695f5SMarkus Elfring 4022b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = dst; 403ad7ed770SSebastian Andrzej Siewior return 0; 404ad7ed770SSebastian Andrzej Siewior } 405ad7ed770SSebastian Andrzej Siewior 406ad7ed770SSebastian Andrzej Siewior static int zswap_dstmem_dead(unsigned int cpu) 407ad7ed770SSebastian Andrzej Siewior { 408ad7ed770SSebastian Andrzej Siewior u8 *dst; 409ad7ed770SSebastian Andrzej Siewior 4102b281117SSeth Jennings dst = per_cpu(zswap_dstmem, cpu); 4112b281117SSeth Jennings kfree(dst); 4122b281117SSeth Jennings per_cpu(zswap_dstmem, cpu) = NULL; 4132b281117SSeth Jennings 4142b281117SSeth Jennings return 0; 415f1c54846SDan Streetman } 416f1c54846SDan Streetman 417cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 418f1c54846SDan Streetman { 419cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 420f1c54846SDan Streetman struct crypto_comp *tfm; 421f1c54846SDan Streetman 422f1c54846SDan Streetman if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu))) 423cab7a7e5SSebastian Andrzej Siewior return 0; 424cab7a7e5SSebastian Andrzej Siewior 425f1c54846SDan Streetman tfm = crypto_alloc_comp(pool->tfm_name, 0, 0); 426f1c54846SDan Streetman if (IS_ERR_OR_NULL(tfm)) { 427f1c54846SDan Streetman pr_err("could not alloc crypto comp %s : %ld\n", 428f1c54846SDan Streetman pool->tfm_name, PTR_ERR(tfm)); 429cab7a7e5SSebastian Andrzej Siewior return -ENOMEM; 430f1c54846SDan Streetman } 431f1c54846SDan Streetman *per_cpu_ptr(pool->tfm, cpu) = tfm; 432cab7a7e5SSebastian Andrzej Siewior return 0; 433cab7a7e5SSebastian Andrzej Siewior } 434cab7a7e5SSebastian Andrzej Siewior 435cab7a7e5SSebastian Andrzej Siewior static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 436cab7a7e5SSebastian Andrzej Siewior { 437cab7a7e5SSebastian Andrzej Siewior struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 438cab7a7e5SSebastian Andrzej Siewior struct crypto_comp *tfm; 439cab7a7e5SSebastian Andrzej Siewior 440f1c54846SDan Streetman tfm = *per_cpu_ptr(pool->tfm, cpu); 441f1c54846SDan Streetman if (!IS_ERR_OR_NULL(tfm)) 442f1c54846SDan Streetman crypto_free_comp(tfm); 443f1c54846SDan Streetman *per_cpu_ptr(pool->tfm, cpu) = NULL; 444f1c54846SDan Streetman return 0; 445f1c54846SDan Streetman } 446f1c54846SDan Streetman 447f1c54846SDan Streetman /********************************* 448f1c54846SDan Streetman * pool functions 449f1c54846SDan Streetman **********************************/ 450f1c54846SDan Streetman 451f1c54846SDan Streetman static struct zswap_pool *__zswap_pool_current(void) 452f1c54846SDan Streetman { 453f1c54846SDan Streetman struct zswap_pool *pool; 454f1c54846SDan Streetman 455f1c54846SDan Streetman pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 456ae3d89a7SDan Streetman WARN_ONCE(!pool && zswap_has_pool, 457ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 458f1c54846SDan Streetman 459f1c54846SDan Streetman return pool; 460f1c54846SDan Streetman } 461f1c54846SDan Streetman 462f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current(void) 463f1c54846SDan Streetman { 464f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 465f1c54846SDan Streetman 466f1c54846SDan Streetman return __zswap_pool_current(); 467f1c54846SDan Streetman } 468f1c54846SDan Streetman 469f1c54846SDan Streetman static struct zswap_pool *zswap_pool_current_get(void) 470f1c54846SDan Streetman { 471f1c54846SDan Streetman struct zswap_pool *pool; 472f1c54846SDan Streetman 473f1c54846SDan Streetman rcu_read_lock(); 474f1c54846SDan Streetman 475f1c54846SDan Streetman pool = __zswap_pool_current(); 476ae3d89a7SDan Streetman if (!zswap_pool_get(pool)) 477f1c54846SDan Streetman pool = NULL; 478f1c54846SDan Streetman 479f1c54846SDan Streetman rcu_read_unlock(); 480f1c54846SDan Streetman 481f1c54846SDan Streetman return pool; 482f1c54846SDan Streetman } 483f1c54846SDan Streetman 484f1c54846SDan Streetman static struct zswap_pool *zswap_pool_last_get(void) 485f1c54846SDan Streetman { 486f1c54846SDan Streetman struct zswap_pool *pool, *last = NULL; 487f1c54846SDan Streetman 488f1c54846SDan Streetman rcu_read_lock(); 489f1c54846SDan Streetman 490f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) 491f1c54846SDan Streetman last = pool; 492ae3d89a7SDan Streetman WARN_ONCE(!last && zswap_has_pool, 493ae3d89a7SDan Streetman "%s: no page storage pool!\n", __func__); 494ae3d89a7SDan Streetman if (!zswap_pool_get(last)) 495f1c54846SDan Streetman last = NULL; 496f1c54846SDan Streetman 497f1c54846SDan Streetman rcu_read_unlock(); 498f1c54846SDan Streetman 499f1c54846SDan Streetman return last; 500f1c54846SDan Streetman } 501f1c54846SDan Streetman 5028bc8b228SDan Streetman /* type and compressor must be null-terminated */ 503f1c54846SDan Streetman static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 504f1c54846SDan Streetman { 505f1c54846SDan Streetman struct zswap_pool *pool; 506f1c54846SDan Streetman 507f1c54846SDan Streetman assert_spin_locked(&zswap_pools_lock); 508f1c54846SDan Streetman 509f1c54846SDan Streetman list_for_each_entry_rcu(pool, &zswap_pools, list) { 5108bc8b228SDan Streetman if (strcmp(pool->tfm_name, compressor)) 511f1c54846SDan Streetman continue; 5128bc8b228SDan Streetman if (strcmp(zpool_get_type(pool->zpool), type)) 513f1c54846SDan Streetman continue; 514f1c54846SDan Streetman /* if we can't get it, it's about to be destroyed */ 515f1c54846SDan Streetman if (!zswap_pool_get(pool)) 516f1c54846SDan Streetman continue; 517f1c54846SDan Streetman return pool; 518f1c54846SDan Streetman } 519f1c54846SDan Streetman 520f1c54846SDan Streetman return NULL; 521f1c54846SDan Streetman } 522f1c54846SDan Streetman 523*45190f01SVitaly Wool static void shrink_worker(struct work_struct *w) 524*45190f01SVitaly Wool { 525*45190f01SVitaly Wool struct zswap_pool *pool = container_of(w, typeof(*pool), 526*45190f01SVitaly Wool shrink_work); 527*45190f01SVitaly Wool 528*45190f01SVitaly Wool if (zpool_shrink(pool->zpool, 1, NULL)) 529*45190f01SVitaly Wool zswap_reject_reclaim_fail++; 530*45190f01SVitaly Wool zswap_pool_put(pool); 531*45190f01SVitaly Wool } 532*45190f01SVitaly Wool 533f1c54846SDan Streetman static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 534f1c54846SDan Streetman { 535f1c54846SDan Streetman struct zswap_pool *pool; 53632a4e169SDan Streetman char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 537d0164adcSMel Gorman gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 538cab7a7e5SSebastian Andrzej Siewior int ret; 539f1c54846SDan Streetman 540bae21db8SDan Streetman if (!zswap_has_pool) { 541bae21db8SDan Streetman /* if either are unset, pool initialization failed, and we 542bae21db8SDan Streetman * need both params to be set correctly before trying to 543bae21db8SDan Streetman * create a pool. 544bae21db8SDan Streetman */ 545bae21db8SDan Streetman if (!strcmp(type, ZSWAP_PARAM_UNSET)) 546bae21db8SDan Streetman return NULL; 547bae21db8SDan Streetman if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 548bae21db8SDan Streetman return NULL; 549bae21db8SDan Streetman } 550bae21db8SDan Streetman 551f1c54846SDan Streetman pool = kzalloc(sizeof(*pool), GFP_KERNEL); 552f4ae0ce0SMarkus Elfring if (!pool) 553f1c54846SDan Streetman return NULL; 554f1c54846SDan Streetman 55532a4e169SDan Streetman /* unique name for each pool specifically required by zsmalloc */ 55632a4e169SDan Streetman snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count)); 55732a4e169SDan Streetman 55832a4e169SDan Streetman pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops); 559f1c54846SDan Streetman if (!pool->zpool) { 560f1c54846SDan Streetman pr_err("%s zpool not available\n", type); 561f1c54846SDan Streetman goto error; 562f1c54846SDan Streetman } 563f1c54846SDan Streetman pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); 564f1c54846SDan Streetman 565f1c54846SDan Streetman strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 566f1c54846SDan Streetman pool->tfm = alloc_percpu(struct crypto_comp *); 567f1c54846SDan Streetman if (!pool->tfm) { 568f1c54846SDan Streetman pr_err("percpu alloc failed\n"); 569f1c54846SDan Streetman goto error; 570f1c54846SDan Streetman } 571f1c54846SDan Streetman 572cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, 573cab7a7e5SSebastian Andrzej Siewior &pool->node); 574cab7a7e5SSebastian Andrzej Siewior if (ret) 575f1c54846SDan Streetman goto error; 576f1c54846SDan Streetman pr_debug("using %s compressor\n", pool->tfm_name); 577f1c54846SDan Streetman 578f1c54846SDan Streetman /* being the current pool takes 1 ref; this func expects the 579f1c54846SDan Streetman * caller to always add the new pool as the current pool 580f1c54846SDan Streetman */ 581f1c54846SDan Streetman kref_init(&pool->kref); 582f1c54846SDan Streetman INIT_LIST_HEAD(&pool->list); 583*45190f01SVitaly Wool INIT_WORK(&pool->shrink_work, shrink_worker); 584f1c54846SDan Streetman 585f1c54846SDan Streetman zswap_pool_debug("created", pool); 586f1c54846SDan Streetman 587f1c54846SDan Streetman return pool; 588f1c54846SDan Streetman 589f1c54846SDan Streetman error: 590f1c54846SDan Streetman free_percpu(pool->tfm); 591f1c54846SDan Streetman if (pool->zpool) 592f1c54846SDan Streetman zpool_destroy_pool(pool->zpool); 593f1c54846SDan Streetman kfree(pool); 594f1c54846SDan Streetman return NULL; 595f1c54846SDan Streetman } 596f1c54846SDan Streetman 597c99b42c3SDan Streetman static __init struct zswap_pool *__zswap_pool_create_fallback(void) 598f1c54846SDan Streetman { 599bae21db8SDan Streetman bool has_comp, has_zpool; 600bae21db8SDan Streetman 601bae21db8SDan Streetman has_comp = crypto_has_comp(zswap_compressor, 0, 0); 602bae21db8SDan Streetman if (!has_comp && strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) { 603f1c54846SDan Streetman pr_err("compressor %s not available, using default %s\n", 604f1c54846SDan Streetman zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT); 605c99b42c3SDan Streetman param_free_charp(&zswap_compressor); 606c99b42c3SDan Streetman zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; 607bae21db8SDan Streetman has_comp = crypto_has_comp(zswap_compressor, 0, 0); 608f1c54846SDan Streetman } 609bae21db8SDan Streetman if (!has_comp) { 610bae21db8SDan Streetman pr_err("default compressor %s not available\n", 611bae21db8SDan Streetman zswap_compressor); 612bae21db8SDan Streetman param_free_charp(&zswap_compressor); 613bae21db8SDan Streetman zswap_compressor = ZSWAP_PARAM_UNSET; 614c99b42c3SDan Streetman } 615bae21db8SDan Streetman 616bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 617bae21db8SDan Streetman if (!has_zpool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) { 618f1c54846SDan Streetman pr_err("zpool %s not available, using default %s\n", 619f1c54846SDan Streetman zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT); 620c99b42c3SDan Streetman param_free_charp(&zswap_zpool_type); 621c99b42c3SDan Streetman zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; 622bae21db8SDan Streetman has_zpool = zpool_has_pool(zswap_zpool_type); 623f1c54846SDan Streetman } 624bae21db8SDan Streetman if (!has_zpool) { 625bae21db8SDan Streetman pr_err("default zpool %s not available\n", 626bae21db8SDan Streetman zswap_zpool_type); 627bae21db8SDan Streetman param_free_charp(&zswap_zpool_type); 628bae21db8SDan Streetman zswap_zpool_type = ZSWAP_PARAM_UNSET; 629bae21db8SDan Streetman } 630bae21db8SDan Streetman 631bae21db8SDan Streetman if (!has_comp || !has_zpool) 632bae21db8SDan Streetman return NULL; 633f1c54846SDan Streetman 634f1c54846SDan Streetman return zswap_pool_create(zswap_zpool_type, zswap_compressor); 635f1c54846SDan Streetman } 636f1c54846SDan Streetman 637f1c54846SDan Streetman static void zswap_pool_destroy(struct zswap_pool *pool) 638f1c54846SDan Streetman { 639f1c54846SDan Streetman zswap_pool_debug("destroying", pool); 640f1c54846SDan Streetman 641cab7a7e5SSebastian Andrzej Siewior cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 642f1c54846SDan Streetman free_percpu(pool->tfm); 643f1c54846SDan Streetman zpool_destroy_pool(pool->zpool); 644f1c54846SDan Streetman kfree(pool); 645f1c54846SDan Streetman } 646f1c54846SDan Streetman 647f1c54846SDan Streetman static int __must_check zswap_pool_get(struct zswap_pool *pool) 648f1c54846SDan Streetman { 649ae3d89a7SDan Streetman if (!pool) 650ae3d89a7SDan Streetman return 0; 651ae3d89a7SDan Streetman 652f1c54846SDan Streetman return kref_get_unless_zero(&pool->kref); 653f1c54846SDan Streetman } 654f1c54846SDan Streetman 655200867afSDan Streetman static void __zswap_pool_release(struct work_struct *work) 656f1c54846SDan Streetman { 657*45190f01SVitaly Wool struct zswap_pool *pool = container_of(work, typeof(*pool), 658*45190f01SVitaly Wool release_work); 659200867afSDan Streetman 660200867afSDan Streetman synchronize_rcu(); 661f1c54846SDan Streetman 662f1c54846SDan Streetman /* nobody should have been able to get a kref... */ 663f1c54846SDan Streetman WARN_ON(kref_get_unless_zero(&pool->kref)); 664f1c54846SDan Streetman 665f1c54846SDan Streetman /* pool is now off zswap_pools list and has no references. */ 666f1c54846SDan Streetman zswap_pool_destroy(pool); 667f1c54846SDan Streetman } 668f1c54846SDan Streetman 669f1c54846SDan Streetman static void __zswap_pool_empty(struct kref *kref) 670f1c54846SDan Streetman { 671f1c54846SDan Streetman struct zswap_pool *pool; 672f1c54846SDan Streetman 673f1c54846SDan Streetman pool = container_of(kref, typeof(*pool), kref); 674f1c54846SDan Streetman 675f1c54846SDan Streetman spin_lock(&zswap_pools_lock); 676f1c54846SDan Streetman 677f1c54846SDan Streetman WARN_ON(pool == zswap_pool_current()); 678f1c54846SDan Streetman 679f1c54846SDan Streetman list_del_rcu(&pool->list); 680200867afSDan Streetman 681*45190f01SVitaly Wool INIT_WORK(&pool->release_work, __zswap_pool_release); 682*45190f01SVitaly Wool schedule_work(&pool->release_work); 683f1c54846SDan Streetman 684f1c54846SDan Streetman spin_unlock(&zswap_pools_lock); 685f1c54846SDan Streetman } 686f1c54846SDan Streetman 687f1c54846SDan Streetman static void zswap_pool_put(struct zswap_pool *pool) 688f1c54846SDan Streetman { 689f1c54846SDan Streetman kref_put(&pool->kref, __zswap_pool_empty); 6902b281117SSeth Jennings } 6912b281117SSeth Jennings 6922b281117SSeth Jennings /********************************* 69390b0fc26SDan Streetman * param callbacks 69490b0fc26SDan Streetman **********************************/ 69590b0fc26SDan Streetman 696c99b42c3SDan Streetman /* val must be a null-terminated string */ 69790b0fc26SDan Streetman static int __zswap_param_set(const char *val, const struct kernel_param *kp, 69890b0fc26SDan Streetman char *type, char *compressor) 69990b0fc26SDan Streetman { 70090b0fc26SDan Streetman struct zswap_pool *pool, *put_pool = NULL; 701c99b42c3SDan Streetman char *s = strstrip((char *)val); 70290b0fc26SDan Streetman int ret; 70390b0fc26SDan Streetman 704d7b028f5SDan Streetman if (zswap_init_failed) { 705d7b028f5SDan Streetman pr_err("can't set param, initialization failed\n"); 706d7b028f5SDan Streetman return -ENODEV; 707d7b028f5SDan Streetman } 708d7b028f5SDan Streetman 709c99b42c3SDan Streetman /* no change required */ 710ae3d89a7SDan Streetman if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 711c99b42c3SDan Streetman return 0; 71290b0fc26SDan Streetman 71390b0fc26SDan Streetman /* if this is load-time (pre-init) param setting, 71490b0fc26SDan Streetman * don't create a pool; that's done during init. 71590b0fc26SDan Streetman */ 71690b0fc26SDan Streetman if (!zswap_init_started) 717c99b42c3SDan Streetman return param_set_charp(s, kp); 71890b0fc26SDan Streetman 71990b0fc26SDan Streetman if (!type) { 720c99b42c3SDan Streetman if (!zpool_has_pool(s)) { 721c99b42c3SDan Streetman pr_err("zpool %s not available\n", s); 722c99b42c3SDan Streetman return -ENOENT; 723c99b42c3SDan Streetman } 72490b0fc26SDan Streetman type = s; 72590b0fc26SDan Streetman } else if (!compressor) { 726c99b42c3SDan Streetman if (!crypto_has_comp(s, 0, 0)) { 727c99b42c3SDan Streetman pr_err("compressor %s not available\n", s); 72890b0fc26SDan Streetman return -ENOENT; 72990b0fc26SDan Streetman } 730c99b42c3SDan Streetman compressor = s; 731c99b42c3SDan Streetman } else { 732c99b42c3SDan Streetman WARN_ON(1); 733c99b42c3SDan Streetman return -EINVAL; 73490b0fc26SDan Streetman } 73590b0fc26SDan Streetman 73690b0fc26SDan Streetman spin_lock(&zswap_pools_lock); 73790b0fc26SDan Streetman 73890b0fc26SDan Streetman pool = zswap_pool_find_get(type, compressor); 73990b0fc26SDan Streetman if (pool) { 74090b0fc26SDan Streetman zswap_pool_debug("using existing", pool); 741fd5bb66cSDan Streetman WARN_ON(pool == zswap_pool_current()); 74290b0fc26SDan Streetman list_del_rcu(&pool->list); 74390b0fc26SDan Streetman } 74490b0fc26SDan Streetman 745fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 746fd5bb66cSDan Streetman 747fd5bb66cSDan Streetman if (!pool) 748fd5bb66cSDan Streetman pool = zswap_pool_create(type, compressor); 749fd5bb66cSDan Streetman 75090b0fc26SDan Streetman if (pool) 751c99b42c3SDan Streetman ret = param_set_charp(s, kp); 75290b0fc26SDan Streetman else 75390b0fc26SDan Streetman ret = -EINVAL; 75490b0fc26SDan Streetman 755fd5bb66cSDan Streetman spin_lock(&zswap_pools_lock); 756fd5bb66cSDan Streetman 75790b0fc26SDan Streetman if (!ret) { 75890b0fc26SDan Streetman put_pool = zswap_pool_current(); 75990b0fc26SDan Streetman list_add_rcu(&pool->list, &zswap_pools); 760ae3d89a7SDan Streetman zswap_has_pool = true; 76190b0fc26SDan Streetman } else if (pool) { 76290b0fc26SDan Streetman /* add the possibly pre-existing pool to the end of the pools 76390b0fc26SDan Streetman * list; if it's new (and empty) then it'll be removed and 76490b0fc26SDan Streetman * destroyed by the put after we drop the lock 76590b0fc26SDan Streetman */ 76690b0fc26SDan Streetman list_add_tail_rcu(&pool->list, &zswap_pools); 76790b0fc26SDan Streetman put_pool = pool; 768fd5bb66cSDan Streetman } 769fd5bb66cSDan Streetman 770fd5bb66cSDan Streetman spin_unlock(&zswap_pools_lock); 771fd5bb66cSDan Streetman 772fd5bb66cSDan Streetman if (!zswap_has_pool && !pool) { 773ae3d89a7SDan Streetman /* if initial pool creation failed, and this pool creation also 774ae3d89a7SDan Streetman * failed, maybe both compressor and zpool params were bad. 775ae3d89a7SDan Streetman * Allow changing this param, so pool creation will succeed 776ae3d89a7SDan Streetman * when the other param is changed. We already verified this 777ae3d89a7SDan Streetman * param is ok in the zpool_has_pool() or crypto_has_comp() 778ae3d89a7SDan Streetman * checks above. 779ae3d89a7SDan Streetman */ 780ae3d89a7SDan Streetman ret = param_set_charp(s, kp); 78190b0fc26SDan Streetman } 78290b0fc26SDan Streetman 78390b0fc26SDan Streetman /* drop the ref from either the old current pool, 78490b0fc26SDan Streetman * or the new pool we failed to add 78590b0fc26SDan Streetman */ 78690b0fc26SDan Streetman if (put_pool) 78790b0fc26SDan Streetman zswap_pool_put(put_pool); 78890b0fc26SDan Streetman 78990b0fc26SDan Streetman return ret; 79090b0fc26SDan Streetman } 79190b0fc26SDan Streetman 79290b0fc26SDan Streetman static int zswap_compressor_param_set(const char *val, 79390b0fc26SDan Streetman const struct kernel_param *kp) 79490b0fc26SDan Streetman { 79590b0fc26SDan Streetman return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 79690b0fc26SDan Streetman } 79790b0fc26SDan Streetman 79890b0fc26SDan Streetman static int zswap_zpool_param_set(const char *val, 79990b0fc26SDan Streetman const struct kernel_param *kp) 80090b0fc26SDan Streetman { 80190b0fc26SDan Streetman return __zswap_param_set(val, kp, NULL, zswap_compressor); 80290b0fc26SDan Streetman } 80390b0fc26SDan Streetman 804d7b028f5SDan Streetman static int zswap_enabled_param_set(const char *val, 805d7b028f5SDan Streetman const struct kernel_param *kp) 806d7b028f5SDan Streetman { 807d7b028f5SDan Streetman if (zswap_init_failed) { 808d7b028f5SDan Streetman pr_err("can't enable, initialization failed\n"); 809d7b028f5SDan Streetman return -ENODEV; 810d7b028f5SDan Streetman } 811ae3d89a7SDan Streetman if (!zswap_has_pool && zswap_init_started) { 812ae3d89a7SDan Streetman pr_err("can't enable, no pool configured\n"); 813ae3d89a7SDan Streetman return -ENODEV; 814ae3d89a7SDan Streetman } 815d7b028f5SDan Streetman 816d7b028f5SDan Streetman return param_set_bool(val, kp); 817d7b028f5SDan Streetman } 818d7b028f5SDan Streetman 81990b0fc26SDan Streetman /********************************* 8202b281117SSeth Jennings * writeback code 8212b281117SSeth Jennings **********************************/ 8222b281117SSeth Jennings /* return enum for zswap_get_swap_cache_page */ 8232b281117SSeth Jennings enum zswap_get_swap_ret { 8242b281117SSeth Jennings ZSWAP_SWAPCACHE_NEW, 8252b281117SSeth Jennings ZSWAP_SWAPCACHE_EXIST, 82667d13fe8SWeijie Yang ZSWAP_SWAPCACHE_FAIL, 8272b281117SSeth Jennings }; 8282b281117SSeth Jennings 8292b281117SSeth Jennings /* 8302b281117SSeth Jennings * zswap_get_swap_cache_page 8312b281117SSeth Jennings * 8322b281117SSeth Jennings * This is an adaption of read_swap_cache_async() 8332b281117SSeth Jennings * 8342b281117SSeth Jennings * This function tries to find a page with the given swap entry 8352b281117SSeth Jennings * in the swapper_space address space (the swap cache). If the page 8362b281117SSeth Jennings * is found, it is returned in retpage. Otherwise, a page is allocated, 8372b281117SSeth Jennings * added to the swap cache, and returned in retpage. 8382b281117SSeth Jennings * 8392b281117SSeth Jennings * If success, the swap cache page is returned in retpage 84067d13fe8SWeijie Yang * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache 84167d13fe8SWeijie Yang * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated, 84267d13fe8SWeijie Yang * the new page is added to swapcache and locked 84367d13fe8SWeijie Yang * Returns ZSWAP_SWAPCACHE_FAIL on error 8442b281117SSeth Jennings */ 8452b281117SSeth Jennings static int zswap_get_swap_cache_page(swp_entry_t entry, 8462b281117SSeth Jennings struct page **retpage) 8472b281117SSeth Jennings { 8485b999aadSDmitry Safonov bool page_was_allocated; 8492b281117SSeth Jennings 8505b999aadSDmitry Safonov *retpage = __read_swap_cache_async(entry, GFP_KERNEL, 8515b999aadSDmitry Safonov NULL, 0, &page_was_allocated); 8525b999aadSDmitry Safonov if (page_was_allocated) 8532b281117SSeth Jennings return ZSWAP_SWAPCACHE_NEW; 8545b999aadSDmitry Safonov if (!*retpage) 85567d13fe8SWeijie Yang return ZSWAP_SWAPCACHE_FAIL; 8562b281117SSeth Jennings return ZSWAP_SWAPCACHE_EXIST; 8572b281117SSeth Jennings } 8582b281117SSeth Jennings 8592b281117SSeth Jennings /* 8602b281117SSeth Jennings * Attempts to free an entry by adding a page to the swap cache, 8612b281117SSeth Jennings * decompressing the entry data into the page, and issuing a 8622b281117SSeth Jennings * bio write to write the page back to the swap device. 8632b281117SSeth Jennings * 8642b281117SSeth Jennings * This can be thought of as a "resumed writeback" of the page 8652b281117SSeth Jennings * to the swap device. We are basically resuming the same swap 8662b281117SSeth Jennings * writeback path that was intercepted with the frontswap_store() 8672b281117SSeth Jennings * in the first place. After the page has been decompressed into 8682b281117SSeth Jennings * the swap cache, the compressed version stored by zswap can be 8692b281117SSeth Jennings * freed. 8702b281117SSeth Jennings */ 87112d79d64SDan Streetman static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) 8722b281117SSeth Jennings { 8732b281117SSeth Jennings struct zswap_header *zhdr; 8742b281117SSeth Jennings swp_entry_t swpentry; 8752b281117SSeth Jennings struct zswap_tree *tree; 8762b281117SSeth Jennings pgoff_t offset; 8772b281117SSeth Jennings struct zswap_entry *entry; 8782b281117SSeth Jennings struct page *page; 879f1c54846SDan Streetman struct crypto_comp *tfm; 8802b281117SSeth Jennings u8 *src, *dst; 8812b281117SSeth Jennings unsigned int dlen; 8820ab0abcfSWeijie Yang int ret; 8832b281117SSeth Jennings struct writeback_control wbc = { 8842b281117SSeth Jennings .sync_mode = WB_SYNC_NONE, 8852b281117SSeth Jennings }; 8862b281117SSeth Jennings 8872b281117SSeth Jennings /* extract swpentry from data */ 88812d79d64SDan Streetman zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO); 8892b281117SSeth Jennings swpentry = zhdr->swpentry; /* here */ 8902b281117SSeth Jennings tree = zswap_trees[swp_type(swpentry)]; 8912b281117SSeth Jennings offset = swp_offset(swpentry); 8922b281117SSeth Jennings 8932b281117SSeth Jennings /* find and ref zswap entry */ 8942b281117SSeth Jennings spin_lock(&tree->lock); 8950ab0abcfSWeijie Yang entry = zswap_entry_find_get(&tree->rbroot, offset); 8962b281117SSeth Jennings if (!entry) { 8972b281117SSeth Jennings /* entry was invalidated */ 8982b281117SSeth Jennings spin_unlock(&tree->lock); 899068619e3SVitaly Wool zpool_unmap_handle(pool, handle); 9002b281117SSeth Jennings return 0; 9012b281117SSeth Jennings } 9022b281117SSeth Jennings spin_unlock(&tree->lock); 9032b281117SSeth Jennings BUG_ON(offset != entry->offset); 9042b281117SSeth Jennings 9052b281117SSeth Jennings /* try to allocate swap cache page */ 9062b281117SSeth Jennings switch (zswap_get_swap_cache_page(swpentry, &page)) { 90767d13fe8SWeijie Yang case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */ 9082b281117SSeth Jennings ret = -ENOMEM; 9092b281117SSeth Jennings goto fail; 9102b281117SSeth Jennings 91167d13fe8SWeijie Yang case ZSWAP_SWAPCACHE_EXIST: 9122b281117SSeth Jennings /* page is already in the swap cache, ignore for now */ 91309cbfeafSKirill A. Shutemov put_page(page); 9142b281117SSeth Jennings ret = -EEXIST; 9152b281117SSeth Jennings goto fail; 9162b281117SSeth Jennings 9172b281117SSeth Jennings case ZSWAP_SWAPCACHE_NEW: /* page is locked */ 9182b281117SSeth Jennings /* decompress */ 9192b281117SSeth Jennings dlen = PAGE_SIZE; 920068619e3SVitaly Wool src = (u8 *)zhdr + sizeof(struct zswap_header); 9212b281117SSeth Jennings dst = kmap_atomic(page); 922f1c54846SDan Streetman tfm = *get_cpu_ptr(entry->pool->tfm); 923f1c54846SDan Streetman ret = crypto_comp_decompress(tfm, src, entry->length, 924f1c54846SDan Streetman dst, &dlen); 925f1c54846SDan Streetman put_cpu_ptr(entry->pool->tfm); 9262b281117SSeth Jennings kunmap_atomic(dst); 9272b281117SSeth Jennings BUG_ON(ret); 9282b281117SSeth Jennings BUG_ON(dlen != PAGE_SIZE); 9292b281117SSeth Jennings 9302b281117SSeth Jennings /* page is up to date */ 9312b281117SSeth Jennings SetPageUptodate(page); 9322b281117SSeth Jennings } 9332b281117SSeth Jennings 934b349acc7SWeijie Yang /* move it to the tail of the inactive list after end_writeback */ 935b349acc7SWeijie Yang SetPageReclaim(page); 936b349acc7SWeijie Yang 9372b281117SSeth Jennings /* start writeback */ 9382b281117SSeth Jennings __swap_writepage(page, &wbc, end_swap_bio_write); 93909cbfeafSKirill A. Shutemov put_page(page); 9402b281117SSeth Jennings zswap_written_back_pages++; 9412b281117SSeth Jennings 9422b281117SSeth Jennings spin_lock(&tree->lock); 9432b281117SSeth Jennings /* drop local reference */ 9440ab0abcfSWeijie Yang zswap_entry_put(tree, entry); 9452b281117SSeth Jennings 9462b281117SSeth Jennings /* 9470ab0abcfSWeijie Yang * There are two possible situations for entry here: 9480ab0abcfSWeijie Yang * (1) refcount is 1(normal case), entry is valid and on the tree 9490ab0abcfSWeijie Yang * (2) refcount is 0, entry is freed and not on the tree 9500ab0abcfSWeijie Yang * because invalidate happened during writeback 9510ab0abcfSWeijie Yang * search the tree and free the entry if find entry 9522b281117SSeth Jennings */ 9530ab0abcfSWeijie Yang if (entry == zswap_rb_search(&tree->rbroot, offset)) 9540ab0abcfSWeijie Yang zswap_entry_put(tree, entry); 9552b281117SSeth Jennings spin_unlock(&tree->lock); 9562b281117SSeth Jennings 9570ab0abcfSWeijie Yang goto end; 9580ab0abcfSWeijie Yang 9590ab0abcfSWeijie Yang /* 9600ab0abcfSWeijie Yang * if we get here due to ZSWAP_SWAPCACHE_EXIST 9610ab0abcfSWeijie Yang * a load may happening concurrently 9620ab0abcfSWeijie Yang * it is safe and okay to not free the entry 9630ab0abcfSWeijie Yang * if we free the entry in the following put 9640ab0abcfSWeijie Yang * it it either okay to return !0 9650ab0abcfSWeijie Yang */ 9662b281117SSeth Jennings fail: 9672b281117SSeth Jennings spin_lock(&tree->lock); 9680ab0abcfSWeijie Yang zswap_entry_put(tree, entry); 9692b281117SSeth Jennings spin_unlock(&tree->lock); 9700ab0abcfSWeijie Yang 9710ab0abcfSWeijie Yang end: 972068619e3SVitaly Wool zpool_unmap_handle(pool, handle); 9732b281117SSeth Jennings return ret; 9742b281117SSeth Jennings } 9752b281117SSeth Jennings 976a85f878bSSrividya Desireddy static int zswap_is_page_same_filled(void *ptr, unsigned long *value) 977a85f878bSSrividya Desireddy { 978a85f878bSSrividya Desireddy unsigned int pos; 979a85f878bSSrividya Desireddy unsigned long *page; 980a85f878bSSrividya Desireddy 981a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 982a85f878bSSrividya Desireddy for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) { 983a85f878bSSrividya Desireddy if (page[pos] != page[0]) 984a85f878bSSrividya Desireddy return 0; 985a85f878bSSrividya Desireddy } 986a85f878bSSrividya Desireddy *value = page[0]; 987a85f878bSSrividya Desireddy return 1; 988a85f878bSSrividya Desireddy } 989a85f878bSSrividya Desireddy 990a85f878bSSrividya Desireddy static void zswap_fill_page(void *ptr, unsigned long value) 991a85f878bSSrividya Desireddy { 992a85f878bSSrividya Desireddy unsigned long *page; 993a85f878bSSrividya Desireddy 994a85f878bSSrividya Desireddy page = (unsigned long *)ptr; 995a85f878bSSrividya Desireddy memset_l(page, value, PAGE_SIZE / sizeof(unsigned long)); 996a85f878bSSrividya Desireddy } 997a85f878bSSrividya Desireddy 9982b281117SSeth Jennings /********************************* 9992b281117SSeth Jennings * frontswap hooks 10002b281117SSeth Jennings **********************************/ 10012b281117SSeth Jennings /* attempts to compress and store an single page */ 10022b281117SSeth Jennings static int zswap_frontswap_store(unsigned type, pgoff_t offset, 10032b281117SSeth Jennings struct page *page) 10042b281117SSeth Jennings { 10052b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 10062b281117SSeth Jennings struct zswap_entry *entry, *dupentry; 1007f1c54846SDan Streetman struct crypto_comp *tfm; 10082b281117SSeth Jennings int ret; 10099c3760ebSYu Zhao unsigned int hlen, dlen = PAGE_SIZE; 1010a85f878bSSrividya Desireddy unsigned long handle, value; 10112b281117SSeth Jennings char *buf; 10122b281117SSeth Jennings u8 *src, *dst; 10139c3760ebSYu Zhao struct zswap_header zhdr = { .swpentry = swp_entry(type, offset) }; 1014d2fcd82bSHui Zhu gfp_t gfp; 10152b281117SSeth Jennings 10167ba71669SHuang Ying /* THP isn't supported */ 10177ba71669SHuang Ying if (PageTransHuge(page)) { 10187ba71669SHuang Ying ret = -EINVAL; 10197ba71669SHuang Ying goto reject; 10207ba71669SHuang Ying } 10217ba71669SHuang Ying 1022c00ed16aSDan Streetman if (!zswap_enabled || !tree) { 10232b281117SSeth Jennings ret = -ENODEV; 10242b281117SSeth Jennings goto reject; 10252b281117SSeth Jennings } 10262b281117SSeth Jennings 10272b281117SSeth Jennings /* reclaim space if needed */ 10282b281117SSeth Jennings if (zswap_is_full()) { 1029*45190f01SVitaly Wool struct zswap_pool *pool; 1030*45190f01SVitaly Wool 10312b281117SSeth Jennings zswap_pool_limit_hit++; 1032*45190f01SVitaly Wool zswap_pool_reached_full = true; 1033*45190f01SVitaly Wool pool = zswap_pool_last_get(); 1034*45190f01SVitaly Wool if (pool) 1035*45190f01SVitaly Wool queue_work(shrink_wq, &pool->shrink_work); 10362b281117SSeth Jennings ret = -ENOMEM; 10372b281117SSeth Jennings goto reject; 10382b281117SSeth Jennings } 103916e536efSLi Wang 1040*45190f01SVitaly Wool if (zswap_pool_reached_full) { 1041*45190f01SVitaly Wool if (!zswap_can_accept()) { 104216e536efSLi Wang ret = -ENOMEM; 104316e536efSLi Wang goto reject; 1044*45190f01SVitaly Wool } else 1045*45190f01SVitaly Wool zswap_pool_reached_full = false; 10462b281117SSeth Jennings } 10472b281117SSeth Jennings 10482b281117SSeth Jennings /* allocate entry */ 10492b281117SSeth Jennings entry = zswap_entry_cache_alloc(GFP_KERNEL); 10502b281117SSeth Jennings if (!entry) { 10512b281117SSeth Jennings zswap_reject_kmemcache_fail++; 10522b281117SSeth Jennings ret = -ENOMEM; 10532b281117SSeth Jennings goto reject; 10542b281117SSeth Jennings } 10552b281117SSeth Jennings 1056a85f878bSSrividya Desireddy if (zswap_same_filled_pages_enabled) { 1057a85f878bSSrividya Desireddy src = kmap_atomic(page); 1058a85f878bSSrividya Desireddy if (zswap_is_page_same_filled(src, &value)) { 1059a85f878bSSrividya Desireddy kunmap_atomic(src); 1060a85f878bSSrividya Desireddy entry->offset = offset; 1061a85f878bSSrividya Desireddy entry->length = 0; 1062a85f878bSSrividya Desireddy entry->value = value; 1063a85f878bSSrividya Desireddy atomic_inc(&zswap_same_filled_pages); 1064a85f878bSSrividya Desireddy goto insert_entry; 1065a85f878bSSrividya Desireddy } 1066a85f878bSSrividya Desireddy kunmap_atomic(src); 1067a85f878bSSrividya Desireddy } 1068a85f878bSSrividya Desireddy 1069f1c54846SDan Streetman /* if entry is successfully added, it keeps the reference */ 1070f1c54846SDan Streetman entry->pool = zswap_pool_current_get(); 1071f1c54846SDan Streetman if (!entry->pool) { 10722b281117SSeth Jennings ret = -EINVAL; 10732b281117SSeth Jennings goto freepage; 10742b281117SSeth Jennings } 10752b281117SSeth Jennings 1076f1c54846SDan Streetman /* compress */ 1077f1c54846SDan Streetman dst = get_cpu_var(zswap_dstmem); 1078f1c54846SDan Streetman tfm = *get_cpu_ptr(entry->pool->tfm); 1079f1c54846SDan Streetman src = kmap_atomic(page); 1080f1c54846SDan Streetman ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen); 1081f1c54846SDan Streetman kunmap_atomic(src); 1082f1c54846SDan Streetman put_cpu_ptr(entry->pool->tfm); 1083f1c54846SDan Streetman if (ret) { 1084f1c54846SDan Streetman ret = -EINVAL; 1085f1c54846SDan Streetman goto put_dstmem; 1086f1c54846SDan Streetman } 1087f1c54846SDan Streetman 10882b281117SSeth Jennings /* store */ 10899c3760ebSYu Zhao hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0; 1090d2fcd82bSHui Zhu gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 1091d2fcd82bSHui Zhu if (zpool_malloc_support_movable(entry->pool->zpool)) 1092d2fcd82bSHui Zhu gfp |= __GFP_HIGHMEM | __GFP_MOVABLE; 1093d2fcd82bSHui Zhu ret = zpool_malloc(entry->pool->zpool, hlen + dlen, gfp, &handle); 10942b281117SSeth Jennings if (ret == -ENOSPC) { 10952b281117SSeth Jennings zswap_reject_compress_poor++; 1096f1c54846SDan Streetman goto put_dstmem; 10972b281117SSeth Jennings } 10982b281117SSeth Jennings if (ret) { 10992b281117SSeth Jennings zswap_reject_alloc_fail++; 1100f1c54846SDan Streetman goto put_dstmem; 11012b281117SSeth Jennings } 11029c3760ebSYu Zhao buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW); 11039c3760ebSYu Zhao memcpy(buf, &zhdr, hlen); 11049c3760ebSYu Zhao memcpy(buf + hlen, dst, dlen); 1105f1c54846SDan Streetman zpool_unmap_handle(entry->pool->zpool, handle); 11062b281117SSeth Jennings put_cpu_var(zswap_dstmem); 11072b281117SSeth Jennings 11082b281117SSeth Jennings /* populate entry */ 11092b281117SSeth Jennings entry->offset = offset; 11102b281117SSeth Jennings entry->handle = handle; 11112b281117SSeth Jennings entry->length = dlen; 11122b281117SSeth Jennings 1113a85f878bSSrividya Desireddy insert_entry: 11142b281117SSeth Jennings /* map */ 11152b281117SSeth Jennings spin_lock(&tree->lock); 11162b281117SSeth Jennings do { 11172b281117SSeth Jennings ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry); 11182b281117SSeth Jennings if (ret == -EEXIST) { 11192b281117SSeth Jennings zswap_duplicate_entry++; 11202b281117SSeth Jennings /* remove from rbtree */ 11210ab0abcfSWeijie Yang zswap_rb_erase(&tree->rbroot, dupentry); 11220ab0abcfSWeijie Yang zswap_entry_put(tree, dupentry); 11232b281117SSeth Jennings } 11242b281117SSeth Jennings } while (ret == -EEXIST); 11252b281117SSeth Jennings spin_unlock(&tree->lock); 11262b281117SSeth Jennings 11272b281117SSeth Jennings /* update stats */ 11282b281117SSeth Jennings atomic_inc(&zswap_stored_pages); 1129f1c54846SDan Streetman zswap_update_total_size(); 11302b281117SSeth Jennings 11312b281117SSeth Jennings return 0; 11322b281117SSeth Jennings 1133f1c54846SDan Streetman put_dstmem: 11342b281117SSeth Jennings put_cpu_var(zswap_dstmem); 1135f1c54846SDan Streetman zswap_pool_put(entry->pool); 1136f1c54846SDan Streetman freepage: 11372b281117SSeth Jennings zswap_entry_cache_free(entry); 11382b281117SSeth Jennings reject: 11392b281117SSeth Jennings return ret; 11402b281117SSeth Jennings } 11412b281117SSeth Jennings 11422b281117SSeth Jennings /* 11432b281117SSeth Jennings * returns 0 if the page was successfully decompressed 11442b281117SSeth Jennings * return -1 on entry not found or error 11452b281117SSeth Jennings */ 11462b281117SSeth Jennings static int zswap_frontswap_load(unsigned type, pgoff_t offset, 11472b281117SSeth Jennings struct page *page) 11482b281117SSeth Jennings { 11492b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 11502b281117SSeth Jennings struct zswap_entry *entry; 1151f1c54846SDan Streetman struct crypto_comp *tfm; 11522b281117SSeth Jennings u8 *src, *dst; 11532b281117SSeth Jennings unsigned int dlen; 11540ab0abcfSWeijie Yang int ret; 11552b281117SSeth Jennings 11562b281117SSeth Jennings /* find */ 11572b281117SSeth Jennings spin_lock(&tree->lock); 11580ab0abcfSWeijie Yang entry = zswap_entry_find_get(&tree->rbroot, offset); 11592b281117SSeth Jennings if (!entry) { 11602b281117SSeth Jennings /* entry was written back */ 11612b281117SSeth Jennings spin_unlock(&tree->lock); 11622b281117SSeth Jennings return -1; 11632b281117SSeth Jennings } 11642b281117SSeth Jennings spin_unlock(&tree->lock); 11652b281117SSeth Jennings 1166a85f878bSSrividya Desireddy if (!entry->length) { 1167a85f878bSSrividya Desireddy dst = kmap_atomic(page); 1168a85f878bSSrividya Desireddy zswap_fill_page(dst, entry->value); 1169a85f878bSSrividya Desireddy kunmap_atomic(dst); 1170a85f878bSSrividya Desireddy goto freeentry; 1171a85f878bSSrividya Desireddy } 1172a85f878bSSrividya Desireddy 11732b281117SSeth Jennings /* decompress */ 11742b281117SSeth Jennings dlen = PAGE_SIZE; 11759c3760ebSYu Zhao src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO); 11769c3760ebSYu Zhao if (zpool_evictable(entry->pool->zpool)) 11779c3760ebSYu Zhao src += sizeof(struct zswap_header); 11782b281117SSeth Jennings dst = kmap_atomic(page); 1179f1c54846SDan Streetman tfm = *get_cpu_ptr(entry->pool->tfm); 1180f1c54846SDan Streetman ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen); 1181f1c54846SDan Streetman put_cpu_ptr(entry->pool->tfm); 11822b281117SSeth Jennings kunmap_atomic(dst); 1183f1c54846SDan Streetman zpool_unmap_handle(entry->pool->zpool, entry->handle); 11842b281117SSeth Jennings BUG_ON(ret); 11852b281117SSeth Jennings 1186a85f878bSSrividya Desireddy freeentry: 11872b281117SSeth Jennings spin_lock(&tree->lock); 11880ab0abcfSWeijie Yang zswap_entry_put(tree, entry); 11892b281117SSeth Jennings spin_unlock(&tree->lock); 11902b281117SSeth Jennings 11912b281117SSeth Jennings return 0; 11922b281117SSeth Jennings } 11932b281117SSeth Jennings 11942b281117SSeth Jennings /* frees an entry in zswap */ 11952b281117SSeth Jennings static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset) 11962b281117SSeth Jennings { 11972b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 11982b281117SSeth Jennings struct zswap_entry *entry; 11992b281117SSeth Jennings 12002b281117SSeth Jennings /* find */ 12012b281117SSeth Jennings spin_lock(&tree->lock); 12022b281117SSeth Jennings entry = zswap_rb_search(&tree->rbroot, offset); 12032b281117SSeth Jennings if (!entry) { 12042b281117SSeth Jennings /* entry was written back */ 12052b281117SSeth Jennings spin_unlock(&tree->lock); 12062b281117SSeth Jennings return; 12072b281117SSeth Jennings } 12082b281117SSeth Jennings 12092b281117SSeth Jennings /* remove from rbtree */ 12100ab0abcfSWeijie Yang zswap_rb_erase(&tree->rbroot, entry); 12112b281117SSeth Jennings 12122b281117SSeth Jennings /* drop the initial reference from entry creation */ 12130ab0abcfSWeijie Yang zswap_entry_put(tree, entry); 12142b281117SSeth Jennings 12152b281117SSeth Jennings spin_unlock(&tree->lock); 12162b281117SSeth Jennings } 12172b281117SSeth Jennings 12182b281117SSeth Jennings /* frees all zswap entries for the given swap type */ 12192b281117SSeth Jennings static void zswap_frontswap_invalidate_area(unsigned type) 12202b281117SSeth Jennings { 12212b281117SSeth Jennings struct zswap_tree *tree = zswap_trees[type]; 12220bd42136SCody P Schafer struct zswap_entry *entry, *n; 12232b281117SSeth Jennings 12242b281117SSeth Jennings if (!tree) 12252b281117SSeth Jennings return; 12262b281117SSeth Jennings 12272b281117SSeth Jennings /* walk the tree and free everything */ 12282b281117SSeth Jennings spin_lock(&tree->lock); 12290ab0abcfSWeijie Yang rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) 123060105e12SMinchan Kim zswap_free_entry(entry); 12312b281117SSeth Jennings tree->rbroot = RB_ROOT; 12322b281117SSeth Jennings spin_unlock(&tree->lock); 1233aa9bca05SWeijie Yang kfree(tree); 1234aa9bca05SWeijie Yang zswap_trees[type] = NULL; 12352b281117SSeth Jennings } 12362b281117SSeth Jennings 12372b281117SSeth Jennings static void zswap_frontswap_init(unsigned type) 12382b281117SSeth Jennings { 12392b281117SSeth Jennings struct zswap_tree *tree; 12402b281117SSeth Jennings 12419cd1f701SMarkus Elfring tree = kzalloc(sizeof(*tree), GFP_KERNEL); 124260105e12SMinchan Kim if (!tree) { 124360105e12SMinchan Kim pr_err("alloc failed, zswap disabled for swap type %d\n", type); 124460105e12SMinchan Kim return; 124560105e12SMinchan Kim } 124660105e12SMinchan Kim 12472b281117SSeth Jennings tree->rbroot = RB_ROOT; 12482b281117SSeth Jennings spin_lock_init(&tree->lock); 12492b281117SSeth Jennings zswap_trees[type] = tree; 12502b281117SSeth Jennings } 12512b281117SSeth Jennings 12522b281117SSeth Jennings static struct frontswap_ops zswap_frontswap_ops = { 12532b281117SSeth Jennings .store = zswap_frontswap_store, 12542b281117SSeth Jennings .load = zswap_frontswap_load, 12552b281117SSeth Jennings .invalidate_page = zswap_frontswap_invalidate_page, 12562b281117SSeth Jennings .invalidate_area = zswap_frontswap_invalidate_area, 12572b281117SSeth Jennings .init = zswap_frontswap_init 12582b281117SSeth Jennings }; 12592b281117SSeth Jennings 12602b281117SSeth Jennings /********************************* 12612b281117SSeth Jennings * debugfs functions 12622b281117SSeth Jennings **********************************/ 12632b281117SSeth Jennings #ifdef CONFIG_DEBUG_FS 12642b281117SSeth Jennings #include <linux/debugfs.h> 12652b281117SSeth Jennings 12662b281117SSeth Jennings static struct dentry *zswap_debugfs_root; 12672b281117SSeth Jennings 12682b281117SSeth Jennings static int __init zswap_debugfs_init(void) 12692b281117SSeth Jennings { 12702b281117SSeth Jennings if (!debugfs_initialized()) 12712b281117SSeth Jennings return -ENODEV; 12722b281117SSeth Jennings 12732b281117SSeth Jennings zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 12742b281117SSeth Jennings 12750825a6f9SJoe Perches debugfs_create_u64("pool_limit_hit", 0444, 12762b281117SSeth Jennings zswap_debugfs_root, &zswap_pool_limit_hit); 12770825a6f9SJoe Perches debugfs_create_u64("reject_reclaim_fail", 0444, 12782b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_reclaim_fail); 12790825a6f9SJoe Perches debugfs_create_u64("reject_alloc_fail", 0444, 12802b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_alloc_fail); 12810825a6f9SJoe Perches debugfs_create_u64("reject_kmemcache_fail", 0444, 12822b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_kmemcache_fail); 12830825a6f9SJoe Perches debugfs_create_u64("reject_compress_poor", 0444, 12842b281117SSeth Jennings zswap_debugfs_root, &zswap_reject_compress_poor); 12850825a6f9SJoe Perches debugfs_create_u64("written_back_pages", 0444, 12862b281117SSeth Jennings zswap_debugfs_root, &zswap_written_back_pages); 12870825a6f9SJoe Perches debugfs_create_u64("duplicate_entry", 0444, 12882b281117SSeth Jennings zswap_debugfs_root, &zswap_duplicate_entry); 12890825a6f9SJoe Perches debugfs_create_u64("pool_total_size", 0444, 129012d79d64SDan Streetman zswap_debugfs_root, &zswap_pool_total_size); 12910825a6f9SJoe Perches debugfs_create_atomic_t("stored_pages", 0444, 12922b281117SSeth Jennings zswap_debugfs_root, &zswap_stored_pages); 1293a85f878bSSrividya Desireddy debugfs_create_atomic_t("same_filled_pages", 0444, 1294a85f878bSSrividya Desireddy zswap_debugfs_root, &zswap_same_filled_pages); 12952b281117SSeth Jennings 12962b281117SSeth Jennings return 0; 12972b281117SSeth Jennings } 12982b281117SSeth Jennings 12992b281117SSeth Jennings static void __exit zswap_debugfs_exit(void) 13002b281117SSeth Jennings { 13012b281117SSeth Jennings debugfs_remove_recursive(zswap_debugfs_root); 13022b281117SSeth Jennings } 13032b281117SSeth Jennings #else 13042b281117SSeth Jennings static int __init zswap_debugfs_init(void) 13052b281117SSeth Jennings { 13062b281117SSeth Jennings return 0; 13072b281117SSeth Jennings } 13082b281117SSeth Jennings 13092b281117SSeth Jennings static void __exit zswap_debugfs_exit(void) { } 13102b281117SSeth Jennings #endif 13112b281117SSeth Jennings 13122b281117SSeth Jennings /********************************* 13132b281117SSeth Jennings * module init and exit 13142b281117SSeth Jennings **********************************/ 13152b281117SSeth Jennings static int __init init_zswap(void) 13162b281117SSeth Jennings { 1317f1c54846SDan Streetman struct zswap_pool *pool; 1318ad7ed770SSebastian Andrzej Siewior int ret; 131960105e12SMinchan Kim 132090b0fc26SDan Streetman zswap_init_started = true; 132190b0fc26SDan Streetman 13222b281117SSeth Jennings if (zswap_entry_cache_create()) { 13232b281117SSeth Jennings pr_err("entry cache creation failed\n"); 1324f1c54846SDan Streetman goto cache_fail; 13252b281117SSeth Jennings } 1326f1c54846SDan Streetman 1327ad7ed770SSebastian Andrzej Siewior ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare", 1328ad7ed770SSebastian Andrzej Siewior zswap_dstmem_prepare, zswap_dstmem_dead); 1329ad7ed770SSebastian Andrzej Siewior if (ret) { 1330f1c54846SDan Streetman pr_err("dstmem alloc failed\n"); 1331f1c54846SDan Streetman goto dstmem_fail; 13322b281117SSeth Jennings } 1333f1c54846SDan Streetman 1334cab7a7e5SSebastian Andrzej Siewior ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, 1335cab7a7e5SSebastian Andrzej Siewior "mm/zswap_pool:prepare", 1336cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_prepare, 1337cab7a7e5SSebastian Andrzej Siewior zswap_cpu_comp_dead); 1338cab7a7e5SSebastian Andrzej Siewior if (ret) 1339cab7a7e5SSebastian Andrzej Siewior goto hp_fail; 1340cab7a7e5SSebastian Andrzej Siewior 1341f1c54846SDan Streetman pool = __zswap_pool_create_fallback(); 1342ae3d89a7SDan Streetman if (pool) { 1343f1c54846SDan Streetman pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1344f1c54846SDan Streetman zpool_get_type(pool->zpool)); 1345f1c54846SDan Streetman list_add(&pool->list, &zswap_pools); 1346ae3d89a7SDan Streetman zswap_has_pool = true; 1347ae3d89a7SDan Streetman } else { 1348ae3d89a7SDan Streetman pr_err("pool creation failed\n"); 1349ae3d89a7SDan Streetman zswap_enabled = false; 1350ae3d89a7SDan Streetman } 135160105e12SMinchan Kim 1352*45190f01SVitaly Wool shrink_wq = create_workqueue("zswap-shrink"); 1353*45190f01SVitaly Wool if (!shrink_wq) 1354*45190f01SVitaly Wool goto fallback_fail; 1355*45190f01SVitaly Wool 13562b281117SSeth Jennings frontswap_register_ops(&zswap_frontswap_ops); 13572b281117SSeth Jennings if (zswap_debugfs_init()) 13582b281117SSeth Jennings pr_warn("debugfs initialization failed\n"); 13592b281117SSeth Jennings return 0; 1360f1c54846SDan Streetman 1361*45190f01SVitaly Wool fallback_fail: 1362*45190f01SVitaly Wool zswap_pool_destroy(pool); 1363cab7a7e5SSebastian Andrzej Siewior hp_fail: 1364ad7ed770SSebastian Andrzej Siewior cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); 1365f1c54846SDan Streetman dstmem_fail: 1366c119239bSFabian Frederick zswap_entry_cache_destroy(); 1367f1c54846SDan Streetman cache_fail: 1368d7b028f5SDan Streetman /* if built-in, we aren't unloaded on failure; don't allow use */ 1369d7b028f5SDan Streetman zswap_init_failed = true; 1370d7b028f5SDan Streetman zswap_enabled = false; 13712b281117SSeth Jennings return -ENOMEM; 13722b281117SSeth Jennings } 13732b281117SSeth Jennings /* must be late so crypto has time to come up */ 13742b281117SSeth Jennings late_initcall(init_zswap); 13752b281117SSeth Jennings 13762b281117SSeth Jennings MODULE_LICENSE("GPL"); 137768386da8SSeth Jennings MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 13782b281117SSeth Jennings MODULE_DESCRIPTION("Compressed cache for swap pages"); 1379