xref: /linux/mm/zswap.c (revision 1110ce6a1e34fe1fdc1bfe4ad52405f327d5083b)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * zswap.c - zswap driver file
4  *
5  * zswap is a cache that takes pages that are in the process
6  * of being swapped out and attempts to compress and store them in a
7  * RAM-based memory pool.  This can result in a significant I/O reduction on
8  * the swap device and, in the case where decompressing from RAM is faster
9  * than reading from the swap device, can also improve workload performance.
10  *
11  * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
12 */
13 
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 
16 #include <linux/module.h>
17 #include <linux/cpu.h>
18 #include <linux/highmem.h>
19 #include <linux/slab.h>
20 #include <linux/spinlock.h>
21 #include <linux/types.h>
22 #include <linux/atomic.h>
23 #include <linux/swap.h>
24 #include <linux/crypto.h>
25 #include <linux/scatterlist.h>
26 #include <linux/mempolicy.h>
27 #include <linux/mempool.h>
28 #include <linux/zpool.h>
29 #include <crypto/acompress.h>
30 #include <linux/zswap.h>
31 #include <linux/mm_types.h>
32 #include <linux/page-flags.h>
33 #include <linux/swapops.h>
34 #include <linux/writeback.h>
35 #include <linux/pagemap.h>
36 #include <linux/workqueue.h>
37 #include <linux/list_lru.h>
38 
39 #include "swap.h"
40 #include "internal.h"
41 
42 /*********************************
43 * statistics
44 **********************************/
45 /* The number of compressed pages currently stored in zswap */
46 atomic_long_t zswap_stored_pages = ATOMIC_LONG_INIT(0);
47 
48 /*
49  * The statistics below are not protected from concurrent access for
50  * performance reasons so they may not be a 100% accurate.  However,
51  * they do provide useful information on roughly how many times a
52  * certain event is occurring.
53 */
54 
55 /* Pool limit was hit (see zswap_max_pool_percent) */
56 static u64 zswap_pool_limit_hit;
57 /* Pages written back when pool limit was reached */
58 static u64 zswap_written_back_pages;
59 /* Store failed due to a reclaim failure after pool limit was reached */
60 static u64 zswap_reject_reclaim_fail;
61 /* Store failed due to compression algorithm failure */
62 static u64 zswap_reject_compress_fail;
63 /* Compressed page was too big for the allocator to (optimally) store */
64 static u64 zswap_reject_compress_poor;
65 /* Store failed because underlying allocator could not get memory */
66 static u64 zswap_reject_alloc_fail;
67 /* Store failed because the entry metadata could not be allocated (rare) */
68 static u64 zswap_reject_kmemcache_fail;
69 
70 /* Shrinker work queue */
71 static struct workqueue_struct *shrink_wq;
72 /* Pool limit was hit, we need to calm down */
73 static bool zswap_pool_reached_full;
74 
75 /*********************************
76 * tunables
77 **********************************/
78 
79 #define ZSWAP_PARAM_UNSET ""
80 
81 static int zswap_setup(void);
82 
83 /* Enable/disable zswap */
84 static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled);
85 static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
86 static int zswap_enabled_param_set(const char *,
87 				   const struct kernel_param *);
88 static const struct kernel_param_ops zswap_enabled_param_ops = {
89 	.set =		zswap_enabled_param_set,
90 	.get =		param_get_bool,
91 };
92 module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
93 
94 /* Crypto compressor to use */
95 static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
96 static int zswap_compressor_param_set(const char *,
97 				      const struct kernel_param *);
98 static const struct kernel_param_ops zswap_compressor_param_ops = {
99 	.set =		zswap_compressor_param_set,
100 	.get =		param_get_charp,
101 	.free =		param_free_charp,
102 };
103 module_param_cb(compressor, &zswap_compressor_param_ops,
104 		&zswap_compressor, 0644);
105 
106 /* Compressed storage zpool to use */
107 static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
108 static int zswap_zpool_param_set(const char *, const struct kernel_param *);
109 static const struct kernel_param_ops zswap_zpool_param_ops = {
110 	.set =		zswap_zpool_param_set,
111 	.get =		param_get_charp,
112 	.free =		param_free_charp,
113 };
114 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
115 
116 /* The maximum percentage of memory that the compressed pool can occupy */
117 static unsigned int zswap_max_pool_percent = 20;
118 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
119 
120 /* The threshold for accepting new pages after the max_pool_percent was hit */
121 static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
122 module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
123 		   uint, 0644);
124 
125 /* Enable/disable memory pressure-based shrinker. */
126 static bool zswap_shrinker_enabled = IS_ENABLED(
127 		CONFIG_ZSWAP_SHRINKER_DEFAULT_ON);
128 module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644);
129 
zswap_is_enabled(void)130 bool zswap_is_enabled(void)
131 {
132 	return zswap_enabled;
133 }
134 
zswap_never_enabled(void)135 bool zswap_never_enabled(void)
136 {
137 	return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled);
138 }
139 
140 /*********************************
141 * data structures
142 **********************************/
143 
144 struct crypto_acomp_ctx {
145 	struct crypto_acomp *acomp;
146 	struct acomp_req *req;
147 	struct crypto_wait wait;
148 	u8 *buffer;
149 	struct mutex mutex;
150 	bool is_sleepable;
151 };
152 
153 /*
154  * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
155  * The only case where lru_lock is not acquired while holding tree.lock is
156  * when a zswap_entry is taken off the lru for writeback, in that case it
157  * needs to be verified that it's still valid in the tree.
158  */
159 struct zswap_pool {
160 	struct zpool *zpool;
161 	struct crypto_acomp_ctx __percpu *acomp_ctx;
162 	struct percpu_ref ref;
163 	struct list_head list;
164 	struct work_struct release_work;
165 	struct hlist_node node;
166 	char tfm_name[CRYPTO_MAX_ALG_NAME];
167 };
168 
169 /* Global LRU lists shared by all zswap pools. */
170 static struct list_lru zswap_list_lru;
171 
172 /* The lock protects zswap_next_shrink updates. */
173 static DEFINE_SPINLOCK(zswap_shrink_lock);
174 static struct mem_cgroup *zswap_next_shrink;
175 static struct work_struct zswap_shrink_work;
176 static struct shrinker *zswap_shrinker;
177 
178 /*
179  * struct zswap_entry
180  *
181  * This structure contains the metadata for tracking a single compressed
182  * page within zswap.
183  *
184  * swpentry - associated swap entry, the offset indexes into the red-black tree
185  * length - the length in bytes of the compressed page data.  Needed during
186  *          decompression.
187  * referenced - true if the entry recently entered the zswap pool. Unset by the
188  *              writeback logic. The entry is only reclaimed by the writeback
189  *              logic if referenced is unset. See comments in the shrinker
190  *              section for context.
191  * pool - the zswap_pool the entry's data is in
192  * handle - zpool allocation handle that stores the compressed page data
193  * objcg - the obj_cgroup that the compressed memory is charged to
194  * lru - handle to the pool's lru used to evict pages.
195  */
196 struct zswap_entry {
197 	swp_entry_t swpentry;
198 	unsigned int length;
199 	bool referenced;
200 	struct zswap_pool *pool;
201 	unsigned long handle;
202 	struct obj_cgroup *objcg;
203 	struct list_head lru;
204 };
205 
206 static struct xarray *zswap_trees[MAX_SWAPFILES];
207 static unsigned int nr_zswap_trees[MAX_SWAPFILES];
208 
209 /* RCU-protected iteration */
210 static LIST_HEAD(zswap_pools);
211 /* protects zswap_pools list modification */
212 static DEFINE_SPINLOCK(zswap_pools_lock);
213 /* pool counter to provide unique names to zpool */
214 static atomic_t zswap_pools_count = ATOMIC_INIT(0);
215 
216 enum zswap_init_type {
217 	ZSWAP_UNINIT,
218 	ZSWAP_INIT_SUCCEED,
219 	ZSWAP_INIT_FAILED
220 };
221 
222 static enum zswap_init_type zswap_init_state;
223 
224 /* used to ensure the integrity of initialization */
225 static DEFINE_MUTEX(zswap_init_lock);
226 
227 /* init completed, but couldn't create the initial pool */
228 static bool zswap_has_pool;
229 
230 /*********************************
231 * helpers and fwd declarations
232 **********************************/
233 
swap_zswap_tree(swp_entry_t swp)234 static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
235 {
236 	return &zswap_trees[swp_type(swp)][swp_offset(swp)
237 		>> SWAP_ADDRESS_SPACE_SHIFT];
238 }
239 
240 #define zswap_pool_debug(msg, p)				\
241 	pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,		\
242 		 zpool_get_type((p)->zpool))
243 
244 /*********************************
245 * pool functions
246 **********************************/
247 static void __zswap_pool_empty(struct percpu_ref *ref);
248 
zswap_pool_create(char * type,char * compressor)249 static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
250 {
251 	struct zswap_pool *pool;
252 	char name[38]; /* 'zswap' + 32 char (max) num + \0 */
253 	gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
254 	int ret, cpu;
255 
256 	if (!zswap_has_pool) {
257 		/* if either are unset, pool initialization failed, and we
258 		 * need both params to be set correctly before trying to
259 		 * create a pool.
260 		 */
261 		if (!strcmp(type, ZSWAP_PARAM_UNSET))
262 			return NULL;
263 		if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
264 			return NULL;
265 	}
266 
267 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
268 	if (!pool)
269 		return NULL;
270 
271 	/* unique name for each pool specifically required by zsmalloc */
272 	snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
273 	pool->zpool = zpool_create_pool(type, name, gfp);
274 	if (!pool->zpool) {
275 		pr_err("%s zpool not available\n", type);
276 		goto error;
277 	}
278 	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
279 
280 	strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
281 
282 	pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);
283 	if (!pool->acomp_ctx) {
284 		pr_err("percpu alloc failed\n");
285 		goto error;
286 	}
287 
288 	for_each_possible_cpu(cpu)
289 		mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex);
290 
291 	ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
292 				       &pool->node);
293 	if (ret)
294 		goto error;
295 
296 	/* being the current pool takes 1 ref; this func expects the
297 	 * caller to always add the new pool as the current pool
298 	 */
299 	ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,
300 			      PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
301 	if (ret)
302 		goto ref_fail;
303 	INIT_LIST_HEAD(&pool->list);
304 
305 	zswap_pool_debug("created", pool);
306 
307 	return pool;
308 
309 ref_fail:
310 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
311 error:
312 	if (pool->acomp_ctx)
313 		free_percpu(pool->acomp_ctx);
314 	if (pool->zpool)
315 		zpool_destroy_pool(pool->zpool);
316 	kfree(pool);
317 	return NULL;
318 }
319 
__zswap_pool_create_fallback(void)320 static struct zswap_pool *__zswap_pool_create_fallback(void)
321 {
322 	bool has_comp, has_zpool;
323 
324 	has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
325 	if (!has_comp && strcmp(zswap_compressor,
326 				CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
327 		pr_err("compressor %s not available, using default %s\n",
328 		       zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
329 		param_free_charp(&zswap_compressor);
330 		zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
331 		has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
332 	}
333 	if (!has_comp) {
334 		pr_err("default compressor %s not available\n",
335 		       zswap_compressor);
336 		param_free_charp(&zswap_compressor);
337 		zswap_compressor = ZSWAP_PARAM_UNSET;
338 	}
339 
340 	has_zpool = zpool_has_pool(zswap_zpool_type);
341 	if (!has_zpool && strcmp(zswap_zpool_type,
342 				 CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
343 		pr_err("zpool %s not available, using default %s\n",
344 		       zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
345 		param_free_charp(&zswap_zpool_type);
346 		zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
347 		has_zpool = zpool_has_pool(zswap_zpool_type);
348 	}
349 	if (!has_zpool) {
350 		pr_err("default zpool %s not available\n",
351 		       zswap_zpool_type);
352 		param_free_charp(&zswap_zpool_type);
353 		zswap_zpool_type = ZSWAP_PARAM_UNSET;
354 	}
355 
356 	if (!has_comp || !has_zpool)
357 		return NULL;
358 
359 	return zswap_pool_create(zswap_zpool_type, zswap_compressor);
360 }
361 
zswap_pool_destroy(struct zswap_pool * pool)362 static void zswap_pool_destroy(struct zswap_pool *pool)
363 {
364 	zswap_pool_debug("destroying", pool);
365 
366 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
367 	free_percpu(pool->acomp_ctx);
368 
369 	zpool_destroy_pool(pool->zpool);
370 	kfree(pool);
371 }
372 
__zswap_pool_release(struct work_struct * work)373 static void __zswap_pool_release(struct work_struct *work)
374 {
375 	struct zswap_pool *pool = container_of(work, typeof(*pool),
376 						release_work);
377 
378 	synchronize_rcu();
379 
380 	/* nobody should have been able to get a ref... */
381 	WARN_ON(!percpu_ref_is_zero(&pool->ref));
382 	percpu_ref_exit(&pool->ref);
383 
384 	/* pool is now off zswap_pools list and has no references. */
385 	zswap_pool_destroy(pool);
386 }
387 
388 static struct zswap_pool *zswap_pool_current(void);
389 
__zswap_pool_empty(struct percpu_ref * ref)390 static void __zswap_pool_empty(struct percpu_ref *ref)
391 {
392 	struct zswap_pool *pool;
393 
394 	pool = container_of(ref, typeof(*pool), ref);
395 
396 	spin_lock_bh(&zswap_pools_lock);
397 
398 	WARN_ON(pool == zswap_pool_current());
399 
400 	list_del_rcu(&pool->list);
401 
402 	INIT_WORK(&pool->release_work, __zswap_pool_release);
403 	schedule_work(&pool->release_work);
404 
405 	spin_unlock_bh(&zswap_pools_lock);
406 }
407 
zswap_pool_tryget(struct zswap_pool * pool)408 static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
409 {
410 	if (!pool)
411 		return 0;
412 
413 	return percpu_ref_tryget(&pool->ref);
414 }
415 
416 /* The caller must already have a reference. */
zswap_pool_get(struct zswap_pool * pool)417 static void zswap_pool_get(struct zswap_pool *pool)
418 {
419 	percpu_ref_get(&pool->ref);
420 }
421 
zswap_pool_put(struct zswap_pool * pool)422 static void zswap_pool_put(struct zswap_pool *pool)
423 {
424 	percpu_ref_put(&pool->ref);
425 }
426 
__zswap_pool_current(void)427 static struct zswap_pool *__zswap_pool_current(void)
428 {
429 	struct zswap_pool *pool;
430 
431 	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
432 	WARN_ONCE(!pool && zswap_has_pool,
433 		  "%s: no page storage pool!\n", __func__);
434 
435 	return pool;
436 }
437 
zswap_pool_current(void)438 static struct zswap_pool *zswap_pool_current(void)
439 {
440 	assert_spin_locked(&zswap_pools_lock);
441 
442 	return __zswap_pool_current();
443 }
444 
zswap_pool_current_get(void)445 static struct zswap_pool *zswap_pool_current_get(void)
446 {
447 	struct zswap_pool *pool;
448 
449 	rcu_read_lock();
450 
451 	pool = __zswap_pool_current();
452 	if (!zswap_pool_tryget(pool))
453 		pool = NULL;
454 
455 	rcu_read_unlock();
456 
457 	return pool;
458 }
459 
460 /* type and compressor must be null-terminated */
zswap_pool_find_get(char * type,char * compressor)461 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
462 {
463 	struct zswap_pool *pool;
464 
465 	assert_spin_locked(&zswap_pools_lock);
466 
467 	list_for_each_entry_rcu(pool, &zswap_pools, list) {
468 		if (strcmp(pool->tfm_name, compressor))
469 			continue;
470 		if (strcmp(zpool_get_type(pool->zpool), type))
471 			continue;
472 		/* if we can't get it, it's about to be destroyed */
473 		if (!zswap_pool_tryget(pool))
474 			continue;
475 		return pool;
476 	}
477 
478 	return NULL;
479 }
480 
zswap_max_pages(void)481 static unsigned long zswap_max_pages(void)
482 {
483 	return totalram_pages() * zswap_max_pool_percent / 100;
484 }
485 
zswap_accept_thr_pages(void)486 static unsigned long zswap_accept_thr_pages(void)
487 {
488 	return zswap_max_pages() * zswap_accept_thr_percent / 100;
489 }
490 
zswap_total_pages(void)491 unsigned long zswap_total_pages(void)
492 {
493 	struct zswap_pool *pool;
494 	unsigned long total = 0;
495 
496 	rcu_read_lock();
497 	list_for_each_entry_rcu(pool, &zswap_pools, list)
498 		total += zpool_get_total_pages(pool->zpool);
499 	rcu_read_unlock();
500 
501 	return total;
502 }
503 
zswap_check_limits(void)504 static bool zswap_check_limits(void)
505 {
506 	unsigned long cur_pages = zswap_total_pages();
507 	unsigned long max_pages = zswap_max_pages();
508 
509 	if (cur_pages >= max_pages) {
510 		zswap_pool_limit_hit++;
511 		zswap_pool_reached_full = true;
512 	} else if (zswap_pool_reached_full &&
513 		   cur_pages <= zswap_accept_thr_pages()) {
514 			zswap_pool_reached_full = false;
515 	}
516 	return zswap_pool_reached_full;
517 }
518 
519 /*********************************
520 * param callbacks
521 **********************************/
522 
zswap_pool_changed(const char * s,const struct kernel_param * kp)523 static bool zswap_pool_changed(const char *s, const struct kernel_param *kp)
524 {
525 	/* no change required */
526 	if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
527 		return false;
528 	return true;
529 }
530 
531 /* val must be a null-terminated string */
__zswap_param_set(const char * val,const struct kernel_param * kp,char * type,char * compressor)532 static int __zswap_param_set(const char *val, const struct kernel_param *kp,
533 			     char *type, char *compressor)
534 {
535 	struct zswap_pool *pool, *put_pool = NULL;
536 	char *s = strstrip((char *)val);
537 	int ret = 0;
538 	bool new_pool = false;
539 
540 	mutex_lock(&zswap_init_lock);
541 	switch (zswap_init_state) {
542 	case ZSWAP_UNINIT:
543 		/* if this is load-time (pre-init) param setting,
544 		 * don't create a pool; that's done during init.
545 		 */
546 		ret = param_set_charp(s, kp);
547 		break;
548 	case ZSWAP_INIT_SUCCEED:
549 		new_pool = zswap_pool_changed(s, kp);
550 		break;
551 	case ZSWAP_INIT_FAILED:
552 		pr_err("can't set param, initialization failed\n");
553 		ret = -ENODEV;
554 	}
555 	mutex_unlock(&zswap_init_lock);
556 
557 	/* no need to create a new pool, return directly */
558 	if (!new_pool)
559 		return ret;
560 
561 	if (!type) {
562 		if (!zpool_has_pool(s)) {
563 			pr_err("zpool %s not available\n", s);
564 			return -ENOENT;
565 		}
566 		type = s;
567 	} else if (!compressor) {
568 		if (!crypto_has_acomp(s, 0, 0)) {
569 			pr_err("compressor %s not available\n", s);
570 			return -ENOENT;
571 		}
572 		compressor = s;
573 	} else {
574 		WARN_ON(1);
575 		return -EINVAL;
576 	}
577 
578 	spin_lock_bh(&zswap_pools_lock);
579 
580 	pool = zswap_pool_find_get(type, compressor);
581 	if (pool) {
582 		zswap_pool_debug("using existing", pool);
583 		WARN_ON(pool == zswap_pool_current());
584 		list_del_rcu(&pool->list);
585 	}
586 
587 	spin_unlock_bh(&zswap_pools_lock);
588 
589 	if (!pool)
590 		pool = zswap_pool_create(type, compressor);
591 	else {
592 		/*
593 		 * Restore the initial ref dropped by percpu_ref_kill()
594 		 * when the pool was decommissioned and switch it again
595 		 * to percpu mode.
596 		 */
597 		percpu_ref_resurrect(&pool->ref);
598 
599 		/* Drop the ref from zswap_pool_find_get(). */
600 		zswap_pool_put(pool);
601 	}
602 
603 	if (pool)
604 		ret = param_set_charp(s, kp);
605 	else
606 		ret = -EINVAL;
607 
608 	spin_lock_bh(&zswap_pools_lock);
609 
610 	if (!ret) {
611 		put_pool = zswap_pool_current();
612 		list_add_rcu(&pool->list, &zswap_pools);
613 		zswap_has_pool = true;
614 	} else if (pool) {
615 		/* add the possibly pre-existing pool to the end of the pools
616 		 * list; if it's new (and empty) then it'll be removed and
617 		 * destroyed by the put after we drop the lock
618 		 */
619 		list_add_tail_rcu(&pool->list, &zswap_pools);
620 		put_pool = pool;
621 	}
622 
623 	spin_unlock_bh(&zswap_pools_lock);
624 
625 	if (!zswap_has_pool && !pool) {
626 		/* if initial pool creation failed, and this pool creation also
627 		 * failed, maybe both compressor and zpool params were bad.
628 		 * Allow changing this param, so pool creation will succeed
629 		 * when the other param is changed. We already verified this
630 		 * param is ok in the zpool_has_pool() or crypto_has_acomp()
631 		 * checks above.
632 		 */
633 		ret = param_set_charp(s, kp);
634 	}
635 
636 	/* drop the ref from either the old current pool,
637 	 * or the new pool we failed to add
638 	 */
639 	if (put_pool)
640 		percpu_ref_kill(&put_pool->ref);
641 
642 	return ret;
643 }
644 
zswap_compressor_param_set(const char * val,const struct kernel_param * kp)645 static int zswap_compressor_param_set(const char *val,
646 				      const struct kernel_param *kp)
647 {
648 	return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
649 }
650 
zswap_zpool_param_set(const char * val,const struct kernel_param * kp)651 static int zswap_zpool_param_set(const char *val,
652 				 const struct kernel_param *kp)
653 {
654 	return __zswap_param_set(val, kp, NULL, zswap_compressor);
655 }
656 
zswap_enabled_param_set(const char * val,const struct kernel_param * kp)657 static int zswap_enabled_param_set(const char *val,
658 				   const struct kernel_param *kp)
659 {
660 	int ret = -ENODEV;
661 
662 	/* if this is load-time (pre-init) param setting, only set param. */
663 	if (system_state != SYSTEM_RUNNING)
664 		return param_set_bool(val, kp);
665 
666 	mutex_lock(&zswap_init_lock);
667 	switch (zswap_init_state) {
668 	case ZSWAP_UNINIT:
669 		if (zswap_setup())
670 			break;
671 		fallthrough;
672 	case ZSWAP_INIT_SUCCEED:
673 		if (!zswap_has_pool)
674 			pr_err("can't enable, no pool configured\n");
675 		else
676 			ret = param_set_bool(val, kp);
677 		break;
678 	case ZSWAP_INIT_FAILED:
679 		pr_err("can't enable, initialization failed\n");
680 	}
681 	mutex_unlock(&zswap_init_lock);
682 
683 	return ret;
684 }
685 
686 /*********************************
687 * lru functions
688 **********************************/
689 
690 /* should be called under RCU */
691 #ifdef CONFIG_MEMCG
mem_cgroup_from_entry(struct zswap_entry * entry)692 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
693 {
694 	return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
695 }
696 #else
mem_cgroup_from_entry(struct zswap_entry * entry)697 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
698 {
699 	return NULL;
700 }
701 #endif
702 
entry_to_nid(struct zswap_entry * entry)703 static inline int entry_to_nid(struct zswap_entry *entry)
704 {
705 	return page_to_nid(virt_to_page(entry));
706 }
707 
zswap_lru_add(struct list_lru * list_lru,struct zswap_entry * entry)708 static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry)
709 {
710 	int nid = entry_to_nid(entry);
711 	struct mem_cgroup *memcg;
712 
713 	/*
714 	 * Note that it is safe to use rcu_read_lock() here, even in the face of
715 	 * concurrent memcg offlining:
716 	 *
717 	 * 1. list_lru_add() is called before list_lru_one is dead. The
718 	 *    new entry will be reparented to memcg's parent's list_lru.
719 	 * 2. list_lru_add() is called after list_lru_one is dead. The
720 	 *    new entry will be added directly to memcg's parent's list_lru.
721 	 *
722 	 * Similar reasoning holds for list_lru_del().
723 	 */
724 	rcu_read_lock();
725 	memcg = mem_cgroup_from_entry(entry);
726 	/* will always succeed */
727 	list_lru_add(list_lru, &entry->lru, nid, memcg);
728 	rcu_read_unlock();
729 }
730 
zswap_lru_del(struct list_lru * list_lru,struct zswap_entry * entry)731 static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry)
732 {
733 	int nid = entry_to_nid(entry);
734 	struct mem_cgroup *memcg;
735 
736 	rcu_read_lock();
737 	memcg = mem_cgroup_from_entry(entry);
738 	/* will always succeed */
739 	list_lru_del(list_lru, &entry->lru, nid, memcg);
740 	rcu_read_unlock();
741 }
742 
zswap_lruvec_state_init(struct lruvec * lruvec)743 void zswap_lruvec_state_init(struct lruvec *lruvec)
744 {
745 	atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
746 }
747 
zswap_folio_swapin(struct folio * folio)748 void zswap_folio_swapin(struct folio *folio)
749 {
750 	struct lruvec *lruvec;
751 
752 	if (folio) {
753 		lruvec = folio_lruvec(folio);
754 		atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);
755 	}
756 }
757 
758 /*
759  * This function should be called when a memcg is being offlined.
760  *
761  * Since the global shrinker shrink_worker() may hold a reference
762  * of the memcg, we must check and release the reference in
763  * zswap_next_shrink.
764  *
765  * shrink_worker() must handle the case where this function releases
766  * the reference of memcg being shrunk.
767  */
zswap_memcg_offline_cleanup(struct mem_cgroup * memcg)768 void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
769 {
770 	/* lock out zswap shrinker walking memcg tree */
771 	spin_lock(&zswap_shrink_lock);
772 	if (zswap_next_shrink == memcg) {
773 		do {
774 			zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
775 		} while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
776 	}
777 	spin_unlock(&zswap_shrink_lock);
778 }
779 
780 /*********************************
781 * zswap entry functions
782 **********************************/
783 static struct kmem_cache *zswap_entry_cache;
784 
zswap_entry_cache_alloc(gfp_t gfp,int nid)785 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid)
786 {
787 	struct zswap_entry *entry;
788 	entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid);
789 	if (!entry)
790 		return NULL;
791 	return entry;
792 }
793 
zswap_entry_cache_free(struct zswap_entry * entry)794 static void zswap_entry_cache_free(struct zswap_entry *entry)
795 {
796 	kmem_cache_free(zswap_entry_cache, entry);
797 }
798 
799 /*
800  * Carries out the common pattern of freeing and entry's zpool allocation,
801  * freeing the entry itself, and decrementing the number of stored pages.
802  */
zswap_entry_free(struct zswap_entry * entry)803 static void zswap_entry_free(struct zswap_entry *entry)
804 {
805 	zswap_lru_del(&zswap_list_lru, entry);
806 	zpool_free(entry->pool->zpool, entry->handle);
807 	zswap_pool_put(entry->pool);
808 	if (entry->objcg) {
809 		obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
810 		obj_cgroup_put(entry->objcg);
811 	}
812 	zswap_entry_cache_free(entry);
813 	atomic_long_dec(&zswap_stored_pages);
814 }
815 
816 /*********************************
817 * compressed storage functions
818 **********************************/
zswap_cpu_comp_prepare(unsigned int cpu,struct hlist_node * node)819 static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
820 {
821 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
822 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
823 	struct crypto_acomp *acomp = NULL;
824 	struct acomp_req *req = NULL;
825 	u8 *buffer = NULL;
826 	int ret;
827 
828 	buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
829 	if (!buffer) {
830 		ret = -ENOMEM;
831 		goto fail;
832 	}
833 
834 	acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
835 	if (IS_ERR(acomp)) {
836 		pr_err("could not alloc crypto acomp %s : %ld\n",
837 				pool->tfm_name, PTR_ERR(acomp));
838 		ret = PTR_ERR(acomp);
839 		goto fail;
840 	}
841 
842 	req = acomp_request_alloc(acomp);
843 	if (!req) {
844 		pr_err("could not alloc crypto acomp_request %s\n",
845 		       pool->tfm_name);
846 		ret = -ENOMEM;
847 		goto fail;
848 	}
849 
850 	/*
851 	 * Only hold the mutex after completing allocations, otherwise we may
852 	 * recurse into zswap through reclaim and attempt to hold the mutex
853 	 * again resulting in a deadlock.
854 	 */
855 	mutex_lock(&acomp_ctx->mutex);
856 	crypto_init_wait(&acomp_ctx->wait);
857 
858 	/*
859 	 * if the backend of acomp is async zip, crypto_req_done() will wakeup
860 	 * crypto_wait_req(); if the backend of acomp is scomp, the callback
861 	 * won't be called, crypto_wait_req() will return without blocking.
862 	 */
863 	acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
864 				   crypto_req_done, &acomp_ctx->wait);
865 
866 	acomp_ctx->buffer = buffer;
867 	acomp_ctx->acomp = acomp;
868 	acomp_ctx->is_sleepable = acomp_is_async(acomp);
869 	acomp_ctx->req = req;
870 	mutex_unlock(&acomp_ctx->mutex);
871 	return 0;
872 
873 fail:
874 	if (acomp)
875 		crypto_free_acomp(acomp);
876 	kfree(buffer);
877 	return ret;
878 }
879 
zswap_cpu_comp_dead(unsigned int cpu,struct hlist_node * node)880 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
881 {
882 	struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
883 	struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
884 
885 	mutex_lock(&acomp_ctx->mutex);
886 	if (!IS_ERR_OR_NULL(acomp_ctx)) {
887 		if (!IS_ERR_OR_NULL(acomp_ctx->req))
888 			acomp_request_free(acomp_ctx->req);
889 		acomp_ctx->req = NULL;
890 		if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
891 			crypto_free_acomp(acomp_ctx->acomp);
892 		kfree(acomp_ctx->buffer);
893 	}
894 	mutex_unlock(&acomp_ctx->mutex);
895 
896 	return 0;
897 }
898 
acomp_ctx_get_cpu_lock(struct zswap_pool * pool)899 static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool)
900 {
901 	struct crypto_acomp_ctx *acomp_ctx;
902 
903 	for (;;) {
904 		acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
905 		mutex_lock(&acomp_ctx->mutex);
906 		if (likely(acomp_ctx->req))
907 			return acomp_ctx;
908 		/*
909 		 * It is possible that we were migrated to a different CPU after
910 		 * getting the per-CPU ctx but before the mutex was acquired. If
911 		 * the old CPU got offlined, zswap_cpu_comp_dead() could have
912 		 * already freed ctx->req (among other things) and set it to
913 		 * NULL. Just try again on the new CPU that we ended up on.
914 		 */
915 		mutex_unlock(&acomp_ctx->mutex);
916 	}
917 }
918 
acomp_ctx_put_unlock(struct crypto_acomp_ctx * acomp_ctx)919 static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx)
920 {
921 	mutex_unlock(&acomp_ctx->mutex);
922 }
923 
zswap_compress(struct page * page,struct zswap_entry * entry,struct zswap_pool * pool)924 static bool zswap_compress(struct page *page, struct zswap_entry *entry,
925 			   struct zswap_pool *pool)
926 {
927 	struct crypto_acomp_ctx *acomp_ctx;
928 	struct scatterlist input, output;
929 	int comp_ret = 0, alloc_ret = 0;
930 	unsigned int dlen = PAGE_SIZE;
931 	unsigned long handle;
932 	struct zpool *zpool;
933 	char *buf;
934 	gfp_t gfp;
935 	u8 *dst;
936 
937 	acomp_ctx = acomp_ctx_get_cpu_lock(pool);
938 	dst = acomp_ctx->buffer;
939 	sg_init_table(&input, 1);
940 	sg_set_page(&input, page, PAGE_SIZE, 0);
941 
942 	/*
943 	 * We need PAGE_SIZE * 2 here since there maybe over-compression case,
944 	 * and hardware-accelerators may won't check the dst buffer size, so
945 	 * giving the dst buffer with enough length to avoid buffer overflow.
946 	 */
947 	sg_init_one(&output, dst, PAGE_SIZE * 2);
948 	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
949 
950 	/*
951 	 * it maybe looks a little bit silly that we send an asynchronous request,
952 	 * then wait for its completion synchronously. This makes the process look
953 	 * synchronous in fact.
954 	 * Theoretically, acomp supports users send multiple acomp requests in one
955 	 * acomp instance, then get those requests done simultaneously. but in this
956 	 * case, zswap actually does store and load page by page, there is no
957 	 * existing method to send the second page before the first page is done
958 	 * in one thread doing zwap.
959 	 * but in different threads running on different cpu, we have different
960 	 * acomp instance, so multiple threads can do (de)compression in parallel.
961 	 */
962 	comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
963 	dlen = acomp_ctx->req->dlen;
964 	if (comp_ret)
965 		goto unlock;
966 
967 	zpool = pool->zpool;
968 	gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
969 	if (zpool_malloc_support_movable(zpool))
970 		gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
971 	alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle);
972 	if (alloc_ret)
973 		goto unlock;
974 
975 	buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
976 	memcpy(buf, dst, dlen);
977 	zpool_unmap_handle(zpool, handle);
978 
979 	entry->handle = handle;
980 	entry->length = dlen;
981 
982 unlock:
983 	if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)
984 		zswap_reject_compress_poor++;
985 	else if (comp_ret)
986 		zswap_reject_compress_fail++;
987 	else if (alloc_ret)
988 		zswap_reject_alloc_fail++;
989 
990 	acomp_ctx_put_unlock(acomp_ctx);
991 	return comp_ret == 0 && alloc_ret == 0;
992 }
993 
zswap_decompress(struct zswap_entry * entry,struct folio * folio)994 static void zswap_decompress(struct zswap_entry *entry, struct folio *folio)
995 {
996 	struct zpool *zpool = entry->pool->zpool;
997 	struct scatterlist input, output;
998 	struct crypto_acomp_ctx *acomp_ctx;
999 	u8 *src;
1000 
1001 	acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
1002 	src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
1003 	/*
1004 	 * If zpool_map_handle is atomic, we cannot reliably utilize its mapped buffer
1005 	 * to do crypto_acomp_decompress() which might sleep. In such cases, we must
1006 	 * resort to copying the buffer to a temporary one.
1007 	 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer,
1008 	 * such as a kmap address of high memory or even ever a vmap address.
1009 	 * However, sg_init_one is only equipped to handle linearly mapped low memory.
1010 	 * In such cases, we also must copy the buffer to a temporary and lowmem one.
1011 	 */
1012 	if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) ||
1013 	    !virt_addr_valid(src)) {
1014 		memcpy(acomp_ctx->buffer, src, entry->length);
1015 		src = acomp_ctx->buffer;
1016 		zpool_unmap_handle(zpool, entry->handle);
1017 	}
1018 
1019 	sg_init_one(&input, src, entry->length);
1020 	sg_init_table(&output, 1);
1021 	sg_set_folio(&output, folio, PAGE_SIZE, 0);
1022 	acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
1023 	BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
1024 	BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
1025 
1026 	if (src != acomp_ctx->buffer)
1027 		zpool_unmap_handle(zpool, entry->handle);
1028 	acomp_ctx_put_unlock(acomp_ctx);
1029 }
1030 
1031 /*********************************
1032 * writeback code
1033 **********************************/
1034 /*
1035  * Attempts to free an entry by adding a folio to the swap cache,
1036  * decompressing the entry data into the folio, and issuing a
1037  * bio write to write the folio back to the swap device.
1038  *
1039  * This can be thought of as a "resumed writeback" of the folio
1040  * to the swap device.  We are basically resuming the same swap
1041  * writeback path that was intercepted with the zswap_store()
1042  * in the first place.  After the folio has been decompressed into
1043  * the swap cache, the compressed version stored by zswap can be
1044  * freed.
1045  */
zswap_writeback_entry(struct zswap_entry * entry,swp_entry_t swpentry)1046 static int zswap_writeback_entry(struct zswap_entry *entry,
1047 				 swp_entry_t swpentry)
1048 {
1049 	struct xarray *tree;
1050 	pgoff_t offset = swp_offset(swpentry);
1051 	struct folio *folio;
1052 	struct mempolicy *mpol;
1053 	bool folio_was_allocated;
1054 	struct writeback_control wbc = {
1055 		.sync_mode = WB_SYNC_NONE,
1056 	};
1057 
1058 	/* try to allocate swap cache folio */
1059 	mpol = get_task_policy(current);
1060 	folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol,
1061 				NO_INTERLEAVE_INDEX, &folio_was_allocated, true);
1062 	if (!folio)
1063 		return -ENOMEM;
1064 
1065 	/*
1066 	 * Found an existing folio, we raced with swapin or concurrent
1067 	 * shrinker. We generally writeback cold folios from zswap, and
1068 	 * swapin means the folio just became hot, so skip this folio.
1069 	 * For unlikely concurrent shrinker case, it will be unlinked
1070 	 * and freed when invalidated by the concurrent shrinker anyway.
1071 	 */
1072 	if (!folio_was_allocated) {
1073 		folio_put(folio);
1074 		return -EEXIST;
1075 	}
1076 
1077 	/*
1078 	 * folio is locked, and the swapcache is now secured against
1079 	 * concurrent swapping to and from the slot, and concurrent
1080 	 * swapoff so we can safely dereference the zswap tree here.
1081 	 * Verify that the swap entry hasn't been invalidated and recycled
1082 	 * behind our backs, to avoid overwriting a new swap folio with
1083 	 * old compressed data. Only when this is successful can the entry
1084 	 * be dereferenced.
1085 	 */
1086 	tree = swap_zswap_tree(swpentry);
1087 	if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) {
1088 		delete_from_swap_cache(folio);
1089 		folio_unlock(folio);
1090 		folio_put(folio);
1091 		return -ENOMEM;
1092 	}
1093 
1094 	zswap_decompress(entry, folio);
1095 
1096 	count_vm_event(ZSWPWB);
1097 	if (entry->objcg)
1098 		count_objcg_events(entry->objcg, ZSWPWB, 1);
1099 
1100 	zswap_entry_free(entry);
1101 
1102 	/* folio is up to date */
1103 	folio_mark_uptodate(folio);
1104 
1105 	/* move it to the tail of the inactive list after end_writeback */
1106 	folio_set_reclaim(folio);
1107 
1108 	/* start writeback */
1109 	__swap_writepage(folio, &wbc);
1110 	folio_put(folio);
1111 
1112 	return 0;
1113 }
1114 
1115 /*********************************
1116 * shrinker functions
1117 **********************************/
1118 /*
1119  * The dynamic shrinker is modulated by the following factors:
1120  *
1121  * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving
1122  *    the entry a second chance) before rotating it in the LRU list. If the
1123  *    entry is considered again by the shrinker, with its referenced bit unset,
1124  *    it is written back. The writeback rate as a result is dynamically
1125  *    adjusted by the pool activities - if the pool is dominated by new entries
1126  *    (i.e lots of recent zswapouts), these entries will be protected and
1127  *    the writeback rate will slow down. On the other hand, if the pool has a
1128  *    lot of stagnant entries, these entries will be reclaimed immediately,
1129  *    effectively increasing the writeback rate.
1130  *
1131  * 2. Swapins counter: If we observe swapins, it is a sign that we are
1132  *    overshrinking and should slow down. We maintain a swapins counter, which
1133  *    is consumed and subtract from the number of eligible objects on the LRU
1134  *    in zswap_shrinker_count().
1135  *
1136  * 3. Compression ratio. The better the workload compresses, the less gains we
1137  *    can expect from writeback. We scale down the number of objects available
1138  *    for reclaim by this ratio.
1139  */
shrink_memcg_cb(struct list_head * item,struct list_lru_one * l,void * arg)1140 static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l,
1141 				       void *arg)
1142 {
1143 	struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
1144 	bool *encountered_page_in_swapcache = (bool *)arg;
1145 	swp_entry_t swpentry;
1146 	enum lru_status ret = LRU_REMOVED_RETRY;
1147 	int writeback_result;
1148 
1149 	/*
1150 	 * Second chance algorithm: if the entry has its referenced bit set, give it
1151 	 * a second chance. Only clear the referenced bit and rotate it in the
1152 	 * zswap's LRU list.
1153 	 */
1154 	if (entry->referenced) {
1155 		entry->referenced = false;
1156 		return LRU_ROTATE;
1157 	}
1158 
1159 	/*
1160 	 * As soon as we drop the LRU lock, the entry can be freed by
1161 	 * a concurrent invalidation. This means the following:
1162 	 *
1163 	 * 1. We extract the swp_entry_t to the stack, allowing
1164 	 *    zswap_writeback_entry() to pin the swap entry and
1165 	 *    then validate the zwap entry against that swap entry's
1166 	 *    tree using pointer value comparison. Only when that
1167 	 *    is successful can the entry be dereferenced.
1168 	 *
1169 	 * 2. Usually, objects are taken off the LRU for reclaim. In
1170 	 *    this case this isn't possible, because if reclaim fails
1171 	 *    for whatever reason, we have no means of knowing if the
1172 	 *    entry is alive to put it back on the LRU.
1173 	 *
1174 	 *    So rotate it before dropping the lock. If the entry is
1175 	 *    written back or invalidated, the free path will unlink
1176 	 *    it. For failures, rotation is the right thing as well.
1177 	 *
1178 	 *    Temporary failures, where the same entry should be tried
1179 	 *    again immediately, almost never happen for this shrinker.
1180 	 *    We don't do any trylocking; -ENOMEM comes closest,
1181 	 *    but that's extremely rare and doesn't happen spuriously
1182 	 *    either. Don't bother distinguishing this case.
1183 	 */
1184 	list_move_tail(item, &l->list);
1185 
1186 	/*
1187 	 * Once the lru lock is dropped, the entry might get freed. The
1188 	 * swpentry is copied to the stack, and entry isn't deref'd again
1189 	 * until the entry is verified to still be alive in the tree.
1190 	 */
1191 	swpentry = entry->swpentry;
1192 
1193 	/*
1194 	 * It's safe to drop the lock here because we return either
1195 	 * LRU_REMOVED_RETRY, LRU_RETRY or LRU_STOP.
1196 	 */
1197 	spin_unlock(&l->lock);
1198 
1199 	writeback_result = zswap_writeback_entry(entry, swpentry);
1200 
1201 	if (writeback_result) {
1202 		zswap_reject_reclaim_fail++;
1203 		ret = LRU_RETRY;
1204 
1205 		/*
1206 		 * Encountering a page already in swap cache is a sign that we are shrinking
1207 		 * into the warmer region. We should terminate shrinking (if we're in the dynamic
1208 		 * shrinker context).
1209 		 */
1210 		if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
1211 			ret = LRU_STOP;
1212 			*encountered_page_in_swapcache = true;
1213 		}
1214 	} else {
1215 		zswap_written_back_pages++;
1216 	}
1217 
1218 	return ret;
1219 }
1220 
zswap_shrinker_scan(struct shrinker * shrinker,struct shrink_control * sc)1221 static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
1222 		struct shrink_control *sc)
1223 {
1224 	unsigned long shrink_ret;
1225 	bool encountered_page_in_swapcache = false;
1226 
1227 	if (!zswap_shrinker_enabled ||
1228 			!mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
1229 		sc->nr_scanned = 0;
1230 		return SHRINK_STOP;
1231 	}
1232 
1233 	shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
1234 		&encountered_page_in_swapcache);
1235 
1236 	if (encountered_page_in_swapcache)
1237 		return SHRINK_STOP;
1238 
1239 	return shrink_ret ? shrink_ret : SHRINK_STOP;
1240 }
1241 
zswap_shrinker_count(struct shrinker * shrinker,struct shrink_control * sc)1242 static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
1243 		struct shrink_control *sc)
1244 {
1245 	struct mem_cgroup *memcg = sc->memcg;
1246 	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
1247 	atomic_long_t *nr_disk_swapins =
1248 		&lruvec->zswap_lruvec_state.nr_disk_swapins;
1249 	unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur,
1250 		nr_remain;
1251 
1252 	if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg))
1253 		return 0;
1254 
1255 	/*
1256 	 * The shrinker resumes swap writeback, which will enter block
1257 	 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS
1258 	 * rules (may_enter_fs()), which apply on a per-folio basis.
1259 	 */
1260 	if (!gfp_has_io_fs(sc->gfp_mask))
1261 		return 0;
1262 
1263 	/*
1264 	 * For memcg, use the cgroup-wide ZSWAP stats since we don't
1265 	 * have them per-node and thus per-lruvec. Careful if memcg is
1266 	 * runtime-disabled: we can get sc->memcg == NULL, which is ok
1267 	 * for the lruvec, but not for memcg_page_state().
1268 	 *
1269 	 * Without memcg, use the zswap pool-wide metrics.
1270 	 */
1271 	if (!mem_cgroup_disabled()) {
1272 		mem_cgroup_flush_stats(memcg);
1273 		nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
1274 		nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
1275 	} else {
1276 		nr_backing = zswap_total_pages();
1277 		nr_stored = atomic_long_read(&zswap_stored_pages);
1278 	}
1279 
1280 	if (!nr_stored)
1281 		return 0;
1282 
1283 	nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc);
1284 	if (!nr_freeable)
1285 		return 0;
1286 
1287 	/*
1288 	 * Subtract from the lru size the number of pages that are recently swapped
1289 	 * in from disk. The idea is that had we protect the zswap's LRU by this
1290 	 * amount of pages, these disk swapins would not have happened.
1291 	 */
1292 	nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins);
1293 	do {
1294 		if (nr_freeable >= nr_disk_swapins_cur)
1295 			nr_remain = 0;
1296 		else
1297 			nr_remain = nr_disk_swapins_cur - nr_freeable;
1298 	} while (!atomic_long_try_cmpxchg(
1299 		nr_disk_swapins, &nr_disk_swapins_cur, nr_remain));
1300 
1301 	nr_freeable -= nr_disk_swapins_cur - nr_remain;
1302 	if (!nr_freeable)
1303 		return 0;
1304 
1305 	/*
1306 	 * Scale the number of freeable pages by the memory saving factor.
1307 	 * This ensures that the better zswap compresses memory, the fewer
1308 	 * pages we will evict to swap (as it will otherwise incur IO for
1309 	 * relatively small memory saving).
1310 	 */
1311 	return mult_frac(nr_freeable, nr_backing, nr_stored);
1312 }
1313 
zswap_alloc_shrinker(void)1314 static struct shrinker *zswap_alloc_shrinker(void)
1315 {
1316 	struct shrinker *shrinker;
1317 
1318 	shrinker =
1319 		shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");
1320 	if (!shrinker)
1321 		return NULL;
1322 
1323 	shrinker->scan_objects = zswap_shrinker_scan;
1324 	shrinker->count_objects = zswap_shrinker_count;
1325 	shrinker->batch = 0;
1326 	shrinker->seeks = DEFAULT_SEEKS;
1327 	return shrinker;
1328 }
1329 
shrink_memcg(struct mem_cgroup * memcg)1330 static int shrink_memcg(struct mem_cgroup *memcg)
1331 {
1332 	int nid, shrunk = 0, scanned = 0;
1333 
1334 	if (!mem_cgroup_zswap_writeback_enabled(memcg))
1335 		return -ENOENT;
1336 
1337 	/*
1338 	 * Skip zombies because their LRUs are reparented and we would be
1339 	 * reclaiming from the parent instead of the dead memcg.
1340 	 */
1341 	if (memcg && !mem_cgroup_online(memcg))
1342 		return -ENOENT;
1343 
1344 	for_each_node_state(nid, N_NORMAL_MEMORY) {
1345 		unsigned long nr_to_walk = 1;
1346 
1347 		shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
1348 					    &shrink_memcg_cb, NULL, &nr_to_walk);
1349 		scanned += 1 - nr_to_walk;
1350 	}
1351 
1352 	if (!scanned)
1353 		return -ENOENT;
1354 
1355 	return shrunk ? 0 : -EAGAIN;
1356 }
1357 
shrink_worker(struct work_struct * w)1358 static void shrink_worker(struct work_struct *w)
1359 {
1360 	struct mem_cgroup *memcg;
1361 	int ret, failures = 0, attempts = 0;
1362 	unsigned long thr;
1363 
1364 	/* Reclaim down to the accept threshold */
1365 	thr = zswap_accept_thr_pages();
1366 
1367 	/*
1368 	 * Global reclaim will select cgroup in a round-robin fashion from all
1369 	 * online memcgs, but memcgs that have no pages in zswap and
1370 	 * writeback-disabled memcgs (memory.zswap.writeback=0) are not
1371 	 * candidates for shrinking.
1372 	 *
1373 	 * Shrinking will be aborted if we encounter the following
1374 	 * MAX_RECLAIM_RETRIES times:
1375 	 * - No writeback-candidate memcgs found in a memcg tree walk.
1376 	 * - Shrinking a writeback-candidate memcg failed.
1377 	 *
1378 	 * We save iteration cursor memcg into zswap_next_shrink,
1379 	 * which can be modified by the offline memcg cleaner
1380 	 * zswap_memcg_offline_cleanup().
1381 	 *
1382 	 * Since the offline cleaner is called only once, we cannot leave an
1383 	 * offline memcg reference in zswap_next_shrink.
1384 	 * We can rely on the cleaner only if we get online memcg under lock.
1385 	 *
1386 	 * If we get an offline memcg, we cannot determine if the cleaner has
1387 	 * already been called or will be called later. We must put back the
1388 	 * reference before returning from this function. Otherwise, the
1389 	 * offline memcg left in zswap_next_shrink will hold the reference
1390 	 * until the next run of shrink_worker().
1391 	 */
1392 	do {
1393 		/*
1394 		 * Start shrinking from the next memcg after zswap_next_shrink.
1395 		 * When the offline cleaner has already advanced the cursor,
1396 		 * advancing the cursor here overlooks one memcg, but this
1397 		 * should be negligibly rare.
1398 		 *
1399 		 * If we get an online memcg, keep the extra reference in case
1400 		 * the original one obtained by mem_cgroup_iter() is dropped by
1401 		 * zswap_memcg_offline_cleanup() while we are shrinking the
1402 		 * memcg.
1403 		 */
1404 		spin_lock(&zswap_shrink_lock);
1405 		do {
1406 			memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
1407 			zswap_next_shrink = memcg;
1408 		} while (memcg && !mem_cgroup_tryget_online(memcg));
1409 		spin_unlock(&zswap_shrink_lock);
1410 
1411 		if (!memcg) {
1412 			/*
1413 			 * Continue shrinking without incrementing failures if
1414 			 * we found candidate memcgs in the last tree walk.
1415 			 */
1416 			if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
1417 				break;
1418 
1419 			attempts = 0;
1420 			goto resched;
1421 		}
1422 
1423 		ret = shrink_memcg(memcg);
1424 		/* drop the extra reference */
1425 		mem_cgroup_put(memcg);
1426 
1427 		/*
1428 		 * There are no writeback-candidate pages in the memcg.
1429 		 * This is not an issue as long as we can find another memcg
1430 		 * with pages in zswap. Skip this without incrementing attempts
1431 		 * and failures.
1432 		 */
1433 		if (ret == -ENOENT)
1434 			continue;
1435 		++attempts;
1436 
1437 		if (ret && ++failures == MAX_RECLAIM_RETRIES)
1438 			break;
1439 resched:
1440 		cond_resched();
1441 	} while (zswap_total_pages() > thr);
1442 }
1443 
1444 /*********************************
1445 * main API
1446 **********************************/
1447 
zswap_store_page(struct page * page,struct obj_cgroup * objcg,struct zswap_pool * pool)1448 static bool zswap_store_page(struct page *page,
1449 			     struct obj_cgroup *objcg,
1450 			     struct zswap_pool *pool)
1451 {
1452 	swp_entry_t page_swpentry = page_swap_entry(page);
1453 	struct zswap_entry *entry, *old;
1454 
1455 	/* allocate entry */
1456 	entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
1457 	if (!entry) {
1458 		zswap_reject_kmemcache_fail++;
1459 		return false;
1460 	}
1461 
1462 	if (!zswap_compress(page, entry, pool))
1463 		goto compress_failed;
1464 
1465 	old = xa_store(swap_zswap_tree(page_swpentry),
1466 		       swp_offset(page_swpentry),
1467 		       entry, GFP_KERNEL);
1468 	if (xa_is_err(old)) {
1469 		int err = xa_err(old);
1470 
1471 		WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
1472 		zswap_reject_alloc_fail++;
1473 		goto store_failed;
1474 	}
1475 
1476 	/*
1477 	 * We may have had an existing entry that became stale when
1478 	 * the folio was redirtied and now the new version is being
1479 	 * swapped out. Get rid of the old.
1480 	 */
1481 	if (old)
1482 		zswap_entry_free(old);
1483 
1484 	/*
1485 	 * The entry is successfully compressed and stored in the tree, there is
1486 	 * no further possibility of failure. Grab refs to the pool and objcg,
1487 	 * charge zswap memory, and increment zswap_stored_pages.
1488 	 * The opposite actions will be performed by zswap_entry_free()
1489 	 * when the entry is removed from the tree.
1490 	 */
1491 	zswap_pool_get(pool);
1492 	if (objcg) {
1493 		obj_cgroup_get(objcg);
1494 		obj_cgroup_charge_zswap(objcg, entry->length);
1495 	}
1496 	atomic_long_inc(&zswap_stored_pages);
1497 
1498 	/*
1499 	 * We finish initializing the entry while it's already in xarray.
1500 	 * This is safe because:
1501 	 *
1502 	 * 1. Concurrent stores and invalidations are excluded by folio lock.
1503 	 *
1504 	 * 2. Writeback is excluded by the entry not being on the LRU yet.
1505 	 *    The publishing order matters to prevent writeback from seeing
1506 	 *    an incoherent entry.
1507 	 */
1508 	entry->pool = pool;
1509 	entry->swpentry = page_swpentry;
1510 	entry->objcg = objcg;
1511 	entry->referenced = true;
1512 	if (entry->length) {
1513 		INIT_LIST_HEAD(&entry->lru);
1514 		zswap_lru_add(&zswap_list_lru, entry);
1515 	}
1516 
1517 	return true;
1518 
1519 store_failed:
1520 	zpool_free(pool->zpool, entry->handle);
1521 compress_failed:
1522 	zswap_entry_cache_free(entry);
1523 	return false;
1524 }
1525 
zswap_store(struct folio * folio)1526 bool zswap_store(struct folio *folio)
1527 {
1528 	long nr_pages = folio_nr_pages(folio);
1529 	swp_entry_t swp = folio->swap;
1530 	struct obj_cgroup *objcg = NULL;
1531 	struct mem_cgroup *memcg = NULL;
1532 	struct zswap_pool *pool;
1533 	bool ret = false;
1534 	long index;
1535 
1536 	VM_WARN_ON_ONCE(!folio_test_locked(folio));
1537 	VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
1538 
1539 	if (!zswap_enabled)
1540 		goto check_old;
1541 
1542 	objcg = get_obj_cgroup_from_folio(folio);
1543 	if (objcg && !obj_cgroup_may_zswap(objcg)) {
1544 		memcg = get_mem_cgroup_from_objcg(objcg);
1545 		if (shrink_memcg(memcg)) {
1546 			mem_cgroup_put(memcg);
1547 			goto put_objcg;
1548 		}
1549 		mem_cgroup_put(memcg);
1550 	}
1551 
1552 	if (zswap_check_limits())
1553 		goto put_objcg;
1554 
1555 	pool = zswap_pool_current_get();
1556 	if (!pool)
1557 		goto put_objcg;
1558 
1559 	if (objcg) {
1560 		memcg = get_mem_cgroup_from_objcg(objcg);
1561 		if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) {
1562 			mem_cgroup_put(memcg);
1563 			goto put_pool;
1564 		}
1565 		mem_cgroup_put(memcg);
1566 	}
1567 
1568 	for (index = 0; index < nr_pages; ++index) {
1569 		struct page *page = folio_page(folio, index);
1570 
1571 		if (!zswap_store_page(page, objcg, pool))
1572 			goto put_pool;
1573 	}
1574 
1575 	if (objcg)
1576 		count_objcg_events(objcg, ZSWPOUT, nr_pages);
1577 
1578 	count_vm_events(ZSWPOUT, nr_pages);
1579 
1580 	ret = true;
1581 
1582 put_pool:
1583 	zswap_pool_put(pool);
1584 put_objcg:
1585 	obj_cgroup_put(objcg);
1586 	if (!ret && zswap_pool_reached_full)
1587 		queue_work(shrink_wq, &zswap_shrink_work);
1588 check_old:
1589 	/*
1590 	 * If the zswap store fails or zswap is disabled, we must invalidate
1591 	 * the possibly stale entries which were previously stored at the
1592 	 * offsets corresponding to each page of the folio. Otherwise,
1593 	 * writeback could overwrite the new data in the swapfile.
1594 	 */
1595 	if (!ret) {
1596 		unsigned type = swp_type(swp);
1597 		pgoff_t offset = swp_offset(swp);
1598 		struct zswap_entry *entry;
1599 		struct xarray *tree;
1600 
1601 		for (index = 0; index < nr_pages; ++index) {
1602 			tree = swap_zswap_tree(swp_entry(type, offset + index));
1603 			entry = xa_erase(tree, offset + index);
1604 			if (entry)
1605 				zswap_entry_free(entry);
1606 		}
1607 	}
1608 
1609 	return ret;
1610 }
1611 
zswap_load(struct folio * folio)1612 bool zswap_load(struct folio *folio)
1613 {
1614 	swp_entry_t swp = folio->swap;
1615 	pgoff_t offset = swp_offset(swp);
1616 	bool swapcache = folio_test_swapcache(folio);
1617 	struct xarray *tree = swap_zswap_tree(swp);
1618 	struct zswap_entry *entry;
1619 
1620 	VM_WARN_ON_ONCE(!folio_test_locked(folio));
1621 
1622 	if (zswap_never_enabled())
1623 		return false;
1624 
1625 	/*
1626 	 * Large folios should not be swapped in while zswap is being used, as
1627 	 * they are not properly handled. Zswap does not properly load large
1628 	 * folios, and a large folio may only be partially in zswap.
1629 	 *
1630 	 * Return true without marking the folio uptodate so that an IO error is
1631 	 * emitted (e.g. do_swap_page() will sigbus).
1632 	 */
1633 	if (WARN_ON_ONCE(folio_test_large(folio)))
1634 		return true;
1635 
1636 	/*
1637 	 * When reading into the swapcache, invalidate our entry. The
1638 	 * swapcache can be the authoritative owner of the page and
1639 	 * its mappings, and the pressure that results from having two
1640 	 * in-memory copies outweighs any benefits of caching the
1641 	 * compression work.
1642 	 *
1643 	 * (Most swapins go through the swapcache. The notable
1644 	 * exception is the singleton fault on SWP_SYNCHRONOUS_IO
1645 	 * files, which reads into a private page and may free it if
1646 	 * the fault fails. We remain the primary owner of the entry.)
1647 	 */
1648 	if (swapcache)
1649 		entry = xa_erase(tree, offset);
1650 	else
1651 		entry = xa_load(tree, offset);
1652 
1653 	if (!entry)
1654 		return false;
1655 
1656 	zswap_decompress(entry, folio);
1657 
1658 	count_vm_event(ZSWPIN);
1659 	if (entry->objcg)
1660 		count_objcg_events(entry->objcg, ZSWPIN, 1);
1661 
1662 	if (swapcache) {
1663 		zswap_entry_free(entry);
1664 		folio_mark_dirty(folio);
1665 	}
1666 
1667 	folio_mark_uptodate(folio);
1668 	return true;
1669 }
1670 
zswap_invalidate(swp_entry_t swp)1671 void zswap_invalidate(swp_entry_t swp)
1672 {
1673 	pgoff_t offset = swp_offset(swp);
1674 	struct xarray *tree = swap_zswap_tree(swp);
1675 	struct zswap_entry *entry;
1676 
1677 	if (xa_empty(tree))
1678 		return;
1679 
1680 	entry = xa_erase(tree, offset);
1681 	if (entry)
1682 		zswap_entry_free(entry);
1683 }
1684 
zswap_swapon(int type,unsigned long nr_pages)1685 int zswap_swapon(int type, unsigned long nr_pages)
1686 {
1687 	struct xarray *trees, *tree;
1688 	unsigned int nr, i;
1689 
1690 	nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
1691 	trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL);
1692 	if (!trees) {
1693 		pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1694 		return -ENOMEM;
1695 	}
1696 
1697 	for (i = 0; i < nr; i++)
1698 		xa_init(trees + i);
1699 
1700 	nr_zswap_trees[type] = nr;
1701 	zswap_trees[type] = trees;
1702 	return 0;
1703 }
1704 
zswap_swapoff(int type)1705 void zswap_swapoff(int type)
1706 {
1707 	struct xarray *trees = zswap_trees[type];
1708 	unsigned int i;
1709 
1710 	if (!trees)
1711 		return;
1712 
1713 	/* try_to_unuse() invalidated all the entries already */
1714 	for (i = 0; i < nr_zswap_trees[type]; i++)
1715 		WARN_ON_ONCE(!xa_empty(trees + i));
1716 
1717 	kvfree(trees);
1718 	nr_zswap_trees[type] = 0;
1719 	zswap_trees[type] = NULL;
1720 }
1721 
1722 /*********************************
1723 * debugfs functions
1724 **********************************/
1725 #ifdef CONFIG_DEBUG_FS
1726 #include <linux/debugfs.h>
1727 
1728 static struct dentry *zswap_debugfs_root;
1729 
debugfs_get_total_size(void * data,u64 * val)1730 static int debugfs_get_total_size(void *data, u64 *val)
1731 {
1732 	*val = zswap_total_pages() * PAGE_SIZE;
1733 	return 0;
1734 }
1735 DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n");
1736 
debugfs_get_stored_pages(void * data,u64 * val)1737 static int debugfs_get_stored_pages(void *data, u64 *val)
1738 {
1739 	*val = atomic_long_read(&zswap_stored_pages);
1740 	return 0;
1741 }
1742 DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n");
1743 
zswap_debugfs_init(void)1744 static int zswap_debugfs_init(void)
1745 {
1746 	if (!debugfs_initialized())
1747 		return -ENODEV;
1748 
1749 	zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1750 
1751 	debugfs_create_u64("pool_limit_hit", 0444,
1752 			   zswap_debugfs_root, &zswap_pool_limit_hit);
1753 	debugfs_create_u64("reject_reclaim_fail", 0444,
1754 			   zswap_debugfs_root, &zswap_reject_reclaim_fail);
1755 	debugfs_create_u64("reject_alloc_fail", 0444,
1756 			   zswap_debugfs_root, &zswap_reject_alloc_fail);
1757 	debugfs_create_u64("reject_kmemcache_fail", 0444,
1758 			   zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1759 	debugfs_create_u64("reject_compress_fail", 0444,
1760 			   zswap_debugfs_root, &zswap_reject_compress_fail);
1761 	debugfs_create_u64("reject_compress_poor", 0444,
1762 			   zswap_debugfs_root, &zswap_reject_compress_poor);
1763 	debugfs_create_u64("written_back_pages", 0444,
1764 			   zswap_debugfs_root, &zswap_written_back_pages);
1765 	debugfs_create_file("pool_total_size", 0444,
1766 			    zswap_debugfs_root, NULL, &total_size_fops);
1767 	debugfs_create_file("stored_pages", 0444,
1768 			    zswap_debugfs_root, NULL, &stored_pages_fops);
1769 
1770 	return 0;
1771 }
1772 #else
zswap_debugfs_init(void)1773 static int zswap_debugfs_init(void)
1774 {
1775 	return 0;
1776 }
1777 #endif
1778 
1779 /*********************************
1780 * module init and exit
1781 **********************************/
zswap_setup(void)1782 static int zswap_setup(void)
1783 {
1784 	struct zswap_pool *pool;
1785 	int ret;
1786 
1787 	zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
1788 	if (!zswap_entry_cache) {
1789 		pr_err("entry cache creation failed\n");
1790 		goto cache_fail;
1791 	}
1792 
1793 	ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1794 				      "mm/zswap_pool:prepare",
1795 				      zswap_cpu_comp_prepare,
1796 				      zswap_cpu_comp_dead);
1797 	if (ret)
1798 		goto hp_fail;
1799 
1800 	shrink_wq = alloc_workqueue("zswap-shrink",
1801 			WQ_UNBOUND|WQ_MEM_RECLAIM, 1);
1802 	if (!shrink_wq)
1803 		goto shrink_wq_fail;
1804 
1805 	zswap_shrinker = zswap_alloc_shrinker();
1806 	if (!zswap_shrinker)
1807 		goto shrinker_fail;
1808 	if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker))
1809 		goto lru_fail;
1810 	shrinker_register(zswap_shrinker);
1811 
1812 	INIT_WORK(&zswap_shrink_work, shrink_worker);
1813 
1814 	pool = __zswap_pool_create_fallback();
1815 	if (pool) {
1816 		pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1817 			zpool_get_type(pool->zpool));
1818 		list_add(&pool->list, &zswap_pools);
1819 		zswap_has_pool = true;
1820 		static_branch_enable(&zswap_ever_enabled);
1821 	} else {
1822 		pr_err("pool creation failed\n");
1823 		zswap_enabled = false;
1824 	}
1825 
1826 	if (zswap_debugfs_init())
1827 		pr_warn("debugfs initialization failed\n");
1828 	zswap_init_state = ZSWAP_INIT_SUCCEED;
1829 	return 0;
1830 
1831 lru_fail:
1832 	shrinker_free(zswap_shrinker);
1833 shrinker_fail:
1834 	destroy_workqueue(shrink_wq);
1835 shrink_wq_fail:
1836 	cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE);
1837 hp_fail:
1838 	kmem_cache_destroy(zswap_entry_cache);
1839 cache_fail:
1840 	/* if built-in, we aren't unloaded on failure; don't allow use */
1841 	zswap_init_state = ZSWAP_INIT_FAILED;
1842 	zswap_enabled = false;
1843 	return -ENOMEM;
1844 }
1845 
zswap_init(void)1846 static int __init zswap_init(void)
1847 {
1848 	if (!zswap_enabled)
1849 		return 0;
1850 	return zswap_setup();
1851 }
1852 /* must be late so crypto has time to come up */
1853 late_initcall(zswap_init);
1854 
1855 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1856 MODULE_DESCRIPTION("Compressed cache for swap pages");
1857