1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * zswap.c - zswap driver file 4 * 5 * zswap is a cache that takes pages that are in the process 6 * of being swapped out and attempts to compress and store them in a 7 * RAM-based memory pool. This can result in a significant I/O reduction on 8 * the swap device and, in the case where decompressing from RAM is faster 9 * than reading from the swap device, can also improve workload performance. 10 * 11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 15 16 #include <linux/module.h> 17 #include <linux/cpu.h> 18 #include <linux/highmem.h> 19 #include <linux/slab.h> 20 #include <linux/spinlock.h> 21 #include <linux/types.h> 22 #include <linux/atomic.h> 23 #include <linux/swap.h> 24 #include <linux/crypto.h> 25 #include <linux/scatterlist.h> 26 #include <linux/mempolicy.h> 27 #include <linux/mempool.h> 28 #include <linux/zpool.h> 29 #include <crypto/acompress.h> 30 #include <linux/zswap.h> 31 #include <linux/mm_types.h> 32 #include <linux/page-flags.h> 33 #include <linux/swapops.h> 34 #include <linux/writeback.h> 35 #include <linux/pagemap.h> 36 #include <linux/workqueue.h> 37 #include <linux/list_lru.h> 38 39 #include "swap.h" 40 #include "internal.h" 41 42 /********************************* 43 * statistics 44 **********************************/ 45 /* The number of pages currently stored in zswap */ 46 atomic_long_t zswap_stored_pages = ATOMIC_LONG_INIT(0); 47 /* The number of incompressible pages currently stored in zswap */ 48 static atomic_long_t zswap_stored_incompressible_pages = ATOMIC_LONG_INIT(0); 49 50 /* 51 * The statistics below are not protected from concurrent access for 52 * performance reasons so they may not be a 100% accurate. However, 53 * they do provide useful information on roughly how many times a 54 * certain event is occurring. 55 */ 56 57 /* Pool limit was hit (see zswap_max_pool_percent) */ 58 static u64 zswap_pool_limit_hit; 59 /* Pages written back when pool limit was reached */ 60 static u64 zswap_written_back_pages; 61 /* Store failed due to a reclaim failure after pool limit was reached */ 62 static u64 zswap_reject_reclaim_fail; 63 /* Store failed due to compression algorithm failure */ 64 static u64 zswap_reject_compress_fail; 65 /* Compressed page was too big for the allocator to (optimally) store */ 66 static u64 zswap_reject_compress_poor; 67 /* Load or writeback failed due to decompression failure */ 68 static u64 zswap_decompress_fail; 69 /* Store failed because underlying allocator could not get memory */ 70 static u64 zswap_reject_alloc_fail; 71 /* Store failed because the entry metadata could not be allocated (rare) */ 72 static u64 zswap_reject_kmemcache_fail; 73 74 /* Shrinker work queue */ 75 static struct workqueue_struct *shrink_wq; 76 /* Pool limit was hit, we need to calm down */ 77 static bool zswap_pool_reached_full; 78 79 /********************************* 80 * tunables 81 **********************************/ 82 83 #define ZSWAP_PARAM_UNSET "" 84 85 static int zswap_setup(void); 86 87 /* Enable/disable zswap */ 88 static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled); 89 static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); 90 static int zswap_enabled_param_set(const char *, 91 const struct kernel_param *); 92 static const struct kernel_param_ops zswap_enabled_param_ops = { 93 .set = zswap_enabled_param_set, 94 .get = param_get_bool, 95 }; 96 module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); 97 98 /* Crypto compressor to use */ 99 static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 100 static int zswap_compressor_param_set(const char *, 101 const struct kernel_param *); 102 static const struct kernel_param_ops zswap_compressor_param_ops = { 103 .set = zswap_compressor_param_set, 104 .get = param_get_charp, 105 .free = param_free_charp, 106 }; 107 module_param_cb(compressor, &zswap_compressor_param_ops, 108 &zswap_compressor, 0644); 109 110 /* Compressed storage zpool to use */ 111 static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 112 static int zswap_zpool_param_set(const char *, const struct kernel_param *); 113 static const struct kernel_param_ops zswap_zpool_param_ops = { 114 .set = zswap_zpool_param_set, 115 .get = param_get_charp, 116 .free = param_free_charp, 117 }; 118 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 119 120 /* The maximum percentage of memory that the compressed pool can occupy */ 121 static unsigned int zswap_max_pool_percent = 20; 122 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 123 124 /* The threshold for accepting new pages after the max_pool_percent was hit */ 125 static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ 126 module_param_named(accept_threshold_percent, zswap_accept_thr_percent, 127 uint, 0644); 128 129 /* Enable/disable memory pressure-based shrinker. */ 130 static bool zswap_shrinker_enabled = IS_ENABLED( 131 CONFIG_ZSWAP_SHRINKER_DEFAULT_ON); 132 module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644); 133 134 bool zswap_is_enabled(void) 135 { 136 return zswap_enabled; 137 } 138 139 bool zswap_never_enabled(void) 140 { 141 return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled); 142 } 143 144 /********************************* 145 * data structures 146 **********************************/ 147 148 struct crypto_acomp_ctx { 149 struct crypto_acomp *acomp; 150 struct acomp_req *req; 151 struct crypto_wait wait; 152 u8 *buffer; 153 struct mutex mutex; 154 bool is_sleepable; 155 }; 156 157 /* 158 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock. 159 * The only case where lru_lock is not acquired while holding tree.lock is 160 * when a zswap_entry is taken off the lru for writeback, in that case it 161 * needs to be verified that it's still valid in the tree. 162 */ 163 struct zswap_pool { 164 struct zpool *zpool; 165 struct crypto_acomp_ctx __percpu *acomp_ctx; 166 struct percpu_ref ref; 167 struct list_head list; 168 struct work_struct release_work; 169 struct hlist_node node; 170 char tfm_name[CRYPTO_MAX_ALG_NAME]; 171 }; 172 173 /* Global LRU lists shared by all zswap pools. */ 174 static struct list_lru zswap_list_lru; 175 176 /* The lock protects zswap_next_shrink updates. */ 177 static DEFINE_SPINLOCK(zswap_shrink_lock); 178 static struct mem_cgroup *zswap_next_shrink; 179 static struct work_struct zswap_shrink_work; 180 static struct shrinker *zswap_shrinker; 181 182 /* 183 * struct zswap_entry 184 * 185 * This structure contains the metadata for tracking a single compressed 186 * page within zswap. 187 * 188 * swpentry - associated swap entry, the offset indexes into the red-black tree 189 * length - the length in bytes of the compressed page data. Needed during 190 * decompression. 191 * referenced - true if the entry recently entered the zswap pool. Unset by the 192 * writeback logic. The entry is only reclaimed by the writeback 193 * logic if referenced is unset. See comments in the shrinker 194 * section for context. 195 * pool - the zswap_pool the entry's data is in 196 * handle - zpool allocation handle that stores the compressed page data 197 * objcg - the obj_cgroup that the compressed memory is charged to 198 * lru - handle to the pool's lru used to evict pages. 199 */ 200 struct zswap_entry { 201 swp_entry_t swpentry; 202 unsigned int length; 203 bool referenced; 204 struct zswap_pool *pool; 205 unsigned long handle; 206 struct obj_cgroup *objcg; 207 struct list_head lru; 208 }; 209 210 static struct xarray *zswap_trees[MAX_SWAPFILES]; 211 static unsigned int nr_zswap_trees[MAX_SWAPFILES]; 212 213 /* RCU-protected iteration */ 214 static LIST_HEAD(zswap_pools); 215 /* protects zswap_pools list modification */ 216 static DEFINE_SPINLOCK(zswap_pools_lock); 217 /* pool counter to provide unique names to zpool */ 218 static atomic_t zswap_pools_count = ATOMIC_INIT(0); 219 220 enum zswap_init_type { 221 ZSWAP_UNINIT, 222 ZSWAP_INIT_SUCCEED, 223 ZSWAP_INIT_FAILED 224 }; 225 226 static enum zswap_init_type zswap_init_state; 227 228 /* used to ensure the integrity of initialization */ 229 static DEFINE_MUTEX(zswap_init_lock); 230 231 /* init completed, but couldn't create the initial pool */ 232 static bool zswap_has_pool; 233 234 /********************************* 235 * helpers and fwd declarations 236 **********************************/ 237 238 static inline struct xarray *swap_zswap_tree(swp_entry_t swp) 239 { 240 return &zswap_trees[swp_type(swp)][swp_offset(swp) 241 >> SWAP_ADDRESS_SPACE_SHIFT]; 242 } 243 244 #define zswap_pool_debug(msg, p) \ 245 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 246 zpool_get_type((p)->zpool)) 247 248 /********************************* 249 * pool functions 250 **********************************/ 251 static void __zswap_pool_empty(struct percpu_ref *ref); 252 253 static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 254 { 255 struct zswap_pool *pool; 256 char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 257 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 258 int ret, cpu; 259 260 if (!zswap_has_pool) { 261 /* if either are unset, pool initialization failed, and we 262 * need both params to be set correctly before trying to 263 * create a pool. 264 */ 265 if (!strcmp(type, ZSWAP_PARAM_UNSET)) 266 return NULL; 267 if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 268 return NULL; 269 } 270 271 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 272 if (!pool) 273 return NULL; 274 275 /* unique name for each pool specifically required by zsmalloc */ 276 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count)); 277 pool->zpool = zpool_create_pool(type, name, gfp); 278 if (!pool->zpool) { 279 pr_err("%s zpool not available\n", type); 280 goto error; 281 } 282 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); 283 284 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 285 286 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx); 287 if (!pool->acomp_ctx) { 288 pr_err("percpu alloc failed\n"); 289 goto error; 290 } 291 292 for_each_possible_cpu(cpu) 293 mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex); 294 295 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, 296 &pool->node); 297 if (ret) 298 goto error; 299 300 /* being the current pool takes 1 ref; this func expects the 301 * caller to always add the new pool as the current pool 302 */ 303 ret = percpu_ref_init(&pool->ref, __zswap_pool_empty, 304 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); 305 if (ret) 306 goto ref_fail; 307 INIT_LIST_HEAD(&pool->list); 308 309 zswap_pool_debug("created", pool); 310 311 return pool; 312 313 ref_fail: 314 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 315 error: 316 if (pool->acomp_ctx) 317 free_percpu(pool->acomp_ctx); 318 if (pool->zpool) 319 zpool_destroy_pool(pool->zpool); 320 kfree(pool); 321 return NULL; 322 } 323 324 static struct zswap_pool *__zswap_pool_create_fallback(void) 325 { 326 bool has_comp, has_zpool; 327 328 has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 329 if (!has_comp && strcmp(zswap_compressor, 330 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 331 pr_err("compressor %s not available, using default %s\n", 332 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); 333 param_free_charp(&zswap_compressor); 334 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 335 has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 336 } 337 if (!has_comp) { 338 pr_err("default compressor %s not available\n", 339 zswap_compressor); 340 param_free_charp(&zswap_compressor); 341 zswap_compressor = ZSWAP_PARAM_UNSET; 342 } 343 344 has_zpool = zpool_has_pool(zswap_zpool_type); 345 if (!has_zpool && strcmp(zswap_zpool_type, 346 CONFIG_ZSWAP_ZPOOL_DEFAULT)) { 347 pr_err("zpool %s not available, using default %s\n", 348 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); 349 param_free_charp(&zswap_zpool_type); 350 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 351 has_zpool = zpool_has_pool(zswap_zpool_type); 352 } 353 if (!has_zpool) { 354 pr_err("default zpool %s not available\n", 355 zswap_zpool_type); 356 param_free_charp(&zswap_zpool_type); 357 zswap_zpool_type = ZSWAP_PARAM_UNSET; 358 } 359 360 if (!has_comp || !has_zpool) 361 return NULL; 362 363 return zswap_pool_create(zswap_zpool_type, zswap_compressor); 364 } 365 366 static void zswap_pool_destroy(struct zswap_pool *pool) 367 { 368 zswap_pool_debug("destroying", pool); 369 370 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 371 free_percpu(pool->acomp_ctx); 372 373 zpool_destroy_pool(pool->zpool); 374 kfree(pool); 375 } 376 377 static void __zswap_pool_release(struct work_struct *work) 378 { 379 struct zswap_pool *pool = container_of(work, typeof(*pool), 380 release_work); 381 382 synchronize_rcu(); 383 384 /* nobody should have been able to get a ref... */ 385 WARN_ON(!percpu_ref_is_zero(&pool->ref)); 386 percpu_ref_exit(&pool->ref); 387 388 /* pool is now off zswap_pools list and has no references. */ 389 zswap_pool_destroy(pool); 390 } 391 392 static struct zswap_pool *zswap_pool_current(void); 393 394 static void __zswap_pool_empty(struct percpu_ref *ref) 395 { 396 struct zswap_pool *pool; 397 398 pool = container_of(ref, typeof(*pool), ref); 399 400 spin_lock_bh(&zswap_pools_lock); 401 402 WARN_ON(pool == zswap_pool_current()); 403 404 list_del_rcu(&pool->list); 405 406 INIT_WORK(&pool->release_work, __zswap_pool_release); 407 schedule_work(&pool->release_work); 408 409 spin_unlock_bh(&zswap_pools_lock); 410 } 411 412 static int __must_check zswap_pool_tryget(struct zswap_pool *pool) 413 { 414 if (!pool) 415 return 0; 416 417 return percpu_ref_tryget(&pool->ref); 418 } 419 420 /* The caller must already have a reference. */ 421 static void zswap_pool_get(struct zswap_pool *pool) 422 { 423 percpu_ref_get(&pool->ref); 424 } 425 426 static void zswap_pool_put(struct zswap_pool *pool) 427 { 428 percpu_ref_put(&pool->ref); 429 } 430 431 static struct zswap_pool *__zswap_pool_current(void) 432 { 433 struct zswap_pool *pool; 434 435 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 436 WARN_ONCE(!pool && zswap_has_pool, 437 "%s: no page storage pool!\n", __func__); 438 439 return pool; 440 } 441 442 static struct zswap_pool *zswap_pool_current(void) 443 { 444 assert_spin_locked(&zswap_pools_lock); 445 446 return __zswap_pool_current(); 447 } 448 449 static struct zswap_pool *zswap_pool_current_get(void) 450 { 451 struct zswap_pool *pool; 452 453 rcu_read_lock(); 454 455 pool = __zswap_pool_current(); 456 if (!zswap_pool_tryget(pool)) 457 pool = NULL; 458 459 rcu_read_unlock(); 460 461 return pool; 462 } 463 464 /* type and compressor must be null-terminated */ 465 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 466 { 467 struct zswap_pool *pool; 468 469 assert_spin_locked(&zswap_pools_lock); 470 471 list_for_each_entry_rcu(pool, &zswap_pools, list) { 472 if (strcmp(pool->tfm_name, compressor)) 473 continue; 474 if (strcmp(zpool_get_type(pool->zpool), type)) 475 continue; 476 /* if we can't get it, it's about to be destroyed */ 477 if (!zswap_pool_tryget(pool)) 478 continue; 479 return pool; 480 } 481 482 return NULL; 483 } 484 485 static unsigned long zswap_max_pages(void) 486 { 487 return totalram_pages() * zswap_max_pool_percent / 100; 488 } 489 490 static unsigned long zswap_accept_thr_pages(void) 491 { 492 return zswap_max_pages() * zswap_accept_thr_percent / 100; 493 } 494 495 unsigned long zswap_total_pages(void) 496 { 497 struct zswap_pool *pool; 498 unsigned long total = 0; 499 500 rcu_read_lock(); 501 list_for_each_entry_rcu(pool, &zswap_pools, list) 502 total += zpool_get_total_pages(pool->zpool); 503 rcu_read_unlock(); 504 505 return total; 506 } 507 508 static bool zswap_check_limits(void) 509 { 510 unsigned long cur_pages = zswap_total_pages(); 511 unsigned long max_pages = zswap_max_pages(); 512 513 if (cur_pages >= max_pages) { 514 zswap_pool_limit_hit++; 515 zswap_pool_reached_full = true; 516 } else if (zswap_pool_reached_full && 517 cur_pages <= zswap_accept_thr_pages()) { 518 zswap_pool_reached_full = false; 519 } 520 return zswap_pool_reached_full; 521 } 522 523 /********************************* 524 * param callbacks 525 **********************************/ 526 527 static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) 528 { 529 /* no change required */ 530 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 531 return false; 532 return true; 533 } 534 535 /* val must be a null-terminated string */ 536 static int __zswap_param_set(const char *val, const struct kernel_param *kp, 537 char *type, char *compressor) 538 { 539 struct zswap_pool *pool, *put_pool = NULL; 540 char *s = strstrip((char *)val); 541 int ret = 0; 542 bool new_pool = false; 543 544 mutex_lock(&zswap_init_lock); 545 switch (zswap_init_state) { 546 case ZSWAP_UNINIT: 547 /* if this is load-time (pre-init) param setting, 548 * don't create a pool; that's done during init. 549 */ 550 ret = param_set_charp(s, kp); 551 break; 552 case ZSWAP_INIT_SUCCEED: 553 new_pool = zswap_pool_changed(s, kp); 554 break; 555 case ZSWAP_INIT_FAILED: 556 pr_err("can't set param, initialization failed\n"); 557 ret = -ENODEV; 558 } 559 mutex_unlock(&zswap_init_lock); 560 561 /* no need to create a new pool, return directly */ 562 if (!new_pool) 563 return ret; 564 565 if (!type) { 566 if (!zpool_has_pool(s)) { 567 pr_err("zpool %s not available\n", s); 568 return -ENOENT; 569 } 570 type = s; 571 } else if (!compressor) { 572 if (!crypto_has_acomp(s, 0, 0)) { 573 pr_err("compressor %s not available\n", s); 574 return -ENOENT; 575 } 576 compressor = s; 577 } else { 578 WARN_ON(1); 579 return -EINVAL; 580 } 581 582 spin_lock_bh(&zswap_pools_lock); 583 584 pool = zswap_pool_find_get(type, compressor); 585 if (pool) { 586 zswap_pool_debug("using existing", pool); 587 WARN_ON(pool == zswap_pool_current()); 588 list_del_rcu(&pool->list); 589 } 590 591 spin_unlock_bh(&zswap_pools_lock); 592 593 if (!pool) 594 pool = zswap_pool_create(type, compressor); 595 else { 596 /* 597 * Restore the initial ref dropped by percpu_ref_kill() 598 * when the pool was decommissioned and switch it again 599 * to percpu mode. 600 */ 601 percpu_ref_resurrect(&pool->ref); 602 603 /* Drop the ref from zswap_pool_find_get(). */ 604 zswap_pool_put(pool); 605 } 606 607 if (pool) 608 ret = param_set_charp(s, kp); 609 else 610 ret = -EINVAL; 611 612 spin_lock_bh(&zswap_pools_lock); 613 614 if (!ret) { 615 put_pool = zswap_pool_current(); 616 list_add_rcu(&pool->list, &zswap_pools); 617 zswap_has_pool = true; 618 } else if (pool) { 619 /* add the possibly pre-existing pool to the end of the pools 620 * list; if it's new (and empty) then it'll be removed and 621 * destroyed by the put after we drop the lock 622 */ 623 list_add_tail_rcu(&pool->list, &zswap_pools); 624 put_pool = pool; 625 } 626 627 spin_unlock_bh(&zswap_pools_lock); 628 629 if (!zswap_has_pool && !pool) { 630 /* if initial pool creation failed, and this pool creation also 631 * failed, maybe both compressor and zpool params were bad. 632 * Allow changing this param, so pool creation will succeed 633 * when the other param is changed. We already verified this 634 * param is ok in the zpool_has_pool() or crypto_has_acomp() 635 * checks above. 636 */ 637 ret = param_set_charp(s, kp); 638 } 639 640 /* drop the ref from either the old current pool, 641 * or the new pool we failed to add 642 */ 643 if (put_pool) 644 percpu_ref_kill(&put_pool->ref); 645 646 return ret; 647 } 648 649 static int zswap_compressor_param_set(const char *val, 650 const struct kernel_param *kp) 651 { 652 return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 653 } 654 655 static int zswap_zpool_param_set(const char *val, 656 const struct kernel_param *kp) 657 { 658 return __zswap_param_set(val, kp, NULL, zswap_compressor); 659 } 660 661 static int zswap_enabled_param_set(const char *val, 662 const struct kernel_param *kp) 663 { 664 int ret = -ENODEV; 665 666 /* if this is load-time (pre-init) param setting, only set param. */ 667 if (system_state != SYSTEM_RUNNING) 668 return param_set_bool(val, kp); 669 670 mutex_lock(&zswap_init_lock); 671 switch (zswap_init_state) { 672 case ZSWAP_UNINIT: 673 if (zswap_setup()) 674 break; 675 fallthrough; 676 case ZSWAP_INIT_SUCCEED: 677 if (!zswap_has_pool) 678 pr_err("can't enable, no pool configured\n"); 679 else 680 ret = param_set_bool(val, kp); 681 break; 682 case ZSWAP_INIT_FAILED: 683 pr_err("can't enable, initialization failed\n"); 684 } 685 mutex_unlock(&zswap_init_lock); 686 687 return ret; 688 } 689 690 /********************************* 691 * lru functions 692 **********************************/ 693 694 /* should be called under RCU */ 695 #ifdef CONFIG_MEMCG 696 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) 697 { 698 return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL; 699 } 700 #else 701 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) 702 { 703 return NULL; 704 } 705 #endif 706 707 static inline int entry_to_nid(struct zswap_entry *entry) 708 { 709 return page_to_nid(virt_to_page(entry)); 710 } 711 712 static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry) 713 { 714 int nid = entry_to_nid(entry); 715 struct mem_cgroup *memcg; 716 717 /* 718 * Note that it is safe to use rcu_read_lock() here, even in the face of 719 * concurrent memcg offlining: 720 * 721 * 1. list_lru_add() is called before list_lru_one is dead. The 722 * new entry will be reparented to memcg's parent's list_lru. 723 * 2. list_lru_add() is called after list_lru_one is dead. The 724 * new entry will be added directly to memcg's parent's list_lru. 725 * 726 * Similar reasoning holds for list_lru_del(). 727 */ 728 rcu_read_lock(); 729 memcg = mem_cgroup_from_entry(entry); 730 /* will always succeed */ 731 list_lru_add(list_lru, &entry->lru, nid, memcg); 732 rcu_read_unlock(); 733 } 734 735 static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry) 736 { 737 int nid = entry_to_nid(entry); 738 struct mem_cgroup *memcg; 739 740 rcu_read_lock(); 741 memcg = mem_cgroup_from_entry(entry); 742 /* will always succeed */ 743 list_lru_del(list_lru, &entry->lru, nid, memcg); 744 rcu_read_unlock(); 745 } 746 747 void zswap_lruvec_state_init(struct lruvec *lruvec) 748 { 749 atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0); 750 } 751 752 void zswap_folio_swapin(struct folio *folio) 753 { 754 struct lruvec *lruvec; 755 756 if (folio) { 757 lruvec = folio_lruvec(folio); 758 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins); 759 } 760 } 761 762 /* 763 * This function should be called when a memcg is being offlined. 764 * 765 * Since the global shrinker shrink_worker() may hold a reference 766 * of the memcg, we must check and release the reference in 767 * zswap_next_shrink. 768 * 769 * shrink_worker() must handle the case where this function releases 770 * the reference of memcg being shrunk. 771 */ 772 void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) 773 { 774 /* lock out zswap shrinker walking memcg tree */ 775 spin_lock(&zswap_shrink_lock); 776 if (zswap_next_shrink == memcg) { 777 do { 778 zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL); 779 } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink)); 780 } 781 spin_unlock(&zswap_shrink_lock); 782 } 783 784 /********************************* 785 * zswap entry functions 786 **********************************/ 787 static struct kmem_cache *zswap_entry_cache; 788 789 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid) 790 { 791 struct zswap_entry *entry; 792 entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid); 793 if (!entry) 794 return NULL; 795 return entry; 796 } 797 798 static void zswap_entry_cache_free(struct zswap_entry *entry) 799 { 800 kmem_cache_free(zswap_entry_cache, entry); 801 } 802 803 /* 804 * Carries out the common pattern of freeing and entry's zpool allocation, 805 * freeing the entry itself, and decrementing the number of stored pages. 806 */ 807 static void zswap_entry_free(struct zswap_entry *entry) 808 { 809 zswap_lru_del(&zswap_list_lru, entry); 810 zpool_free(entry->pool->zpool, entry->handle); 811 zswap_pool_put(entry->pool); 812 if (entry->objcg) { 813 obj_cgroup_uncharge_zswap(entry->objcg, entry->length); 814 obj_cgroup_put(entry->objcg); 815 } 816 if (entry->length == PAGE_SIZE) 817 atomic_long_dec(&zswap_stored_incompressible_pages); 818 zswap_entry_cache_free(entry); 819 atomic_long_dec(&zswap_stored_pages); 820 } 821 822 /********************************* 823 * compressed storage functions 824 **********************************/ 825 static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 826 { 827 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 828 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 829 struct crypto_acomp *acomp = NULL; 830 struct acomp_req *req = NULL; 831 u8 *buffer = NULL; 832 int ret; 833 834 buffer = kmalloc_node(PAGE_SIZE, GFP_KERNEL, cpu_to_node(cpu)); 835 if (!buffer) { 836 ret = -ENOMEM; 837 goto fail; 838 } 839 840 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu)); 841 if (IS_ERR(acomp)) { 842 pr_err("could not alloc crypto acomp %s : %ld\n", 843 pool->tfm_name, PTR_ERR(acomp)); 844 ret = PTR_ERR(acomp); 845 goto fail; 846 } 847 848 req = acomp_request_alloc(acomp); 849 if (!req) { 850 pr_err("could not alloc crypto acomp_request %s\n", 851 pool->tfm_name); 852 ret = -ENOMEM; 853 goto fail; 854 } 855 856 /* 857 * Only hold the mutex after completing allocations, otherwise we may 858 * recurse into zswap through reclaim and attempt to hold the mutex 859 * again resulting in a deadlock. 860 */ 861 mutex_lock(&acomp_ctx->mutex); 862 crypto_init_wait(&acomp_ctx->wait); 863 864 /* 865 * if the backend of acomp is async zip, crypto_req_done() will wakeup 866 * crypto_wait_req(); if the backend of acomp is scomp, the callback 867 * won't be called, crypto_wait_req() will return without blocking. 868 */ 869 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 870 crypto_req_done, &acomp_ctx->wait); 871 872 acomp_ctx->buffer = buffer; 873 acomp_ctx->acomp = acomp; 874 acomp_ctx->is_sleepable = acomp_is_async(acomp); 875 acomp_ctx->req = req; 876 mutex_unlock(&acomp_ctx->mutex); 877 return 0; 878 879 fail: 880 if (acomp) 881 crypto_free_acomp(acomp); 882 kfree(buffer); 883 return ret; 884 } 885 886 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 887 { 888 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 889 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 890 struct acomp_req *req; 891 struct crypto_acomp *acomp; 892 u8 *buffer; 893 894 if (IS_ERR_OR_NULL(acomp_ctx)) 895 return 0; 896 897 mutex_lock(&acomp_ctx->mutex); 898 req = acomp_ctx->req; 899 acomp = acomp_ctx->acomp; 900 buffer = acomp_ctx->buffer; 901 acomp_ctx->req = NULL; 902 acomp_ctx->acomp = NULL; 903 acomp_ctx->buffer = NULL; 904 mutex_unlock(&acomp_ctx->mutex); 905 906 /* 907 * Do the actual freeing after releasing the mutex to avoid subtle 908 * locking dependencies causing deadlocks. 909 */ 910 if (!IS_ERR_OR_NULL(req)) 911 acomp_request_free(req); 912 if (!IS_ERR_OR_NULL(acomp)) 913 crypto_free_acomp(acomp); 914 kfree(buffer); 915 916 return 0; 917 } 918 919 static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool) 920 { 921 struct crypto_acomp_ctx *acomp_ctx; 922 923 for (;;) { 924 acomp_ctx = raw_cpu_ptr(pool->acomp_ctx); 925 mutex_lock(&acomp_ctx->mutex); 926 if (likely(acomp_ctx->req)) 927 return acomp_ctx; 928 /* 929 * It is possible that we were migrated to a different CPU after 930 * getting the per-CPU ctx but before the mutex was acquired. If 931 * the old CPU got offlined, zswap_cpu_comp_dead() could have 932 * already freed ctx->req (among other things) and set it to 933 * NULL. Just try again on the new CPU that we ended up on. 934 */ 935 mutex_unlock(&acomp_ctx->mutex); 936 } 937 } 938 939 static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx) 940 { 941 mutex_unlock(&acomp_ctx->mutex); 942 } 943 944 static bool zswap_compress(struct page *page, struct zswap_entry *entry, 945 struct zswap_pool *pool) 946 { 947 struct crypto_acomp_ctx *acomp_ctx; 948 struct scatterlist input, output; 949 int comp_ret = 0, alloc_ret = 0; 950 unsigned int dlen = PAGE_SIZE; 951 unsigned long handle; 952 struct zpool *zpool; 953 gfp_t gfp; 954 u8 *dst; 955 bool mapped = false; 956 957 acomp_ctx = acomp_ctx_get_cpu_lock(pool); 958 dst = acomp_ctx->buffer; 959 sg_init_table(&input, 1); 960 sg_set_page(&input, page, PAGE_SIZE, 0); 961 962 sg_init_one(&output, dst, PAGE_SIZE); 963 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); 964 965 /* 966 * it maybe looks a little bit silly that we send an asynchronous request, 967 * then wait for its completion synchronously. This makes the process look 968 * synchronous in fact. 969 * Theoretically, acomp supports users send multiple acomp requests in one 970 * acomp instance, then get those requests done simultaneously. but in this 971 * case, zswap actually does store and load page by page, there is no 972 * existing method to send the second page before the first page is done 973 * in one thread doing zwap. 974 * but in different threads running on different cpu, we have different 975 * acomp instance, so multiple threads can do (de)compression in parallel. 976 */ 977 comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait); 978 dlen = acomp_ctx->req->dlen; 979 980 /* 981 * If a page cannot be compressed into a size smaller than PAGE_SIZE, 982 * save the content as is without a compression, to keep the LRU order 983 * of writebacks. If writeback is disabled, reject the page since it 984 * only adds metadata overhead. swap_writeout() will put the page back 985 * to the active LRU list in the case. 986 */ 987 if (comp_ret || !dlen || dlen >= PAGE_SIZE) { 988 dlen = PAGE_SIZE; 989 if (!mem_cgroup_zswap_writeback_enabled( 990 folio_memcg(page_folio(page)))) { 991 comp_ret = comp_ret ? comp_ret : -EINVAL; 992 goto unlock; 993 } 994 comp_ret = 0; 995 dlen = PAGE_SIZE; 996 dst = kmap_local_page(page); 997 mapped = true; 998 } 999 1000 zpool = pool->zpool; 1001 gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE; 1002 alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle, page_to_nid(page)); 1003 if (alloc_ret) 1004 goto unlock; 1005 1006 zpool_obj_write(zpool, handle, dst, dlen); 1007 entry->handle = handle; 1008 entry->length = dlen; 1009 1010 unlock: 1011 if (mapped) 1012 kunmap_local(dst); 1013 if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC) 1014 zswap_reject_compress_poor++; 1015 else if (comp_ret) 1016 zswap_reject_compress_fail++; 1017 else if (alloc_ret) 1018 zswap_reject_alloc_fail++; 1019 1020 acomp_ctx_put_unlock(acomp_ctx); 1021 return comp_ret == 0 && alloc_ret == 0; 1022 } 1023 1024 static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio) 1025 { 1026 struct zpool *zpool = entry->pool->zpool; 1027 struct scatterlist input, output; 1028 struct crypto_acomp_ctx *acomp_ctx; 1029 int decomp_ret = 0, dlen = PAGE_SIZE; 1030 u8 *src, *obj; 1031 1032 acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool); 1033 obj = zpool_obj_read_begin(zpool, entry->handle, acomp_ctx->buffer); 1034 1035 /* zswap entries of length PAGE_SIZE are not compressed. */ 1036 if (entry->length == PAGE_SIZE) { 1037 memcpy_to_folio(folio, 0, obj, entry->length); 1038 goto read_done; 1039 } 1040 1041 /* 1042 * zpool_obj_read_begin() might return a kmap address of highmem when 1043 * acomp_ctx->buffer is not used. However, sg_init_one() does not 1044 * handle highmem addresses, so copy the object to acomp_ctx->buffer. 1045 */ 1046 if (virt_addr_valid(obj)) { 1047 src = obj; 1048 } else { 1049 WARN_ON_ONCE(obj == acomp_ctx->buffer); 1050 memcpy(acomp_ctx->buffer, obj, entry->length); 1051 src = acomp_ctx->buffer; 1052 } 1053 1054 sg_init_one(&input, src, entry->length); 1055 sg_init_table(&output, 1); 1056 sg_set_folio(&output, folio, PAGE_SIZE, 0); 1057 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE); 1058 decomp_ret = crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait); 1059 dlen = acomp_ctx->req->dlen; 1060 1061 read_done: 1062 zpool_obj_read_end(zpool, entry->handle, obj); 1063 acomp_ctx_put_unlock(acomp_ctx); 1064 1065 if (!decomp_ret && dlen == PAGE_SIZE) 1066 return true; 1067 1068 zswap_decompress_fail++; 1069 pr_alert_ratelimited("Decompression error from zswap (%d:%lu %s %u->%d)\n", 1070 swp_type(entry->swpentry), 1071 swp_offset(entry->swpentry), 1072 entry->pool->tfm_name, entry->length, dlen); 1073 return false; 1074 } 1075 1076 /********************************* 1077 * writeback code 1078 **********************************/ 1079 /* 1080 * Attempts to free an entry by adding a folio to the swap cache, 1081 * decompressing the entry data into the folio, and issuing a 1082 * bio write to write the folio back to the swap device. 1083 * 1084 * This can be thought of as a "resumed writeback" of the folio 1085 * to the swap device. We are basically resuming the same swap 1086 * writeback path that was intercepted with the zswap_store() 1087 * in the first place. After the folio has been decompressed into 1088 * the swap cache, the compressed version stored by zswap can be 1089 * freed. 1090 */ 1091 static int zswap_writeback_entry(struct zswap_entry *entry, 1092 swp_entry_t swpentry) 1093 { 1094 struct xarray *tree; 1095 pgoff_t offset = swp_offset(swpentry); 1096 struct folio *folio; 1097 struct mempolicy *mpol; 1098 bool folio_was_allocated; 1099 struct swap_info_struct *si; 1100 int ret = 0; 1101 1102 /* try to allocate swap cache folio */ 1103 si = get_swap_device(swpentry); 1104 if (!si) 1105 return -EEXIST; 1106 1107 mpol = get_task_policy(current); 1108 folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol, 1109 NO_INTERLEAVE_INDEX, &folio_was_allocated, true); 1110 put_swap_device(si); 1111 if (!folio) 1112 return -ENOMEM; 1113 1114 /* 1115 * Found an existing folio, we raced with swapin or concurrent 1116 * shrinker. We generally writeback cold folios from zswap, and 1117 * swapin means the folio just became hot, so skip this folio. 1118 * For unlikely concurrent shrinker case, it will be unlinked 1119 * and freed when invalidated by the concurrent shrinker anyway. 1120 */ 1121 if (!folio_was_allocated) { 1122 ret = -EEXIST; 1123 goto out; 1124 } 1125 1126 /* 1127 * folio is locked, and the swapcache is now secured against 1128 * concurrent swapping to and from the slot, and concurrent 1129 * swapoff so we can safely dereference the zswap tree here. 1130 * Verify that the swap entry hasn't been invalidated and recycled 1131 * behind our backs, to avoid overwriting a new swap folio with 1132 * old compressed data. Only when this is successful can the entry 1133 * be dereferenced. 1134 */ 1135 tree = swap_zswap_tree(swpentry); 1136 if (entry != xa_load(tree, offset)) { 1137 ret = -ENOMEM; 1138 goto out; 1139 } 1140 1141 if (!zswap_decompress(entry, folio)) { 1142 ret = -EIO; 1143 goto out; 1144 } 1145 1146 xa_erase(tree, offset); 1147 1148 count_vm_event(ZSWPWB); 1149 if (entry->objcg) 1150 count_objcg_events(entry->objcg, ZSWPWB, 1); 1151 1152 zswap_entry_free(entry); 1153 1154 /* folio is up to date */ 1155 folio_mark_uptodate(folio); 1156 1157 /* move it to the tail of the inactive list after end_writeback */ 1158 folio_set_reclaim(folio); 1159 1160 /* start writeback */ 1161 __swap_writepage(folio, NULL); 1162 1163 out: 1164 if (ret && ret != -EEXIST) { 1165 delete_from_swap_cache(folio); 1166 folio_unlock(folio); 1167 } 1168 folio_put(folio); 1169 return ret; 1170 } 1171 1172 /********************************* 1173 * shrinker functions 1174 **********************************/ 1175 /* 1176 * The dynamic shrinker is modulated by the following factors: 1177 * 1178 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving 1179 * the entry a second chance) before rotating it in the LRU list. If the 1180 * entry is considered again by the shrinker, with its referenced bit unset, 1181 * it is written back. The writeback rate as a result is dynamically 1182 * adjusted by the pool activities - if the pool is dominated by new entries 1183 * (i.e lots of recent zswapouts), these entries will be protected and 1184 * the writeback rate will slow down. On the other hand, if the pool has a 1185 * lot of stagnant entries, these entries will be reclaimed immediately, 1186 * effectively increasing the writeback rate. 1187 * 1188 * 2. Swapins counter: If we observe swapins, it is a sign that we are 1189 * overshrinking and should slow down. We maintain a swapins counter, which 1190 * is consumed and subtract from the number of eligible objects on the LRU 1191 * in zswap_shrinker_count(). 1192 * 1193 * 3. Compression ratio. The better the workload compresses, the less gains we 1194 * can expect from writeback. We scale down the number of objects available 1195 * for reclaim by this ratio. 1196 */ 1197 static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l, 1198 void *arg) 1199 { 1200 struct zswap_entry *entry = container_of(item, struct zswap_entry, lru); 1201 bool *encountered_page_in_swapcache = (bool *)arg; 1202 swp_entry_t swpentry; 1203 enum lru_status ret = LRU_REMOVED_RETRY; 1204 int writeback_result; 1205 1206 /* 1207 * Second chance algorithm: if the entry has its referenced bit set, give it 1208 * a second chance. Only clear the referenced bit and rotate it in the 1209 * zswap's LRU list. 1210 */ 1211 if (entry->referenced) { 1212 entry->referenced = false; 1213 return LRU_ROTATE; 1214 } 1215 1216 /* 1217 * As soon as we drop the LRU lock, the entry can be freed by 1218 * a concurrent invalidation. This means the following: 1219 * 1220 * 1. We extract the swp_entry_t to the stack, allowing 1221 * zswap_writeback_entry() to pin the swap entry and 1222 * then validate the zwap entry against that swap entry's 1223 * tree using pointer value comparison. Only when that 1224 * is successful can the entry be dereferenced. 1225 * 1226 * 2. Usually, objects are taken off the LRU for reclaim. In 1227 * this case this isn't possible, because if reclaim fails 1228 * for whatever reason, we have no means of knowing if the 1229 * entry is alive to put it back on the LRU. 1230 * 1231 * So rotate it before dropping the lock. If the entry is 1232 * written back or invalidated, the free path will unlink 1233 * it. For failures, rotation is the right thing as well. 1234 * 1235 * Temporary failures, where the same entry should be tried 1236 * again immediately, almost never happen for this shrinker. 1237 * We don't do any trylocking; -ENOMEM comes closest, 1238 * but that's extremely rare and doesn't happen spuriously 1239 * either. Don't bother distinguishing this case. 1240 */ 1241 list_move_tail(item, &l->list); 1242 1243 /* 1244 * Once the lru lock is dropped, the entry might get freed. The 1245 * swpentry is copied to the stack, and entry isn't deref'd again 1246 * until the entry is verified to still be alive in the tree. 1247 */ 1248 swpentry = entry->swpentry; 1249 1250 /* 1251 * It's safe to drop the lock here because we return either 1252 * LRU_REMOVED_RETRY, LRU_RETRY or LRU_STOP. 1253 */ 1254 spin_unlock(&l->lock); 1255 1256 writeback_result = zswap_writeback_entry(entry, swpentry); 1257 1258 if (writeback_result) { 1259 zswap_reject_reclaim_fail++; 1260 ret = LRU_RETRY; 1261 1262 /* 1263 * Encountering a page already in swap cache is a sign that we are shrinking 1264 * into the warmer region. We should terminate shrinking (if we're in the dynamic 1265 * shrinker context). 1266 */ 1267 if (writeback_result == -EEXIST && encountered_page_in_swapcache) { 1268 ret = LRU_STOP; 1269 *encountered_page_in_swapcache = true; 1270 } 1271 } else { 1272 zswap_written_back_pages++; 1273 } 1274 1275 return ret; 1276 } 1277 1278 static unsigned long zswap_shrinker_scan(struct shrinker *shrinker, 1279 struct shrink_control *sc) 1280 { 1281 unsigned long shrink_ret; 1282 bool encountered_page_in_swapcache = false; 1283 1284 if (!zswap_shrinker_enabled || 1285 !mem_cgroup_zswap_writeback_enabled(sc->memcg)) { 1286 sc->nr_scanned = 0; 1287 return SHRINK_STOP; 1288 } 1289 1290 shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb, 1291 &encountered_page_in_swapcache); 1292 1293 if (encountered_page_in_swapcache) 1294 return SHRINK_STOP; 1295 1296 return shrink_ret ? shrink_ret : SHRINK_STOP; 1297 } 1298 1299 static unsigned long zswap_shrinker_count(struct shrinker *shrinker, 1300 struct shrink_control *sc) 1301 { 1302 struct mem_cgroup *memcg = sc->memcg; 1303 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid)); 1304 atomic_long_t *nr_disk_swapins = 1305 &lruvec->zswap_lruvec_state.nr_disk_swapins; 1306 unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur, 1307 nr_remain; 1308 1309 if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg)) 1310 return 0; 1311 1312 /* 1313 * The shrinker resumes swap writeback, which will enter block 1314 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS 1315 * rules (may_enter_fs()), which apply on a per-folio basis. 1316 */ 1317 if (!gfp_has_io_fs(sc->gfp_mask)) 1318 return 0; 1319 1320 /* 1321 * For memcg, use the cgroup-wide ZSWAP stats since we don't 1322 * have them per-node and thus per-lruvec. Careful if memcg is 1323 * runtime-disabled: we can get sc->memcg == NULL, which is ok 1324 * for the lruvec, but not for memcg_page_state(). 1325 * 1326 * Without memcg, use the zswap pool-wide metrics. 1327 */ 1328 if (!mem_cgroup_disabled()) { 1329 mem_cgroup_flush_stats(memcg); 1330 nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; 1331 nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); 1332 } else { 1333 nr_backing = zswap_total_pages(); 1334 nr_stored = atomic_long_read(&zswap_stored_pages); 1335 } 1336 1337 if (!nr_stored) 1338 return 0; 1339 1340 nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc); 1341 if (!nr_freeable) 1342 return 0; 1343 1344 /* 1345 * Subtract from the lru size the number of pages that are recently swapped 1346 * in from disk. The idea is that had we protect the zswap's LRU by this 1347 * amount of pages, these disk swapins would not have happened. 1348 */ 1349 nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins); 1350 do { 1351 if (nr_freeable >= nr_disk_swapins_cur) 1352 nr_remain = 0; 1353 else 1354 nr_remain = nr_disk_swapins_cur - nr_freeable; 1355 } while (!atomic_long_try_cmpxchg( 1356 nr_disk_swapins, &nr_disk_swapins_cur, nr_remain)); 1357 1358 nr_freeable -= nr_disk_swapins_cur - nr_remain; 1359 if (!nr_freeable) 1360 return 0; 1361 1362 /* 1363 * Scale the number of freeable pages by the memory saving factor. 1364 * This ensures that the better zswap compresses memory, the fewer 1365 * pages we will evict to swap (as it will otherwise incur IO for 1366 * relatively small memory saving). 1367 */ 1368 return mult_frac(nr_freeable, nr_backing, nr_stored); 1369 } 1370 1371 static struct shrinker *zswap_alloc_shrinker(void) 1372 { 1373 struct shrinker *shrinker; 1374 1375 shrinker = 1376 shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap"); 1377 if (!shrinker) 1378 return NULL; 1379 1380 shrinker->scan_objects = zswap_shrinker_scan; 1381 shrinker->count_objects = zswap_shrinker_count; 1382 shrinker->batch = 0; 1383 shrinker->seeks = DEFAULT_SEEKS; 1384 return shrinker; 1385 } 1386 1387 static int shrink_memcg(struct mem_cgroup *memcg) 1388 { 1389 int nid, shrunk = 0, scanned = 0; 1390 1391 if (!mem_cgroup_zswap_writeback_enabled(memcg)) 1392 return -ENOENT; 1393 1394 /* 1395 * Skip zombies because their LRUs are reparented and we would be 1396 * reclaiming from the parent instead of the dead memcg. 1397 */ 1398 if (memcg && !mem_cgroup_online(memcg)) 1399 return -ENOENT; 1400 1401 for_each_node_state(nid, N_NORMAL_MEMORY) { 1402 unsigned long nr_to_walk = 1; 1403 1404 shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg, 1405 &shrink_memcg_cb, NULL, &nr_to_walk); 1406 scanned += 1 - nr_to_walk; 1407 } 1408 1409 if (!scanned) 1410 return -ENOENT; 1411 1412 return shrunk ? 0 : -EAGAIN; 1413 } 1414 1415 static void shrink_worker(struct work_struct *w) 1416 { 1417 struct mem_cgroup *memcg; 1418 int ret, failures = 0, attempts = 0; 1419 unsigned long thr; 1420 1421 /* Reclaim down to the accept threshold */ 1422 thr = zswap_accept_thr_pages(); 1423 1424 /* 1425 * Global reclaim will select cgroup in a round-robin fashion from all 1426 * online memcgs, but memcgs that have no pages in zswap and 1427 * writeback-disabled memcgs (memory.zswap.writeback=0) are not 1428 * candidates for shrinking. 1429 * 1430 * Shrinking will be aborted if we encounter the following 1431 * MAX_RECLAIM_RETRIES times: 1432 * - No writeback-candidate memcgs found in a memcg tree walk. 1433 * - Shrinking a writeback-candidate memcg failed. 1434 * 1435 * We save iteration cursor memcg into zswap_next_shrink, 1436 * which can be modified by the offline memcg cleaner 1437 * zswap_memcg_offline_cleanup(). 1438 * 1439 * Since the offline cleaner is called only once, we cannot leave an 1440 * offline memcg reference in zswap_next_shrink. 1441 * We can rely on the cleaner only if we get online memcg under lock. 1442 * 1443 * If we get an offline memcg, we cannot determine if the cleaner has 1444 * already been called or will be called later. We must put back the 1445 * reference before returning from this function. Otherwise, the 1446 * offline memcg left in zswap_next_shrink will hold the reference 1447 * until the next run of shrink_worker(). 1448 */ 1449 do { 1450 /* 1451 * Start shrinking from the next memcg after zswap_next_shrink. 1452 * When the offline cleaner has already advanced the cursor, 1453 * advancing the cursor here overlooks one memcg, but this 1454 * should be negligibly rare. 1455 * 1456 * If we get an online memcg, keep the extra reference in case 1457 * the original one obtained by mem_cgroup_iter() is dropped by 1458 * zswap_memcg_offline_cleanup() while we are shrinking the 1459 * memcg. 1460 */ 1461 spin_lock(&zswap_shrink_lock); 1462 do { 1463 memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL); 1464 zswap_next_shrink = memcg; 1465 } while (memcg && !mem_cgroup_tryget_online(memcg)); 1466 spin_unlock(&zswap_shrink_lock); 1467 1468 if (!memcg) { 1469 /* 1470 * Continue shrinking without incrementing failures if 1471 * we found candidate memcgs in the last tree walk. 1472 */ 1473 if (!attempts && ++failures == MAX_RECLAIM_RETRIES) 1474 break; 1475 1476 attempts = 0; 1477 goto resched; 1478 } 1479 1480 ret = shrink_memcg(memcg); 1481 /* drop the extra reference */ 1482 mem_cgroup_put(memcg); 1483 1484 /* 1485 * There are no writeback-candidate pages in the memcg. 1486 * This is not an issue as long as we can find another memcg 1487 * with pages in zswap. Skip this without incrementing attempts 1488 * and failures. 1489 */ 1490 if (ret == -ENOENT) 1491 continue; 1492 ++attempts; 1493 1494 if (ret && ++failures == MAX_RECLAIM_RETRIES) 1495 break; 1496 resched: 1497 cond_resched(); 1498 } while (zswap_total_pages() > thr); 1499 } 1500 1501 /********************************* 1502 * main API 1503 **********************************/ 1504 1505 static bool zswap_store_page(struct page *page, 1506 struct obj_cgroup *objcg, 1507 struct zswap_pool *pool) 1508 { 1509 swp_entry_t page_swpentry = page_swap_entry(page); 1510 struct zswap_entry *entry, *old; 1511 1512 /* allocate entry */ 1513 entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page)); 1514 if (!entry) { 1515 zswap_reject_kmemcache_fail++; 1516 return false; 1517 } 1518 1519 if (!zswap_compress(page, entry, pool)) 1520 goto compress_failed; 1521 1522 old = xa_store(swap_zswap_tree(page_swpentry), 1523 swp_offset(page_swpentry), 1524 entry, GFP_KERNEL); 1525 if (xa_is_err(old)) { 1526 int err = xa_err(old); 1527 1528 WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err); 1529 zswap_reject_alloc_fail++; 1530 goto store_failed; 1531 } 1532 1533 /* 1534 * We may have had an existing entry that became stale when 1535 * the folio was redirtied and now the new version is being 1536 * swapped out. Get rid of the old. 1537 */ 1538 if (old) 1539 zswap_entry_free(old); 1540 1541 /* 1542 * The entry is successfully compressed and stored in the tree, there is 1543 * no further possibility of failure. Grab refs to the pool and objcg, 1544 * charge zswap memory, and increment zswap_stored_pages. 1545 * The opposite actions will be performed by zswap_entry_free() 1546 * when the entry is removed from the tree. 1547 */ 1548 zswap_pool_get(pool); 1549 if (objcg) { 1550 obj_cgroup_get(objcg); 1551 obj_cgroup_charge_zswap(objcg, entry->length); 1552 } 1553 atomic_long_inc(&zswap_stored_pages); 1554 if (entry->length == PAGE_SIZE) 1555 atomic_long_inc(&zswap_stored_incompressible_pages); 1556 1557 /* 1558 * We finish initializing the entry while it's already in xarray. 1559 * This is safe because: 1560 * 1561 * 1. Concurrent stores and invalidations are excluded by folio lock. 1562 * 1563 * 2. Writeback is excluded by the entry not being on the LRU yet. 1564 * The publishing order matters to prevent writeback from seeing 1565 * an incoherent entry. 1566 */ 1567 entry->pool = pool; 1568 entry->swpentry = page_swpentry; 1569 entry->objcg = objcg; 1570 entry->referenced = true; 1571 if (entry->length) { 1572 INIT_LIST_HEAD(&entry->lru); 1573 zswap_lru_add(&zswap_list_lru, entry); 1574 } 1575 1576 return true; 1577 1578 store_failed: 1579 zpool_free(pool->zpool, entry->handle); 1580 compress_failed: 1581 zswap_entry_cache_free(entry); 1582 return false; 1583 } 1584 1585 bool zswap_store(struct folio *folio) 1586 { 1587 long nr_pages = folio_nr_pages(folio); 1588 swp_entry_t swp = folio->swap; 1589 struct obj_cgroup *objcg = NULL; 1590 struct mem_cgroup *memcg = NULL; 1591 struct zswap_pool *pool; 1592 bool ret = false; 1593 long index; 1594 1595 VM_WARN_ON_ONCE(!folio_test_locked(folio)); 1596 VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); 1597 1598 if (!zswap_enabled) 1599 goto check_old; 1600 1601 objcg = get_obj_cgroup_from_folio(folio); 1602 if (objcg && !obj_cgroup_may_zswap(objcg)) { 1603 memcg = get_mem_cgroup_from_objcg(objcg); 1604 if (shrink_memcg(memcg)) { 1605 mem_cgroup_put(memcg); 1606 goto put_objcg; 1607 } 1608 mem_cgroup_put(memcg); 1609 } 1610 1611 if (zswap_check_limits()) 1612 goto put_objcg; 1613 1614 pool = zswap_pool_current_get(); 1615 if (!pool) 1616 goto put_objcg; 1617 1618 if (objcg) { 1619 memcg = get_mem_cgroup_from_objcg(objcg); 1620 if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) { 1621 mem_cgroup_put(memcg); 1622 goto put_pool; 1623 } 1624 mem_cgroup_put(memcg); 1625 } 1626 1627 for (index = 0; index < nr_pages; ++index) { 1628 struct page *page = folio_page(folio, index); 1629 1630 if (!zswap_store_page(page, objcg, pool)) 1631 goto put_pool; 1632 } 1633 1634 if (objcg) 1635 count_objcg_events(objcg, ZSWPOUT, nr_pages); 1636 1637 count_vm_events(ZSWPOUT, nr_pages); 1638 1639 ret = true; 1640 1641 put_pool: 1642 zswap_pool_put(pool); 1643 put_objcg: 1644 obj_cgroup_put(objcg); 1645 if (!ret && zswap_pool_reached_full) 1646 queue_work(shrink_wq, &zswap_shrink_work); 1647 check_old: 1648 /* 1649 * If the zswap store fails or zswap is disabled, we must invalidate 1650 * the possibly stale entries which were previously stored at the 1651 * offsets corresponding to each page of the folio. Otherwise, 1652 * writeback could overwrite the new data in the swapfile. 1653 */ 1654 if (!ret) { 1655 unsigned type = swp_type(swp); 1656 pgoff_t offset = swp_offset(swp); 1657 struct zswap_entry *entry; 1658 struct xarray *tree; 1659 1660 for (index = 0; index < nr_pages; ++index) { 1661 tree = swap_zswap_tree(swp_entry(type, offset + index)); 1662 entry = xa_erase(tree, offset + index); 1663 if (entry) 1664 zswap_entry_free(entry); 1665 } 1666 } 1667 1668 return ret; 1669 } 1670 1671 /** 1672 * zswap_load() - load a folio from zswap 1673 * @folio: folio to load 1674 * 1675 * Return: 0 on success, with the folio unlocked and marked up-to-date, or one 1676 * of the following error codes: 1677 * 1678 * -EIO: if the swapped out content was in zswap, but could not be loaded 1679 * into the page due to a decompression failure. The folio is unlocked, but 1680 * NOT marked up-to-date, so that an IO error is emitted (e.g. do_swap_page() 1681 * will SIGBUS). 1682 * 1683 * -EINVAL: if the swapped out content was in zswap, but the page belongs 1684 * to a large folio, which is not supported by zswap. The folio is unlocked, 1685 * but NOT marked up-to-date, so that an IO error is emitted (e.g. 1686 * do_swap_page() will SIGBUS). 1687 * 1688 * -ENOENT: if the swapped out content was not in zswap. The folio remains 1689 * locked on return. 1690 */ 1691 int zswap_load(struct folio *folio) 1692 { 1693 swp_entry_t swp = folio->swap; 1694 pgoff_t offset = swp_offset(swp); 1695 bool swapcache = folio_test_swapcache(folio); 1696 struct xarray *tree = swap_zswap_tree(swp); 1697 struct zswap_entry *entry; 1698 1699 VM_WARN_ON_ONCE(!folio_test_locked(folio)); 1700 1701 if (zswap_never_enabled()) 1702 return -ENOENT; 1703 1704 /* 1705 * Large folios should not be swapped in while zswap is being used, as 1706 * they are not properly handled. Zswap does not properly load large 1707 * folios, and a large folio may only be partially in zswap. 1708 */ 1709 if (WARN_ON_ONCE(folio_test_large(folio))) { 1710 folio_unlock(folio); 1711 return -EINVAL; 1712 } 1713 1714 entry = xa_load(tree, offset); 1715 if (!entry) 1716 return -ENOENT; 1717 1718 if (!zswap_decompress(entry, folio)) { 1719 folio_unlock(folio); 1720 return -EIO; 1721 } 1722 1723 folio_mark_uptodate(folio); 1724 1725 count_vm_event(ZSWPIN); 1726 if (entry->objcg) 1727 count_objcg_events(entry->objcg, ZSWPIN, 1); 1728 1729 /* 1730 * When reading into the swapcache, invalidate our entry. The 1731 * swapcache can be the authoritative owner of the page and 1732 * its mappings, and the pressure that results from having two 1733 * in-memory copies outweighs any benefits of caching the 1734 * compression work. 1735 * 1736 * (Most swapins go through the swapcache. The notable 1737 * exception is the singleton fault on SWP_SYNCHRONOUS_IO 1738 * files, which reads into a private page and may free it if 1739 * the fault fails. We remain the primary owner of the entry.) 1740 */ 1741 if (swapcache) { 1742 folio_mark_dirty(folio); 1743 xa_erase(tree, offset); 1744 zswap_entry_free(entry); 1745 } 1746 1747 folio_unlock(folio); 1748 return 0; 1749 } 1750 1751 void zswap_invalidate(swp_entry_t swp) 1752 { 1753 pgoff_t offset = swp_offset(swp); 1754 struct xarray *tree = swap_zswap_tree(swp); 1755 struct zswap_entry *entry; 1756 1757 if (xa_empty(tree)) 1758 return; 1759 1760 entry = xa_erase(tree, offset); 1761 if (entry) 1762 zswap_entry_free(entry); 1763 } 1764 1765 int zswap_swapon(int type, unsigned long nr_pages) 1766 { 1767 struct xarray *trees, *tree; 1768 unsigned int nr, i; 1769 1770 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); 1771 trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL); 1772 if (!trees) { 1773 pr_err("alloc failed, zswap disabled for swap type %d\n", type); 1774 return -ENOMEM; 1775 } 1776 1777 for (i = 0; i < nr; i++) 1778 xa_init(trees + i); 1779 1780 nr_zswap_trees[type] = nr; 1781 zswap_trees[type] = trees; 1782 return 0; 1783 } 1784 1785 void zswap_swapoff(int type) 1786 { 1787 struct xarray *trees = zswap_trees[type]; 1788 unsigned int i; 1789 1790 if (!trees) 1791 return; 1792 1793 /* try_to_unuse() invalidated all the entries already */ 1794 for (i = 0; i < nr_zswap_trees[type]; i++) 1795 WARN_ON_ONCE(!xa_empty(trees + i)); 1796 1797 kvfree(trees); 1798 nr_zswap_trees[type] = 0; 1799 zswap_trees[type] = NULL; 1800 } 1801 1802 /********************************* 1803 * debugfs functions 1804 **********************************/ 1805 #ifdef CONFIG_DEBUG_FS 1806 #include <linux/debugfs.h> 1807 1808 static struct dentry *zswap_debugfs_root; 1809 1810 static int debugfs_get_total_size(void *data, u64 *val) 1811 { 1812 *val = zswap_total_pages() * PAGE_SIZE; 1813 return 0; 1814 } 1815 DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n"); 1816 1817 static int debugfs_get_stored_pages(void *data, u64 *val) 1818 { 1819 *val = atomic_long_read(&zswap_stored_pages); 1820 return 0; 1821 } 1822 DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n"); 1823 1824 static int debugfs_get_stored_incompressible_pages(void *data, u64 *val) 1825 { 1826 *val = atomic_long_read(&zswap_stored_incompressible_pages); 1827 return 0; 1828 } 1829 DEFINE_DEBUGFS_ATTRIBUTE(stored_incompressible_pages_fops, 1830 debugfs_get_stored_incompressible_pages, NULL, "%llu\n"); 1831 1832 static int zswap_debugfs_init(void) 1833 { 1834 if (!debugfs_initialized()) 1835 return -ENODEV; 1836 1837 zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 1838 1839 debugfs_create_u64("pool_limit_hit", 0444, 1840 zswap_debugfs_root, &zswap_pool_limit_hit); 1841 debugfs_create_u64("reject_reclaim_fail", 0444, 1842 zswap_debugfs_root, &zswap_reject_reclaim_fail); 1843 debugfs_create_u64("reject_alloc_fail", 0444, 1844 zswap_debugfs_root, &zswap_reject_alloc_fail); 1845 debugfs_create_u64("reject_kmemcache_fail", 0444, 1846 zswap_debugfs_root, &zswap_reject_kmemcache_fail); 1847 debugfs_create_u64("reject_compress_fail", 0444, 1848 zswap_debugfs_root, &zswap_reject_compress_fail); 1849 debugfs_create_u64("reject_compress_poor", 0444, 1850 zswap_debugfs_root, &zswap_reject_compress_poor); 1851 debugfs_create_u64("decompress_fail", 0444, 1852 zswap_debugfs_root, &zswap_decompress_fail); 1853 debugfs_create_u64("written_back_pages", 0444, 1854 zswap_debugfs_root, &zswap_written_back_pages); 1855 debugfs_create_file("pool_total_size", 0444, 1856 zswap_debugfs_root, NULL, &total_size_fops); 1857 debugfs_create_file("stored_pages", 0444, 1858 zswap_debugfs_root, NULL, &stored_pages_fops); 1859 debugfs_create_file("stored_incompressible_pages", 0444, 1860 zswap_debugfs_root, NULL, 1861 &stored_incompressible_pages_fops); 1862 1863 return 0; 1864 } 1865 #else 1866 static int zswap_debugfs_init(void) 1867 { 1868 return 0; 1869 } 1870 #endif 1871 1872 /********************************* 1873 * module init and exit 1874 **********************************/ 1875 static int zswap_setup(void) 1876 { 1877 struct zswap_pool *pool; 1878 int ret; 1879 1880 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 1881 if (!zswap_entry_cache) { 1882 pr_err("entry cache creation failed\n"); 1883 goto cache_fail; 1884 } 1885 1886 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, 1887 "mm/zswap_pool:prepare", 1888 zswap_cpu_comp_prepare, 1889 zswap_cpu_comp_dead); 1890 if (ret) 1891 goto hp_fail; 1892 1893 shrink_wq = alloc_workqueue("zswap-shrink", 1894 WQ_UNBOUND|WQ_MEM_RECLAIM, 1); 1895 if (!shrink_wq) 1896 goto shrink_wq_fail; 1897 1898 zswap_shrinker = zswap_alloc_shrinker(); 1899 if (!zswap_shrinker) 1900 goto shrinker_fail; 1901 if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker)) 1902 goto lru_fail; 1903 shrinker_register(zswap_shrinker); 1904 1905 INIT_WORK(&zswap_shrink_work, shrink_worker); 1906 1907 pool = __zswap_pool_create_fallback(); 1908 if (pool) { 1909 pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1910 zpool_get_type(pool->zpool)); 1911 list_add(&pool->list, &zswap_pools); 1912 zswap_has_pool = true; 1913 static_branch_enable(&zswap_ever_enabled); 1914 } else { 1915 pr_err("pool creation failed\n"); 1916 zswap_enabled = false; 1917 } 1918 1919 if (zswap_debugfs_init()) 1920 pr_warn("debugfs initialization failed\n"); 1921 zswap_init_state = ZSWAP_INIT_SUCCEED; 1922 return 0; 1923 1924 lru_fail: 1925 shrinker_free(zswap_shrinker); 1926 shrinker_fail: 1927 destroy_workqueue(shrink_wq); 1928 shrink_wq_fail: 1929 cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE); 1930 hp_fail: 1931 kmem_cache_destroy(zswap_entry_cache); 1932 cache_fail: 1933 /* if built-in, we aren't unloaded on failure; don't allow use */ 1934 zswap_init_state = ZSWAP_INIT_FAILED; 1935 zswap_enabled = false; 1936 return -ENOMEM; 1937 } 1938 1939 static int __init zswap_init(void) 1940 { 1941 if (!zswap_enabled) 1942 return 0; 1943 return zswap_setup(); 1944 } 1945 /* must be late so crypto has time to come up */ 1946 late_initcall(zswap_init); 1947 1948 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 1949 MODULE_DESCRIPTION("Compressed cache for swap pages"); 1950