1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * zswap.c - zswap driver file 4 * 5 * zswap is a cache that takes pages that are in the process 6 * of being swapped out and attempts to compress and store them in a 7 * RAM-based memory pool. This can result in a significant I/O reduction on 8 * the swap device and, in the case where decompressing from RAM is faster 9 * than reading from the swap device, can also improve workload performance. 10 * 11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 15 16 #include <linux/module.h> 17 #include <linux/cpu.h> 18 #include <linux/highmem.h> 19 #include <linux/slab.h> 20 #include <linux/spinlock.h> 21 #include <linux/types.h> 22 #include <linux/atomic.h> 23 #include <linux/swap.h> 24 #include <linux/crypto.h> 25 #include <linux/scatterlist.h> 26 #include <linux/mempolicy.h> 27 #include <linux/mempool.h> 28 #include <linux/zpool.h> 29 #include <crypto/acompress.h> 30 #include <linux/zswap.h> 31 #include <linux/mm_types.h> 32 #include <linux/page-flags.h> 33 #include <linux/swapops.h> 34 #include <linux/writeback.h> 35 #include <linux/pagemap.h> 36 #include <linux/workqueue.h> 37 #include <linux/list_lru.h> 38 39 #include "swap.h" 40 #include "internal.h" 41 42 /********************************* 43 * statistics 44 **********************************/ 45 /* The number of compressed pages currently stored in zswap */ 46 atomic_long_t zswap_stored_pages = ATOMIC_LONG_INIT(0); 47 48 /* 49 * The statistics below are not protected from concurrent access for 50 * performance reasons so they may not be a 100% accurate. However, 51 * they do provide useful information on roughly how many times a 52 * certain event is occurring. 53 */ 54 55 /* Pool limit was hit (see zswap_max_pool_percent) */ 56 static u64 zswap_pool_limit_hit; 57 /* Pages written back when pool limit was reached */ 58 static u64 zswap_written_back_pages; 59 /* Store failed due to a reclaim failure after pool limit was reached */ 60 static u64 zswap_reject_reclaim_fail; 61 /* Store failed due to compression algorithm failure */ 62 static u64 zswap_reject_compress_fail; 63 /* Compressed page was too big for the allocator to (optimally) store */ 64 static u64 zswap_reject_compress_poor; 65 /* Store failed because underlying allocator could not get memory */ 66 static u64 zswap_reject_alloc_fail; 67 /* Store failed because the entry metadata could not be allocated (rare) */ 68 static u64 zswap_reject_kmemcache_fail; 69 70 /* Shrinker work queue */ 71 static struct workqueue_struct *shrink_wq; 72 /* Pool limit was hit, we need to calm down */ 73 static bool zswap_pool_reached_full; 74 75 /********************************* 76 * tunables 77 **********************************/ 78 79 #define ZSWAP_PARAM_UNSET "" 80 81 static int zswap_setup(void); 82 83 /* Enable/disable zswap */ 84 static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled); 85 static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); 86 static int zswap_enabled_param_set(const char *, 87 const struct kernel_param *); 88 static const struct kernel_param_ops zswap_enabled_param_ops = { 89 .set = zswap_enabled_param_set, 90 .get = param_get_bool, 91 }; 92 module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); 93 94 /* Crypto compressor to use */ 95 static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 96 static int zswap_compressor_param_set(const char *, 97 const struct kernel_param *); 98 static const struct kernel_param_ops zswap_compressor_param_ops = { 99 .set = zswap_compressor_param_set, 100 .get = param_get_charp, 101 .free = param_free_charp, 102 }; 103 module_param_cb(compressor, &zswap_compressor_param_ops, 104 &zswap_compressor, 0644); 105 106 /* Compressed storage zpool to use */ 107 static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 108 static int zswap_zpool_param_set(const char *, const struct kernel_param *); 109 static const struct kernel_param_ops zswap_zpool_param_ops = { 110 .set = zswap_zpool_param_set, 111 .get = param_get_charp, 112 .free = param_free_charp, 113 }; 114 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 115 116 /* The maximum percentage of memory that the compressed pool can occupy */ 117 static unsigned int zswap_max_pool_percent = 20; 118 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 119 120 /* The threshold for accepting new pages after the max_pool_percent was hit */ 121 static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ 122 module_param_named(accept_threshold_percent, zswap_accept_thr_percent, 123 uint, 0644); 124 125 /* Enable/disable memory pressure-based shrinker. */ 126 static bool zswap_shrinker_enabled = IS_ENABLED( 127 CONFIG_ZSWAP_SHRINKER_DEFAULT_ON); 128 module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644); 129 130 bool zswap_is_enabled(void) 131 { 132 return zswap_enabled; 133 } 134 135 bool zswap_never_enabled(void) 136 { 137 return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled); 138 } 139 140 /********************************* 141 * data structures 142 **********************************/ 143 144 struct crypto_acomp_ctx { 145 struct crypto_acomp *acomp; 146 struct acomp_req *req; 147 struct crypto_wait wait; 148 u8 *buffer; 149 struct mutex mutex; 150 bool is_sleepable; 151 }; 152 153 /* 154 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock. 155 * The only case where lru_lock is not acquired while holding tree.lock is 156 * when a zswap_entry is taken off the lru for writeback, in that case it 157 * needs to be verified that it's still valid in the tree. 158 */ 159 struct zswap_pool { 160 struct zpool *zpool; 161 struct crypto_acomp_ctx __percpu *acomp_ctx; 162 struct percpu_ref ref; 163 struct list_head list; 164 struct work_struct release_work; 165 struct hlist_node node; 166 char tfm_name[CRYPTO_MAX_ALG_NAME]; 167 }; 168 169 /* Global LRU lists shared by all zswap pools. */ 170 static struct list_lru zswap_list_lru; 171 172 /* The lock protects zswap_next_shrink updates. */ 173 static DEFINE_SPINLOCK(zswap_shrink_lock); 174 static struct mem_cgroup *zswap_next_shrink; 175 static struct work_struct zswap_shrink_work; 176 static struct shrinker *zswap_shrinker; 177 178 /* 179 * struct zswap_entry 180 * 181 * This structure contains the metadata for tracking a single compressed 182 * page within zswap. 183 * 184 * swpentry - associated swap entry, the offset indexes into the red-black tree 185 * length - the length in bytes of the compressed page data. Needed during 186 * decompression. 187 * referenced - true if the entry recently entered the zswap pool. Unset by the 188 * writeback logic. The entry is only reclaimed by the writeback 189 * logic if referenced is unset. See comments in the shrinker 190 * section for context. 191 * pool - the zswap_pool the entry's data is in 192 * handle - zpool allocation handle that stores the compressed page data 193 * objcg - the obj_cgroup that the compressed memory is charged to 194 * lru - handle to the pool's lru used to evict pages. 195 */ 196 struct zswap_entry { 197 swp_entry_t swpentry; 198 unsigned int length; 199 bool referenced; 200 struct zswap_pool *pool; 201 unsigned long handle; 202 struct obj_cgroup *objcg; 203 struct list_head lru; 204 }; 205 206 static struct xarray *zswap_trees[MAX_SWAPFILES]; 207 static unsigned int nr_zswap_trees[MAX_SWAPFILES]; 208 209 /* RCU-protected iteration */ 210 static LIST_HEAD(zswap_pools); 211 /* protects zswap_pools list modification */ 212 static DEFINE_SPINLOCK(zswap_pools_lock); 213 /* pool counter to provide unique names to zpool */ 214 static atomic_t zswap_pools_count = ATOMIC_INIT(0); 215 216 enum zswap_init_type { 217 ZSWAP_UNINIT, 218 ZSWAP_INIT_SUCCEED, 219 ZSWAP_INIT_FAILED 220 }; 221 222 static enum zswap_init_type zswap_init_state; 223 224 /* used to ensure the integrity of initialization */ 225 static DEFINE_MUTEX(zswap_init_lock); 226 227 /* init completed, but couldn't create the initial pool */ 228 static bool zswap_has_pool; 229 230 /********************************* 231 * helpers and fwd declarations 232 **********************************/ 233 234 static inline struct xarray *swap_zswap_tree(swp_entry_t swp) 235 { 236 return &zswap_trees[swp_type(swp)][swp_offset(swp) 237 >> SWAP_ADDRESS_SPACE_SHIFT]; 238 } 239 240 #define zswap_pool_debug(msg, p) \ 241 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 242 zpool_get_type((p)->zpool)) 243 244 /********************************* 245 * pool functions 246 **********************************/ 247 static void __zswap_pool_empty(struct percpu_ref *ref); 248 249 static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 250 { 251 struct zswap_pool *pool; 252 char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 253 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 254 int ret, cpu; 255 256 if (!zswap_has_pool) { 257 /* if either are unset, pool initialization failed, and we 258 * need both params to be set correctly before trying to 259 * create a pool. 260 */ 261 if (!strcmp(type, ZSWAP_PARAM_UNSET)) 262 return NULL; 263 if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) 264 return NULL; 265 } 266 267 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 268 if (!pool) 269 return NULL; 270 271 /* unique name for each pool specifically required by zsmalloc */ 272 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count)); 273 pool->zpool = zpool_create_pool(type, name, gfp); 274 if (!pool->zpool) { 275 pr_err("%s zpool not available\n", type); 276 goto error; 277 } 278 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); 279 280 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 281 282 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx); 283 if (!pool->acomp_ctx) { 284 pr_err("percpu alloc failed\n"); 285 goto error; 286 } 287 288 for_each_possible_cpu(cpu) 289 mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex); 290 291 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, 292 &pool->node); 293 if (ret) 294 goto error; 295 296 /* being the current pool takes 1 ref; this func expects the 297 * caller to always add the new pool as the current pool 298 */ 299 ret = percpu_ref_init(&pool->ref, __zswap_pool_empty, 300 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); 301 if (ret) 302 goto ref_fail; 303 INIT_LIST_HEAD(&pool->list); 304 305 zswap_pool_debug("created", pool); 306 307 return pool; 308 309 ref_fail: 310 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 311 error: 312 if (pool->acomp_ctx) 313 free_percpu(pool->acomp_ctx); 314 if (pool->zpool) 315 zpool_destroy_pool(pool->zpool); 316 kfree(pool); 317 return NULL; 318 } 319 320 static struct zswap_pool *__zswap_pool_create_fallback(void) 321 { 322 bool has_comp, has_zpool; 323 324 has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 325 if (!has_comp && strcmp(zswap_compressor, 326 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { 327 pr_err("compressor %s not available, using default %s\n", 328 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); 329 param_free_charp(&zswap_compressor); 330 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; 331 has_comp = crypto_has_acomp(zswap_compressor, 0, 0); 332 } 333 if (!has_comp) { 334 pr_err("default compressor %s not available\n", 335 zswap_compressor); 336 param_free_charp(&zswap_compressor); 337 zswap_compressor = ZSWAP_PARAM_UNSET; 338 } 339 340 has_zpool = zpool_has_pool(zswap_zpool_type); 341 if (!has_zpool && strcmp(zswap_zpool_type, 342 CONFIG_ZSWAP_ZPOOL_DEFAULT)) { 343 pr_err("zpool %s not available, using default %s\n", 344 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); 345 param_free_charp(&zswap_zpool_type); 346 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; 347 has_zpool = zpool_has_pool(zswap_zpool_type); 348 } 349 if (!has_zpool) { 350 pr_err("default zpool %s not available\n", 351 zswap_zpool_type); 352 param_free_charp(&zswap_zpool_type); 353 zswap_zpool_type = ZSWAP_PARAM_UNSET; 354 } 355 356 if (!has_comp || !has_zpool) 357 return NULL; 358 359 return zswap_pool_create(zswap_zpool_type, zswap_compressor); 360 } 361 362 static void zswap_pool_destroy(struct zswap_pool *pool) 363 { 364 zswap_pool_debug("destroying", pool); 365 366 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); 367 free_percpu(pool->acomp_ctx); 368 369 zpool_destroy_pool(pool->zpool); 370 kfree(pool); 371 } 372 373 static void __zswap_pool_release(struct work_struct *work) 374 { 375 struct zswap_pool *pool = container_of(work, typeof(*pool), 376 release_work); 377 378 synchronize_rcu(); 379 380 /* nobody should have been able to get a ref... */ 381 WARN_ON(!percpu_ref_is_zero(&pool->ref)); 382 percpu_ref_exit(&pool->ref); 383 384 /* pool is now off zswap_pools list and has no references. */ 385 zswap_pool_destroy(pool); 386 } 387 388 static struct zswap_pool *zswap_pool_current(void); 389 390 static void __zswap_pool_empty(struct percpu_ref *ref) 391 { 392 struct zswap_pool *pool; 393 394 pool = container_of(ref, typeof(*pool), ref); 395 396 spin_lock_bh(&zswap_pools_lock); 397 398 WARN_ON(pool == zswap_pool_current()); 399 400 list_del_rcu(&pool->list); 401 402 INIT_WORK(&pool->release_work, __zswap_pool_release); 403 schedule_work(&pool->release_work); 404 405 spin_unlock_bh(&zswap_pools_lock); 406 } 407 408 static int __must_check zswap_pool_tryget(struct zswap_pool *pool) 409 { 410 if (!pool) 411 return 0; 412 413 return percpu_ref_tryget(&pool->ref); 414 } 415 416 /* The caller must already have a reference. */ 417 static void zswap_pool_get(struct zswap_pool *pool) 418 { 419 percpu_ref_get(&pool->ref); 420 } 421 422 static void zswap_pool_put(struct zswap_pool *pool) 423 { 424 percpu_ref_put(&pool->ref); 425 } 426 427 static struct zswap_pool *__zswap_pool_current(void) 428 { 429 struct zswap_pool *pool; 430 431 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 432 WARN_ONCE(!pool && zswap_has_pool, 433 "%s: no page storage pool!\n", __func__); 434 435 return pool; 436 } 437 438 static struct zswap_pool *zswap_pool_current(void) 439 { 440 assert_spin_locked(&zswap_pools_lock); 441 442 return __zswap_pool_current(); 443 } 444 445 static struct zswap_pool *zswap_pool_current_get(void) 446 { 447 struct zswap_pool *pool; 448 449 rcu_read_lock(); 450 451 pool = __zswap_pool_current(); 452 if (!zswap_pool_tryget(pool)) 453 pool = NULL; 454 455 rcu_read_unlock(); 456 457 return pool; 458 } 459 460 /* type and compressor must be null-terminated */ 461 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 462 { 463 struct zswap_pool *pool; 464 465 assert_spin_locked(&zswap_pools_lock); 466 467 list_for_each_entry_rcu(pool, &zswap_pools, list) { 468 if (strcmp(pool->tfm_name, compressor)) 469 continue; 470 if (strcmp(zpool_get_type(pool->zpool), type)) 471 continue; 472 /* if we can't get it, it's about to be destroyed */ 473 if (!zswap_pool_tryget(pool)) 474 continue; 475 return pool; 476 } 477 478 return NULL; 479 } 480 481 static unsigned long zswap_max_pages(void) 482 { 483 return totalram_pages() * zswap_max_pool_percent / 100; 484 } 485 486 static unsigned long zswap_accept_thr_pages(void) 487 { 488 return zswap_max_pages() * zswap_accept_thr_percent / 100; 489 } 490 491 unsigned long zswap_total_pages(void) 492 { 493 struct zswap_pool *pool; 494 unsigned long total = 0; 495 496 rcu_read_lock(); 497 list_for_each_entry_rcu(pool, &zswap_pools, list) 498 total += zpool_get_total_pages(pool->zpool); 499 rcu_read_unlock(); 500 501 return total; 502 } 503 504 static bool zswap_check_limits(void) 505 { 506 unsigned long cur_pages = zswap_total_pages(); 507 unsigned long max_pages = zswap_max_pages(); 508 509 if (cur_pages >= max_pages) { 510 zswap_pool_limit_hit++; 511 zswap_pool_reached_full = true; 512 } else if (zswap_pool_reached_full && 513 cur_pages <= zswap_accept_thr_pages()) { 514 zswap_pool_reached_full = false; 515 } 516 return zswap_pool_reached_full; 517 } 518 519 /********************************* 520 * param callbacks 521 **********************************/ 522 523 static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) 524 { 525 /* no change required */ 526 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) 527 return false; 528 return true; 529 } 530 531 /* val must be a null-terminated string */ 532 static int __zswap_param_set(const char *val, const struct kernel_param *kp, 533 char *type, char *compressor) 534 { 535 struct zswap_pool *pool, *put_pool = NULL; 536 char *s = strstrip((char *)val); 537 int ret = 0; 538 bool new_pool = false; 539 540 mutex_lock(&zswap_init_lock); 541 switch (zswap_init_state) { 542 case ZSWAP_UNINIT: 543 /* if this is load-time (pre-init) param setting, 544 * don't create a pool; that's done during init. 545 */ 546 ret = param_set_charp(s, kp); 547 break; 548 case ZSWAP_INIT_SUCCEED: 549 new_pool = zswap_pool_changed(s, kp); 550 break; 551 case ZSWAP_INIT_FAILED: 552 pr_err("can't set param, initialization failed\n"); 553 ret = -ENODEV; 554 } 555 mutex_unlock(&zswap_init_lock); 556 557 /* no need to create a new pool, return directly */ 558 if (!new_pool) 559 return ret; 560 561 if (!type) { 562 if (!zpool_has_pool(s)) { 563 pr_err("zpool %s not available\n", s); 564 return -ENOENT; 565 } 566 type = s; 567 } else if (!compressor) { 568 if (!crypto_has_acomp(s, 0, 0)) { 569 pr_err("compressor %s not available\n", s); 570 return -ENOENT; 571 } 572 compressor = s; 573 } else { 574 WARN_ON(1); 575 return -EINVAL; 576 } 577 578 spin_lock_bh(&zswap_pools_lock); 579 580 pool = zswap_pool_find_get(type, compressor); 581 if (pool) { 582 zswap_pool_debug("using existing", pool); 583 WARN_ON(pool == zswap_pool_current()); 584 list_del_rcu(&pool->list); 585 } 586 587 spin_unlock_bh(&zswap_pools_lock); 588 589 if (!pool) 590 pool = zswap_pool_create(type, compressor); 591 else { 592 /* 593 * Restore the initial ref dropped by percpu_ref_kill() 594 * when the pool was decommissioned and switch it again 595 * to percpu mode. 596 */ 597 percpu_ref_resurrect(&pool->ref); 598 599 /* Drop the ref from zswap_pool_find_get(). */ 600 zswap_pool_put(pool); 601 } 602 603 if (pool) 604 ret = param_set_charp(s, kp); 605 else 606 ret = -EINVAL; 607 608 spin_lock_bh(&zswap_pools_lock); 609 610 if (!ret) { 611 put_pool = zswap_pool_current(); 612 list_add_rcu(&pool->list, &zswap_pools); 613 zswap_has_pool = true; 614 } else if (pool) { 615 /* add the possibly pre-existing pool to the end of the pools 616 * list; if it's new (and empty) then it'll be removed and 617 * destroyed by the put after we drop the lock 618 */ 619 list_add_tail_rcu(&pool->list, &zswap_pools); 620 put_pool = pool; 621 } 622 623 spin_unlock_bh(&zswap_pools_lock); 624 625 if (!zswap_has_pool && !pool) { 626 /* if initial pool creation failed, and this pool creation also 627 * failed, maybe both compressor and zpool params were bad. 628 * Allow changing this param, so pool creation will succeed 629 * when the other param is changed. We already verified this 630 * param is ok in the zpool_has_pool() or crypto_has_acomp() 631 * checks above. 632 */ 633 ret = param_set_charp(s, kp); 634 } 635 636 /* drop the ref from either the old current pool, 637 * or the new pool we failed to add 638 */ 639 if (put_pool) 640 percpu_ref_kill(&put_pool->ref); 641 642 return ret; 643 } 644 645 static int zswap_compressor_param_set(const char *val, 646 const struct kernel_param *kp) 647 { 648 return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 649 } 650 651 static int zswap_zpool_param_set(const char *val, 652 const struct kernel_param *kp) 653 { 654 return __zswap_param_set(val, kp, NULL, zswap_compressor); 655 } 656 657 static int zswap_enabled_param_set(const char *val, 658 const struct kernel_param *kp) 659 { 660 int ret = -ENODEV; 661 662 /* if this is load-time (pre-init) param setting, only set param. */ 663 if (system_state != SYSTEM_RUNNING) 664 return param_set_bool(val, kp); 665 666 mutex_lock(&zswap_init_lock); 667 switch (zswap_init_state) { 668 case ZSWAP_UNINIT: 669 if (zswap_setup()) 670 break; 671 fallthrough; 672 case ZSWAP_INIT_SUCCEED: 673 if (!zswap_has_pool) 674 pr_err("can't enable, no pool configured\n"); 675 else 676 ret = param_set_bool(val, kp); 677 break; 678 case ZSWAP_INIT_FAILED: 679 pr_err("can't enable, initialization failed\n"); 680 } 681 mutex_unlock(&zswap_init_lock); 682 683 return ret; 684 } 685 686 /********************************* 687 * lru functions 688 **********************************/ 689 690 /* should be called under RCU */ 691 #ifdef CONFIG_MEMCG 692 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) 693 { 694 return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL; 695 } 696 #else 697 static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry) 698 { 699 return NULL; 700 } 701 #endif 702 703 static inline int entry_to_nid(struct zswap_entry *entry) 704 { 705 return page_to_nid(virt_to_page(entry)); 706 } 707 708 static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry) 709 { 710 int nid = entry_to_nid(entry); 711 struct mem_cgroup *memcg; 712 713 /* 714 * Note that it is safe to use rcu_read_lock() here, even in the face of 715 * concurrent memcg offlining: 716 * 717 * 1. list_lru_add() is called before list_lru_one is dead. The 718 * new entry will be reparented to memcg's parent's list_lru. 719 * 2. list_lru_add() is called after list_lru_one is dead. The 720 * new entry will be added directly to memcg's parent's list_lru. 721 * 722 * Similar reasoning holds for list_lru_del(). 723 */ 724 rcu_read_lock(); 725 memcg = mem_cgroup_from_entry(entry); 726 /* will always succeed */ 727 list_lru_add(list_lru, &entry->lru, nid, memcg); 728 rcu_read_unlock(); 729 } 730 731 static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry) 732 { 733 int nid = entry_to_nid(entry); 734 struct mem_cgroup *memcg; 735 736 rcu_read_lock(); 737 memcg = mem_cgroup_from_entry(entry); 738 /* will always succeed */ 739 list_lru_del(list_lru, &entry->lru, nid, memcg); 740 rcu_read_unlock(); 741 } 742 743 void zswap_lruvec_state_init(struct lruvec *lruvec) 744 { 745 atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0); 746 } 747 748 void zswap_folio_swapin(struct folio *folio) 749 { 750 struct lruvec *lruvec; 751 752 if (folio) { 753 lruvec = folio_lruvec(folio); 754 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins); 755 } 756 } 757 758 /* 759 * This function should be called when a memcg is being offlined. 760 * 761 * Since the global shrinker shrink_worker() may hold a reference 762 * of the memcg, we must check and release the reference in 763 * zswap_next_shrink. 764 * 765 * shrink_worker() must handle the case where this function releases 766 * the reference of memcg being shrunk. 767 */ 768 void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) 769 { 770 /* lock out zswap shrinker walking memcg tree */ 771 spin_lock(&zswap_shrink_lock); 772 if (zswap_next_shrink == memcg) { 773 do { 774 zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL); 775 } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink)); 776 } 777 spin_unlock(&zswap_shrink_lock); 778 } 779 780 /********************************* 781 * zswap entry functions 782 **********************************/ 783 static struct kmem_cache *zswap_entry_cache; 784 785 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid) 786 { 787 struct zswap_entry *entry; 788 entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid); 789 if (!entry) 790 return NULL; 791 return entry; 792 } 793 794 static void zswap_entry_cache_free(struct zswap_entry *entry) 795 { 796 kmem_cache_free(zswap_entry_cache, entry); 797 } 798 799 /* 800 * Carries out the common pattern of freeing and entry's zpool allocation, 801 * freeing the entry itself, and decrementing the number of stored pages. 802 */ 803 static void zswap_entry_free(struct zswap_entry *entry) 804 { 805 zswap_lru_del(&zswap_list_lru, entry); 806 zpool_free(entry->pool->zpool, entry->handle); 807 zswap_pool_put(entry->pool); 808 if (entry->objcg) { 809 obj_cgroup_uncharge_zswap(entry->objcg, entry->length); 810 obj_cgroup_put(entry->objcg); 811 } 812 zswap_entry_cache_free(entry); 813 atomic_long_dec(&zswap_stored_pages); 814 } 815 816 /********************************* 817 * compressed storage functions 818 **********************************/ 819 static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) 820 { 821 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 822 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 823 struct crypto_acomp *acomp = NULL; 824 struct acomp_req *req = NULL; 825 u8 *buffer = NULL; 826 int ret; 827 828 buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); 829 if (!buffer) { 830 ret = -ENOMEM; 831 goto fail; 832 } 833 834 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu)); 835 if (IS_ERR(acomp)) { 836 pr_err("could not alloc crypto acomp %s : %ld\n", 837 pool->tfm_name, PTR_ERR(acomp)); 838 ret = PTR_ERR(acomp); 839 goto fail; 840 } 841 842 req = acomp_request_alloc(acomp); 843 if (!req) { 844 pr_err("could not alloc crypto acomp_request %s\n", 845 pool->tfm_name); 846 ret = -ENOMEM; 847 goto fail; 848 } 849 850 /* 851 * Only hold the mutex after completing allocations, otherwise we may 852 * recurse into zswap through reclaim and attempt to hold the mutex 853 * again resulting in a deadlock. 854 */ 855 mutex_lock(&acomp_ctx->mutex); 856 crypto_init_wait(&acomp_ctx->wait); 857 858 /* 859 * if the backend of acomp is async zip, crypto_req_done() will wakeup 860 * crypto_wait_req(); if the backend of acomp is scomp, the callback 861 * won't be called, crypto_wait_req() will return without blocking. 862 */ 863 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 864 crypto_req_done, &acomp_ctx->wait); 865 866 acomp_ctx->buffer = buffer; 867 acomp_ctx->acomp = acomp; 868 acomp_ctx->is_sleepable = acomp_is_async(acomp); 869 acomp_ctx->req = req; 870 mutex_unlock(&acomp_ctx->mutex); 871 return 0; 872 873 fail: 874 if (acomp) 875 crypto_free_acomp(acomp); 876 kfree(buffer); 877 return ret; 878 } 879 880 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) 881 { 882 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); 883 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); 884 885 mutex_lock(&acomp_ctx->mutex); 886 if (!IS_ERR_OR_NULL(acomp_ctx)) { 887 if (!IS_ERR_OR_NULL(acomp_ctx->req)) 888 acomp_request_free(acomp_ctx->req); 889 acomp_ctx->req = NULL; 890 if (!IS_ERR_OR_NULL(acomp_ctx->acomp)) 891 crypto_free_acomp(acomp_ctx->acomp); 892 kfree(acomp_ctx->buffer); 893 } 894 mutex_unlock(&acomp_ctx->mutex); 895 896 return 0; 897 } 898 899 static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool) 900 { 901 struct crypto_acomp_ctx *acomp_ctx; 902 903 for (;;) { 904 acomp_ctx = raw_cpu_ptr(pool->acomp_ctx); 905 mutex_lock(&acomp_ctx->mutex); 906 if (likely(acomp_ctx->req)) 907 return acomp_ctx; 908 /* 909 * It is possible that we were migrated to a different CPU after 910 * getting the per-CPU ctx but before the mutex was acquired. If 911 * the old CPU got offlined, zswap_cpu_comp_dead() could have 912 * already freed ctx->req (among other things) and set it to 913 * NULL. Just try again on the new CPU that we ended up on. 914 */ 915 mutex_unlock(&acomp_ctx->mutex); 916 } 917 } 918 919 static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx) 920 { 921 mutex_unlock(&acomp_ctx->mutex); 922 } 923 924 static bool zswap_compress(struct page *page, struct zswap_entry *entry, 925 struct zswap_pool *pool) 926 { 927 struct crypto_acomp_ctx *acomp_ctx; 928 struct scatterlist input, output; 929 int comp_ret = 0, alloc_ret = 0; 930 unsigned int dlen = PAGE_SIZE; 931 unsigned long handle; 932 struct zpool *zpool; 933 char *buf; 934 gfp_t gfp; 935 u8 *dst; 936 937 acomp_ctx = acomp_ctx_get_cpu_lock(pool); 938 dst = acomp_ctx->buffer; 939 sg_init_table(&input, 1); 940 sg_set_page(&input, page, PAGE_SIZE, 0); 941 942 /* 943 * We need PAGE_SIZE * 2 here since there maybe over-compression case, 944 * and hardware-accelerators may won't check the dst buffer size, so 945 * giving the dst buffer with enough length to avoid buffer overflow. 946 */ 947 sg_init_one(&output, dst, PAGE_SIZE * 2); 948 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen); 949 950 /* 951 * it maybe looks a little bit silly that we send an asynchronous request, 952 * then wait for its completion synchronously. This makes the process look 953 * synchronous in fact. 954 * Theoretically, acomp supports users send multiple acomp requests in one 955 * acomp instance, then get those requests done simultaneously. but in this 956 * case, zswap actually does store and load page by page, there is no 957 * existing method to send the second page before the first page is done 958 * in one thread doing zwap. 959 * but in different threads running on different cpu, we have different 960 * acomp instance, so multiple threads can do (de)compression in parallel. 961 */ 962 comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait); 963 dlen = acomp_ctx->req->dlen; 964 if (comp_ret) 965 goto unlock; 966 967 zpool = pool->zpool; 968 gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 969 if (zpool_malloc_support_movable(zpool)) 970 gfp |= __GFP_HIGHMEM | __GFP_MOVABLE; 971 alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle); 972 if (alloc_ret) 973 goto unlock; 974 975 buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO); 976 memcpy(buf, dst, dlen); 977 zpool_unmap_handle(zpool, handle); 978 979 entry->handle = handle; 980 entry->length = dlen; 981 982 unlock: 983 if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC) 984 zswap_reject_compress_poor++; 985 else if (comp_ret) 986 zswap_reject_compress_fail++; 987 else if (alloc_ret) 988 zswap_reject_alloc_fail++; 989 990 acomp_ctx_put_unlock(acomp_ctx); 991 return comp_ret == 0 && alloc_ret == 0; 992 } 993 994 static void zswap_decompress(struct zswap_entry *entry, struct folio *folio) 995 { 996 struct zpool *zpool = entry->pool->zpool; 997 struct scatterlist input, output; 998 struct crypto_acomp_ctx *acomp_ctx; 999 u8 *src; 1000 1001 acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool); 1002 src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO); 1003 /* 1004 * If zpool_map_handle is atomic, we cannot reliably utilize its mapped buffer 1005 * to do crypto_acomp_decompress() which might sleep. In such cases, we must 1006 * resort to copying the buffer to a temporary one. 1007 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer, 1008 * such as a kmap address of high memory or even ever a vmap address. 1009 * However, sg_init_one is only equipped to handle linearly mapped low memory. 1010 * In such cases, we also must copy the buffer to a temporary and lowmem one. 1011 */ 1012 if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) || 1013 !virt_addr_valid(src)) { 1014 memcpy(acomp_ctx->buffer, src, entry->length); 1015 src = acomp_ctx->buffer; 1016 zpool_unmap_handle(zpool, entry->handle); 1017 } 1018 1019 sg_init_one(&input, src, entry->length); 1020 sg_init_table(&output, 1); 1021 sg_set_folio(&output, folio, PAGE_SIZE, 0); 1022 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE); 1023 BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait)); 1024 BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE); 1025 1026 if (src != acomp_ctx->buffer) 1027 zpool_unmap_handle(zpool, entry->handle); 1028 acomp_ctx_put_unlock(acomp_ctx); 1029 } 1030 1031 /********************************* 1032 * writeback code 1033 **********************************/ 1034 /* 1035 * Attempts to free an entry by adding a folio to the swap cache, 1036 * decompressing the entry data into the folio, and issuing a 1037 * bio write to write the folio back to the swap device. 1038 * 1039 * This can be thought of as a "resumed writeback" of the folio 1040 * to the swap device. We are basically resuming the same swap 1041 * writeback path that was intercepted with the zswap_store() 1042 * in the first place. After the folio has been decompressed into 1043 * the swap cache, the compressed version stored by zswap can be 1044 * freed. 1045 */ 1046 static int zswap_writeback_entry(struct zswap_entry *entry, 1047 swp_entry_t swpentry) 1048 { 1049 struct xarray *tree; 1050 pgoff_t offset = swp_offset(swpentry); 1051 struct folio *folio; 1052 struct mempolicy *mpol; 1053 bool folio_was_allocated; 1054 struct writeback_control wbc = { 1055 .sync_mode = WB_SYNC_NONE, 1056 }; 1057 1058 /* try to allocate swap cache folio */ 1059 mpol = get_task_policy(current); 1060 folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol, 1061 NO_INTERLEAVE_INDEX, &folio_was_allocated, true); 1062 if (!folio) 1063 return -ENOMEM; 1064 1065 /* 1066 * Found an existing folio, we raced with swapin or concurrent 1067 * shrinker. We generally writeback cold folios from zswap, and 1068 * swapin means the folio just became hot, so skip this folio. 1069 * For unlikely concurrent shrinker case, it will be unlinked 1070 * and freed when invalidated by the concurrent shrinker anyway. 1071 */ 1072 if (!folio_was_allocated) { 1073 folio_put(folio); 1074 return -EEXIST; 1075 } 1076 1077 /* 1078 * folio is locked, and the swapcache is now secured against 1079 * concurrent swapping to and from the slot, and concurrent 1080 * swapoff so we can safely dereference the zswap tree here. 1081 * Verify that the swap entry hasn't been invalidated and recycled 1082 * behind our backs, to avoid overwriting a new swap folio with 1083 * old compressed data. Only when this is successful can the entry 1084 * be dereferenced. 1085 */ 1086 tree = swap_zswap_tree(swpentry); 1087 if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) { 1088 delete_from_swap_cache(folio); 1089 folio_unlock(folio); 1090 folio_put(folio); 1091 return -ENOMEM; 1092 } 1093 1094 zswap_decompress(entry, folio); 1095 1096 count_vm_event(ZSWPWB); 1097 if (entry->objcg) 1098 count_objcg_events(entry->objcg, ZSWPWB, 1); 1099 1100 zswap_entry_free(entry); 1101 1102 /* folio is up to date */ 1103 folio_mark_uptodate(folio); 1104 1105 /* move it to the tail of the inactive list after end_writeback */ 1106 folio_set_reclaim(folio); 1107 1108 /* start writeback */ 1109 __swap_writepage(folio, &wbc); 1110 folio_put(folio); 1111 1112 return 0; 1113 } 1114 1115 /********************************* 1116 * shrinker functions 1117 **********************************/ 1118 /* 1119 * The dynamic shrinker is modulated by the following factors: 1120 * 1121 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving 1122 * the entry a second chance) before rotating it in the LRU list. If the 1123 * entry is considered again by the shrinker, with its referenced bit unset, 1124 * it is written back. The writeback rate as a result is dynamically 1125 * adjusted by the pool activities - if the pool is dominated by new entries 1126 * (i.e lots of recent zswapouts), these entries will be protected and 1127 * the writeback rate will slow down. On the other hand, if the pool has a 1128 * lot of stagnant entries, these entries will be reclaimed immediately, 1129 * effectively increasing the writeback rate. 1130 * 1131 * 2. Swapins counter: If we observe swapins, it is a sign that we are 1132 * overshrinking and should slow down. We maintain a swapins counter, which 1133 * is consumed and subtract from the number of eligible objects on the LRU 1134 * in zswap_shrinker_count(). 1135 * 1136 * 3. Compression ratio. The better the workload compresses, the less gains we 1137 * can expect from writeback. We scale down the number of objects available 1138 * for reclaim by this ratio. 1139 */ 1140 static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l, 1141 void *arg) 1142 { 1143 struct zswap_entry *entry = container_of(item, struct zswap_entry, lru); 1144 bool *encountered_page_in_swapcache = (bool *)arg; 1145 swp_entry_t swpentry; 1146 enum lru_status ret = LRU_REMOVED_RETRY; 1147 int writeback_result; 1148 1149 /* 1150 * Second chance algorithm: if the entry has its referenced bit set, give it 1151 * a second chance. Only clear the referenced bit and rotate it in the 1152 * zswap's LRU list. 1153 */ 1154 if (entry->referenced) { 1155 entry->referenced = false; 1156 return LRU_ROTATE; 1157 } 1158 1159 /* 1160 * As soon as we drop the LRU lock, the entry can be freed by 1161 * a concurrent invalidation. This means the following: 1162 * 1163 * 1. We extract the swp_entry_t to the stack, allowing 1164 * zswap_writeback_entry() to pin the swap entry and 1165 * then validate the zwap entry against that swap entry's 1166 * tree using pointer value comparison. Only when that 1167 * is successful can the entry be dereferenced. 1168 * 1169 * 2. Usually, objects are taken off the LRU for reclaim. In 1170 * this case this isn't possible, because if reclaim fails 1171 * for whatever reason, we have no means of knowing if the 1172 * entry is alive to put it back on the LRU. 1173 * 1174 * So rotate it before dropping the lock. If the entry is 1175 * written back or invalidated, the free path will unlink 1176 * it. For failures, rotation is the right thing as well. 1177 * 1178 * Temporary failures, where the same entry should be tried 1179 * again immediately, almost never happen for this shrinker. 1180 * We don't do any trylocking; -ENOMEM comes closest, 1181 * but that's extremely rare and doesn't happen spuriously 1182 * either. Don't bother distinguishing this case. 1183 */ 1184 list_move_tail(item, &l->list); 1185 1186 /* 1187 * Once the lru lock is dropped, the entry might get freed. The 1188 * swpentry is copied to the stack, and entry isn't deref'd again 1189 * until the entry is verified to still be alive in the tree. 1190 */ 1191 swpentry = entry->swpentry; 1192 1193 /* 1194 * It's safe to drop the lock here because we return either 1195 * LRU_REMOVED_RETRY, LRU_RETRY or LRU_STOP. 1196 */ 1197 spin_unlock(&l->lock); 1198 1199 writeback_result = zswap_writeback_entry(entry, swpentry); 1200 1201 if (writeback_result) { 1202 zswap_reject_reclaim_fail++; 1203 ret = LRU_RETRY; 1204 1205 /* 1206 * Encountering a page already in swap cache is a sign that we are shrinking 1207 * into the warmer region. We should terminate shrinking (if we're in the dynamic 1208 * shrinker context). 1209 */ 1210 if (writeback_result == -EEXIST && encountered_page_in_swapcache) { 1211 ret = LRU_STOP; 1212 *encountered_page_in_swapcache = true; 1213 } 1214 } else { 1215 zswap_written_back_pages++; 1216 } 1217 1218 return ret; 1219 } 1220 1221 static unsigned long zswap_shrinker_scan(struct shrinker *shrinker, 1222 struct shrink_control *sc) 1223 { 1224 unsigned long shrink_ret; 1225 bool encountered_page_in_swapcache = false; 1226 1227 if (!zswap_shrinker_enabled || 1228 !mem_cgroup_zswap_writeback_enabled(sc->memcg)) { 1229 sc->nr_scanned = 0; 1230 return SHRINK_STOP; 1231 } 1232 1233 shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb, 1234 &encountered_page_in_swapcache); 1235 1236 if (encountered_page_in_swapcache) 1237 return SHRINK_STOP; 1238 1239 return shrink_ret ? shrink_ret : SHRINK_STOP; 1240 } 1241 1242 static unsigned long zswap_shrinker_count(struct shrinker *shrinker, 1243 struct shrink_control *sc) 1244 { 1245 struct mem_cgroup *memcg = sc->memcg; 1246 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid)); 1247 atomic_long_t *nr_disk_swapins = 1248 &lruvec->zswap_lruvec_state.nr_disk_swapins; 1249 unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur, 1250 nr_remain; 1251 1252 if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg)) 1253 return 0; 1254 1255 /* 1256 * The shrinker resumes swap writeback, which will enter block 1257 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS 1258 * rules (may_enter_fs()), which apply on a per-folio basis. 1259 */ 1260 if (!gfp_has_io_fs(sc->gfp_mask)) 1261 return 0; 1262 1263 /* 1264 * For memcg, use the cgroup-wide ZSWAP stats since we don't 1265 * have them per-node and thus per-lruvec. Careful if memcg is 1266 * runtime-disabled: we can get sc->memcg == NULL, which is ok 1267 * for the lruvec, but not for memcg_page_state(). 1268 * 1269 * Without memcg, use the zswap pool-wide metrics. 1270 */ 1271 if (!mem_cgroup_disabled()) { 1272 mem_cgroup_flush_stats(memcg); 1273 nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT; 1274 nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED); 1275 } else { 1276 nr_backing = zswap_total_pages(); 1277 nr_stored = atomic_long_read(&zswap_stored_pages); 1278 } 1279 1280 if (!nr_stored) 1281 return 0; 1282 1283 nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc); 1284 if (!nr_freeable) 1285 return 0; 1286 1287 /* 1288 * Subtract from the lru size the number of pages that are recently swapped 1289 * in from disk. The idea is that had we protect the zswap's LRU by this 1290 * amount of pages, these disk swapins would not have happened. 1291 */ 1292 nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins); 1293 do { 1294 if (nr_freeable >= nr_disk_swapins_cur) 1295 nr_remain = 0; 1296 else 1297 nr_remain = nr_disk_swapins_cur - nr_freeable; 1298 } while (!atomic_long_try_cmpxchg( 1299 nr_disk_swapins, &nr_disk_swapins_cur, nr_remain)); 1300 1301 nr_freeable -= nr_disk_swapins_cur - nr_remain; 1302 if (!nr_freeable) 1303 return 0; 1304 1305 /* 1306 * Scale the number of freeable pages by the memory saving factor. 1307 * This ensures that the better zswap compresses memory, the fewer 1308 * pages we will evict to swap (as it will otherwise incur IO for 1309 * relatively small memory saving). 1310 */ 1311 return mult_frac(nr_freeable, nr_backing, nr_stored); 1312 } 1313 1314 static struct shrinker *zswap_alloc_shrinker(void) 1315 { 1316 struct shrinker *shrinker; 1317 1318 shrinker = 1319 shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap"); 1320 if (!shrinker) 1321 return NULL; 1322 1323 shrinker->scan_objects = zswap_shrinker_scan; 1324 shrinker->count_objects = zswap_shrinker_count; 1325 shrinker->batch = 0; 1326 shrinker->seeks = DEFAULT_SEEKS; 1327 return shrinker; 1328 } 1329 1330 static int shrink_memcg(struct mem_cgroup *memcg) 1331 { 1332 int nid, shrunk = 0, scanned = 0; 1333 1334 if (!mem_cgroup_zswap_writeback_enabled(memcg)) 1335 return -ENOENT; 1336 1337 /* 1338 * Skip zombies because their LRUs are reparented and we would be 1339 * reclaiming from the parent instead of the dead memcg. 1340 */ 1341 if (memcg && !mem_cgroup_online(memcg)) 1342 return -ENOENT; 1343 1344 for_each_node_state(nid, N_NORMAL_MEMORY) { 1345 unsigned long nr_to_walk = 1; 1346 1347 shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg, 1348 &shrink_memcg_cb, NULL, &nr_to_walk); 1349 scanned += 1 - nr_to_walk; 1350 } 1351 1352 if (!scanned) 1353 return -ENOENT; 1354 1355 return shrunk ? 0 : -EAGAIN; 1356 } 1357 1358 static void shrink_worker(struct work_struct *w) 1359 { 1360 struct mem_cgroup *memcg; 1361 int ret, failures = 0, attempts = 0; 1362 unsigned long thr; 1363 1364 /* Reclaim down to the accept threshold */ 1365 thr = zswap_accept_thr_pages(); 1366 1367 /* 1368 * Global reclaim will select cgroup in a round-robin fashion from all 1369 * online memcgs, but memcgs that have no pages in zswap and 1370 * writeback-disabled memcgs (memory.zswap.writeback=0) are not 1371 * candidates for shrinking. 1372 * 1373 * Shrinking will be aborted if we encounter the following 1374 * MAX_RECLAIM_RETRIES times: 1375 * - No writeback-candidate memcgs found in a memcg tree walk. 1376 * - Shrinking a writeback-candidate memcg failed. 1377 * 1378 * We save iteration cursor memcg into zswap_next_shrink, 1379 * which can be modified by the offline memcg cleaner 1380 * zswap_memcg_offline_cleanup(). 1381 * 1382 * Since the offline cleaner is called only once, we cannot leave an 1383 * offline memcg reference in zswap_next_shrink. 1384 * We can rely on the cleaner only if we get online memcg under lock. 1385 * 1386 * If we get an offline memcg, we cannot determine if the cleaner has 1387 * already been called or will be called later. We must put back the 1388 * reference before returning from this function. Otherwise, the 1389 * offline memcg left in zswap_next_shrink will hold the reference 1390 * until the next run of shrink_worker(). 1391 */ 1392 do { 1393 /* 1394 * Start shrinking from the next memcg after zswap_next_shrink. 1395 * When the offline cleaner has already advanced the cursor, 1396 * advancing the cursor here overlooks one memcg, but this 1397 * should be negligibly rare. 1398 * 1399 * If we get an online memcg, keep the extra reference in case 1400 * the original one obtained by mem_cgroup_iter() is dropped by 1401 * zswap_memcg_offline_cleanup() while we are shrinking the 1402 * memcg. 1403 */ 1404 spin_lock(&zswap_shrink_lock); 1405 do { 1406 memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL); 1407 zswap_next_shrink = memcg; 1408 } while (memcg && !mem_cgroup_tryget_online(memcg)); 1409 spin_unlock(&zswap_shrink_lock); 1410 1411 if (!memcg) { 1412 /* 1413 * Continue shrinking without incrementing failures if 1414 * we found candidate memcgs in the last tree walk. 1415 */ 1416 if (!attempts && ++failures == MAX_RECLAIM_RETRIES) 1417 break; 1418 1419 attempts = 0; 1420 goto resched; 1421 } 1422 1423 ret = shrink_memcg(memcg); 1424 /* drop the extra reference */ 1425 mem_cgroup_put(memcg); 1426 1427 /* 1428 * There are no writeback-candidate pages in the memcg. 1429 * This is not an issue as long as we can find another memcg 1430 * with pages in zswap. Skip this without incrementing attempts 1431 * and failures. 1432 */ 1433 if (ret == -ENOENT) 1434 continue; 1435 ++attempts; 1436 1437 if (ret && ++failures == MAX_RECLAIM_RETRIES) 1438 break; 1439 resched: 1440 cond_resched(); 1441 } while (zswap_total_pages() > thr); 1442 } 1443 1444 /********************************* 1445 * main API 1446 **********************************/ 1447 1448 static bool zswap_store_page(struct page *page, 1449 struct obj_cgroup *objcg, 1450 struct zswap_pool *pool) 1451 { 1452 swp_entry_t page_swpentry = page_swap_entry(page); 1453 struct zswap_entry *entry, *old; 1454 1455 /* allocate entry */ 1456 entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page)); 1457 if (!entry) { 1458 zswap_reject_kmemcache_fail++; 1459 return false; 1460 } 1461 1462 if (!zswap_compress(page, entry, pool)) 1463 goto compress_failed; 1464 1465 old = xa_store(swap_zswap_tree(page_swpentry), 1466 swp_offset(page_swpentry), 1467 entry, GFP_KERNEL); 1468 if (xa_is_err(old)) { 1469 int err = xa_err(old); 1470 1471 WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err); 1472 zswap_reject_alloc_fail++; 1473 goto store_failed; 1474 } 1475 1476 /* 1477 * We may have had an existing entry that became stale when 1478 * the folio was redirtied and now the new version is being 1479 * swapped out. Get rid of the old. 1480 */ 1481 if (old) 1482 zswap_entry_free(old); 1483 1484 /* 1485 * The entry is successfully compressed and stored in the tree, there is 1486 * no further possibility of failure. Grab refs to the pool and objcg, 1487 * charge zswap memory, and increment zswap_stored_pages. 1488 * The opposite actions will be performed by zswap_entry_free() 1489 * when the entry is removed from the tree. 1490 */ 1491 zswap_pool_get(pool); 1492 if (objcg) { 1493 obj_cgroup_get(objcg); 1494 obj_cgroup_charge_zswap(objcg, entry->length); 1495 } 1496 atomic_long_inc(&zswap_stored_pages); 1497 1498 /* 1499 * We finish initializing the entry while it's already in xarray. 1500 * This is safe because: 1501 * 1502 * 1. Concurrent stores and invalidations are excluded by folio lock. 1503 * 1504 * 2. Writeback is excluded by the entry not being on the LRU yet. 1505 * The publishing order matters to prevent writeback from seeing 1506 * an incoherent entry. 1507 */ 1508 entry->pool = pool; 1509 entry->swpentry = page_swpentry; 1510 entry->objcg = objcg; 1511 entry->referenced = true; 1512 if (entry->length) { 1513 INIT_LIST_HEAD(&entry->lru); 1514 zswap_lru_add(&zswap_list_lru, entry); 1515 } 1516 1517 return true; 1518 1519 store_failed: 1520 zpool_free(pool->zpool, entry->handle); 1521 compress_failed: 1522 zswap_entry_cache_free(entry); 1523 return false; 1524 } 1525 1526 bool zswap_store(struct folio *folio) 1527 { 1528 long nr_pages = folio_nr_pages(folio); 1529 swp_entry_t swp = folio->swap; 1530 struct obj_cgroup *objcg = NULL; 1531 struct mem_cgroup *memcg = NULL; 1532 struct zswap_pool *pool; 1533 bool ret = false; 1534 long index; 1535 1536 VM_WARN_ON_ONCE(!folio_test_locked(folio)); 1537 VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); 1538 1539 if (!zswap_enabled) 1540 goto check_old; 1541 1542 objcg = get_obj_cgroup_from_folio(folio); 1543 if (objcg && !obj_cgroup_may_zswap(objcg)) { 1544 memcg = get_mem_cgroup_from_objcg(objcg); 1545 if (shrink_memcg(memcg)) { 1546 mem_cgroup_put(memcg); 1547 goto put_objcg; 1548 } 1549 mem_cgroup_put(memcg); 1550 } 1551 1552 if (zswap_check_limits()) 1553 goto put_objcg; 1554 1555 pool = zswap_pool_current_get(); 1556 if (!pool) 1557 goto put_objcg; 1558 1559 if (objcg) { 1560 memcg = get_mem_cgroup_from_objcg(objcg); 1561 if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) { 1562 mem_cgroup_put(memcg); 1563 goto put_pool; 1564 } 1565 mem_cgroup_put(memcg); 1566 } 1567 1568 for (index = 0; index < nr_pages; ++index) { 1569 struct page *page = folio_page(folio, index); 1570 1571 if (!zswap_store_page(page, objcg, pool)) 1572 goto put_pool; 1573 } 1574 1575 if (objcg) 1576 count_objcg_events(objcg, ZSWPOUT, nr_pages); 1577 1578 count_vm_events(ZSWPOUT, nr_pages); 1579 1580 ret = true; 1581 1582 put_pool: 1583 zswap_pool_put(pool); 1584 put_objcg: 1585 obj_cgroup_put(objcg); 1586 if (!ret && zswap_pool_reached_full) 1587 queue_work(shrink_wq, &zswap_shrink_work); 1588 check_old: 1589 /* 1590 * If the zswap store fails or zswap is disabled, we must invalidate 1591 * the possibly stale entries which were previously stored at the 1592 * offsets corresponding to each page of the folio. Otherwise, 1593 * writeback could overwrite the new data in the swapfile. 1594 */ 1595 if (!ret) { 1596 unsigned type = swp_type(swp); 1597 pgoff_t offset = swp_offset(swp); 1598 struct zswap_entry *entry; 1599 struct xarray *tree; 1600 1601 for (index = 0; index < nr_pages; ++index) { 1602 tree = swap_zswap_tree(swp_entry(type, offset + index)); 1603 entry = xa_erase(tree, offset + index); 1604 if (entry) 1605 zswap_entry_free(entry); 1606 } 1607 } 1608 1609 return ret; 1610 } 1611 1612 bool zswap_load(struct folio *folio) 1613 { 1614 swp_entry_t swp = folio->swap; 1615 pgoff_t offset = swp_offset(swp); 1616 bool swapcache = folio_test_swapcache(folio); 1617 struct xarray *tree = swap_zswap_tree(swp); 1618 struct zswap_entry *entry; 1619 1620 VM_WARN_ON_ONCE(!folio_test_locked(folio)); 1621 1622 if (zswap_never_enabled()) 1623 return false; 1624 1625 /* 1626 * Large folios should not be swapped in while zswap is being used, as 1627 * they are not properly handled. Zswap does not properly load large 1628 * folios, and a large folio may only be partially in zswap. 1629 * 1630 * Return true without marking the folio uptodate so that an IO error is 1631 * emitted (e.g. do_swap_page() will sigbus). 1632 */ 1633 if (WARN_ON_ONCE(folio_test_large(folio))) 1634 return true; 1635 1636 /* 1637 * When reading into the swapcache, invalidate our entry. The 1638 * swapcache can be the authoritative owner of the page and 1639 * its mappings, and the pressure that results from having two 1640 * in-memory copies outweighs any benefits of caching the 1641 * compression work. 1642 * 1643 * (Most swapins go through the swapcache. The notable 1644 * exception is the singleton fault on SWP_SYNCHRONOUS_IO 1645 * files, which reads into a private page and may free it if 1646 * the fault fails. We remain the primary owner of the entry.) 1647 */ 1648 if (swapcache) 1649 entry = xa_erase(tree, offset); 1650 else 1651 entry = xa_load(tree, offset); 1652 1653 if (!entry) 1654 return false; 1655 1656 zswap_decompress(entry, folio); 1657 1658 count_vm_event(ZSWPIN); 1659 if (entry->objcg) 1660 count_objcg_events(entry->objcg, ZSWPIN, 1); 1661 1662 if (swapcache) { 1663 zswap_entry_free(entry); 1664 folio_mark_dirty(folio); 1665 } 1666 1667 folio_mark_uptodate(folio); 1668 return true; 1669 } 1670 1671 void zswap_invalidate(swp_entry_t swp) 1672 { 1673 pgoff_t offset = swp_offset(swp); 1674 struct xarray *tree = swap_zswap_tree(swp); 1675 struct zswap_entry *entry; 1676 1677 if (xa_empty(tree)) 1678 return; 1679 1680 entry = xa_erase(tree, offset); 1681 if (entry) 1682 zswap_entry_free(entry); 1683 } 1684 1685 int zswap_swapon(int type, unsigned long nr_pages) 1686 { 1687 struct xarray *trees, *tree; 1688 unsigned int nr, i; 1689 1690 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); 1691 trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL); 1692 if (!trees) { 1693 pr_err("alloc failed, zswap disabled for swap type %d\n", type); 1694 return -ENOMEM; 1695 } 1696 1697 for (i = 0; i < nr; i++) 1698 xa_init(trees + i); 1699 1700 nr_zswap_trees[type] = nr; 1701 zswap_trees[type] = trees; 1702 return 0; 1703 } 1704 1705 void zswap_swapoff(int type) 1706 { 1707 struct xarray *trees = zswap_trees[type]; 1708 unsigned int i; 1709 1710 if (!trees) 1711 return; 1712 1713 /* try_to_unuse() invalidated all the entries already */ 1714 for (i = 0; i < nr_zswap_trees[type]; i++) 1715 WARN_ON_ONCE(!xa_empty(trees + i)); 1716 1717 kvfree(trees); 1718 nr_zswap_trees[type] = 0; 1719 zswap_trees[type] = NULL; 1720 } 1721 1722 /********************************* 1723 * debugfs functions 1724 **********************************/ 1725 #ifdef CONFIG_DEBUG_FS 1726 #include <linux/debugfs.h> 1727 1728 static struct dentry *zswap_debugfs_root; 1729 1730 static int debugfs_get_total_size(void *data, u64 *val) 1731 { 1732 *val = zswap_total_pages() * PAGE_SIZE; 1733 return 0; 1734 } 1735 DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n"); 1736 1737 static int debugfs_get_stored_pages(void *data, u64 *val) 1738 { 1739 *val = atomic_long_read(&zswap_stored_pages); 1740 return 0; 1741 } 1742 DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n"); 1743 1744 static int zswap_debugfs_init(void) 1745 { 1746 if (!debugfs_initialized()) 1747 return -ENODEV; 1748 1749 zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 1750 1751 debugfs_create_u64("pool_limit_hit", 0444, 1752 zswap_debugfs_root, &zswap_pool_limit_hit); 1753 debugfs_create_u64("reject_reclaim_fail", 0444, 1754 zswap_debugfs_root, &zswap_reject_reclaim_fail); 1755 debugfs_create_u64("reject_alloc_fail", 0444, 1756 zswap_debugfs_root, &zswap_reject_alloc_fail); 1757 debugfs_create_u64("reject_kmemcache_fail", 0444, 1758 zswap_debugfs_root, &zswap_reject_kmemcache_fail); 1759 debugfs_create_u64("reject_compress_fail", 0444, 1760 zswap_debugfs_root, &zswap_reject_compress_fail); 1761 debugfs_create_u64("reject_compress_poor", 0444, 1762 zswap_debugfs_root, &zswap_reject_compress_poor); 1763 debugfs_create_u64("written_back_pages", 0444, 1764 zswap_debugfs_root, &zswap_written_back_pages); 1765 debugfs_create_file("pool_total_size", 0444, 1766 zswap_debugfs_root, NULL, &total_size_fops); 1767 debugfs_create_file("stored_pages", 0444, 1768 zswap_debugfs_root, NULL, &stored_pages_fops); 1769 1770 return 0; 1771 } 1772 #else 1773 static int zswap_debugfs_init(void) 1774 { 1775 return 0; 1776 } 1777 #endif 1778 1779 /********************************* 1780 * module init and exit 1781 **********************************/ 1782 static int zswap_setup(void) 1783 { 1784 struct zswap_pool *pool; 1785 int ret; 1786 1787 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 1788 if (!zswap_entry_cache) { 1789 pr_err("entry cache creation failed\n"); 1790 goto cache_fail; 1791 } 1792 1793 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, 1794 "mm/zswap_pool:prepare", 1795 zswap_cpu_comp_prepare, 1796 zswap_cpu_comp_dead); 1797 if (ret) 1798 goto hp_fail; 1799 1800 shrink_wq = alloc_workqueue("zswap-shrink", 1801 WQ_UNBOUND|WQ_MEM_RECLAIM, 1); 1802 if (!shrink_wq) 1803 goto shrink_wq_fail; 1804 1805 zswap_shrinker = zswap_alloc_shrinker(); 1806 if (!zswap_shrinker) 1807 goto shrinker_fail; 1808 if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker)) 1809 goto lru_fail; 1810 shrinker_register(zswap_shrinker); 1811 1812 INIT_WORK(&zswap_shrink_work, shrink_worker); 1813 1814 pool = __zswap_pool_create_fallback(); 1815 if (pool) { 1816 pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1817 zpool_get_type(pool->zpool)); 1818 list_add(&pool->list, &zswap_pools); 1819 zswap_has_pool = true; 1820 static_branch_enable(&zswap_ever_enabled); 1821 } else { 1822 pr_err("pool creation failed\n"); 1823 zswap_enabled = false; 1824 } 1825 1826 if (zswap_debugfs_init()) 1827 pr_warn("debugfs initialization failed\n"); 1828 zswap_init_state = ZSWAP_INIT_SUCCEED; 1829 return 0; 1830 1831 lru_fail: 1832 shrinker_free(zswap_shrinker); 1833 shrinker_fail: 1834 destroy_workqueue(shrink_wq); 1835 shrink_wq_fail: 1836 cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE); 1837 hp_fail: 1838 kmem_cache_destroy(zswap_entry_cache); 1839 cache_fail: 1840 /* if built-in, we aren't unloaded on failure; don't allow use */ 1841 zswap_init_state = ZSWAP_INIT_FAILED; 1842 zswap_enabled = false; 1843 return -ENOMEM; 1844 } 1845 1846 static int __init zswap_init(void) 1847 { 1848 if (!zswap_enabled) 1849 return 0; 1850 return zswap_setup(); 1851 } 1852 /* must be late so crypto has time to come up */ 1853 late_initcall(zswap_init); 1854 1855 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 1856 MODULE_DESCRIPTION("Compressed cache for swap pages"); 1857