1 /* 2 * zswap.c - zswap driver file 3 * 4 * zswap is a backend for frontswap that takes pages that are in the process 5 * of being swapped out and attempts to compress and store them in a 6 * RAM-based memory pool. This can result in a significant I/O reduction on 7 * the swap device and, in the case where decompressing from RAM is faster 8 * than reading from the swap device, can also improve workload performance. 9 * 10 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 2 15 * of the License, or (at your option) any later version. 16 * 17 * This program is distributed in the hope that it will be useful, 18 * but WITHOUT ANY WARRANTY; without even the implied warranty of 19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 * GNU General Public License for more details. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/module.h> 26 #include <linux/cpu.h> 27 #include <linux/highmem.h> 28 #include <linux/slab.h> 29 #include <linux/spinlock.h> 30 #include <linux/types.h> 31 #include <linux/atomic.h> 32 #include <linux/frontswap.h> 33 #include <linux/rbtree.h> 34 #include <linux/swap.h> 35 #include <linux/crypto.h> 36 #include <linux/mempool.h> 37 #include <linux/zpool.h> 38 39 #include <linux/mm_types.h> 40 #include <linux/page-flags.h> 41 #include <linux/swapops.h> 42 #include <linux/writeback.h> 43 #include <linux/pagemap.h> 44 45 /********************************* 46 * statistics 47 **********************************/ 48 /* Total bytes used by the compressed storage */ 49 static u64 zswap_pool_total_size; 50 /* The number of compressed pages currently stored in zswap */ 51 static atomic_t zswap_stored_pages = ATOMIC_INIT(0); 52 53 /* 54 * The statistics below are not protected from concurrent access for 55 * performance reasons so they may not be a 100% accurate. However, 56 * they do provide useful information on roughly how many times a 57 * certain event is occurring. 58 */ 59 60 /* Pool limit was hit (see zswap_max_pool_percent) */ 61 static u64 zswap_pool_limit_hit; 62 /* Pages written back when pool limit was reached */ 63 static u64 zswap_written_back_pages; 64 /* Store failed due to a reclaim failure after pool limit was reached */ 65 static u64 zswap_reject_reclaim_fail; 66 /* Compressed page was too big for the allocator to (optimally) store */ 67 static u64 zswap_reject_compress_poor; 68 /* Store failed because underlying allocator could not get memory */ 69 static u64 zswap_reject_alloc_fail; 70 /* Store failed because the entry metadata could not be allocated (rare) */ 71 static u64 zswap_reject_kmemcache_fail; 72 /* Duplicate store was encountered (rare) */ 73 static u64 zswap_duplicate_entry; 74 75 /********************************* 76 * tunables 77 **********************************/ 78 79 /* Enable/disable zswap (disabled by default) */ 80 static bool zswap_enabled; 81 module_param_named(enabled, zswap_enabled, bool, 0644); 82 83 /* Crypto compressor to use */ 84 #define ZSWAP_COMPRESSOR_DEFAULT "lzo" 85 static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; 86 static int zswap_compressor_param_set(const char *, 87 const struct kernel_param *); 88 static struct kernel_param_ops zswap_compressor_param_ops = { 89 .set = zswap_compressor_param_set, 90 .get = param_get_charp, 91 .free = param_free_charp, 92 }; 93 module_param_cb(compressor, &zswap_compressor_param_ops, 94 &zswap_compressor, 0644); 95 96 /* Compressed storage zpool to use */ 97 #define ZSWAP_ZPOOL_DEFAULT "zbud" 98 static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; 99 static int zswap_zpool_param_set(const char *, const struct kernel_param *); 100 static struct kernel_param_ops zswap_zpool_param_ops = { 101 .set = zswap_zpool_param_set, 102 .get = param_get_charp, 103 .free = param_free_charp, 104 }; 105 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); 106 107 /* The maximum percentage of memory that the compressed pool can occupy */ 108 static unsigned int zswap_max_pool_percent = 20; 109 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); 110 111 /********************************* 112 * data structures 113 **********************************/ 114 115 struct zswap_pool { 116 struct zpool *zpool; 117 struct crypto_comp * __percpu *tfm; 118 struct kref kref; 119 struct list_head list; 120 struct rcu_head rcu_head; 121 struct notifier_block notifier; 122 char tfm_name[CRYPTO_MAX_ALG_NAME]; 123 }; 124 125 /* 126 * struct zswap_entry 127 * 128 * This structure contains the metadata for tracking a single compressed 129 * page within zswap. 130 * 131 * rbnode - links the entry into red-black tree for the appropriate swap type 132 * offset - the swap offset for the entry. Index into the red-black tree. 133 * refcount - the number of outstanding reference to the entry. This is needed 134 * to protect against premature freeing of the entry by code 135 * concurrent calls to load, invalidate, and writeback. The lock 136 * for the zswap_tree structure that contains the entry must 137 * be held while changing the refcount. Since the lock must 138 * be held, there is no reason to also make refcount atomic. 139 * length - the length in bytes of the compressed page data. Needed during 140 * decompression 141 * pool - the zswap_pool the entry's data is in 142 * handle - zpool allocation handle that stores the compressed page data 143 */ 144 struct zswap_entry { 145 struct rb_node rbnode; 146 pgoff_t offset; 147 int refcount; 148 unsigned int length; 149 struct zswap_pool *pool; 150 unsigned long handle; 151 }; 152 153 struct zswap_header { 154 swp_entry_t swpentry; 155 }; 156 157 /* 158 * The tree lock in the zswap_tree struct protects a few things: 159 * - the rbtree 160 * - the refcount field of each entry in the tree 161 */ 162 struct zswap_tree { 163 struct rb_root rbroot; 164 spinlock_t lock; 165 }; 166 167 static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; 168 169 /* RCU-protected iteration */ 170 static LIST_HEAD(zswap_pools); 171 /* protects zswap_pools list modification */ 172 static DEFINE_SPINLOCK(zswap_pools_lock); 173 /* pool counter to provide unique names to zpool */ 174 static atomic_t zswap_pools_count = ATOMIC_INIT(0); 175 176 /* used by param callback function */ 177 static bool zswap_init_started; 178 179 /********************************* 180 * helpers and fwd declarations 181 **********************************/ 182 183 #define zswap_pool_debug(msg, p) \ 184 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ 185 zpool_get_type((p)->zpool)) 186 187 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle); 188 static int zswap_pool_get(struct zswap_pool *pool); 189 static void zswap_pool_put(struct zswap_pool *pool); 190 191 static const struct zpool_ops zswap_zpool_ops = { 192 .evict = zswap_writeback_entry 193 }; 194 195 static bool zswap_is_full(void) 196 { 197 return totalram_pages * zswap_max_pool_percent / 100 < 198 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); 199 } 200 201 static void zswap_update_total_size(void) 202 { 203 struct zswap_pool *pool; 204 u64 total = 0; 205 206 rcu_read_lock(); 207 208 list_for_each_entry_rcu(pool, &zswap_pools, list) 209 total += zpool_get_total_size(pool->zpool); 210 211 rcu_read_unlock(); 212 213 zswap_pool_total_size = total; 214 } 215 216 /********************************* 217 * zswap entry functions 218 **********************************/ 219 static struct kmem_cache *zswap_entry_cache; 220 221 static int __init zswap_entry_cache_create(void) 222 { 223 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); 224 return zswap_entry_cache == NULL; 225 } 226 227 static void __init zswap_entry_cache_destroy(void) 228 { 229 kmem_cache_destroy(zswap_entry_cache); 230 } 231 232 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) 233 { 234 struct zswap_entry *entry; 235 entry = kmem_cache_alloc(zswap_entry_cache, gfp); 236 if (!entry) 237 return NULL; 238 entry->refcount = 1; 239 RB_CLEAR_NODE(&entry->rbnode); 240 return entry; 241 } 242 243 static void zswap_entry_cache_free(struct zswap_entry *entry) 244 { 245 kmem_cache_free(zswap_entry_cache, entry); 246 } 247 248 /********************************* 249 * rbtree functions 250 **********************************/ 251 static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) 252 { 253 struct rb_node *node = root->rb_node; 254 struct zswap_entry *entry; 255 256 while (node) { 257 entry = rb_entry(node, struct zswap_entry, rbnode); 258 if (entry->offset > offset) 259 node = node->rb_left; 260 else if (entry->offset < offset) 261 node = node->rb_right; 262 else 263 return entry; 264 } 265 return NULL; 266 } 267 268 /* 269 * In the case that a entry with the same offset is found, a pointer to 270 * the existing entry is stored in dupentry and the function returns -EEXIST 271 */ 272 static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, 273 struct zswap_entry **dupentry) 274 { 275 struct rb_node **link = &root->rb_node, *parent = NULL; 276 struct zswap_entry *myentry; 277 278 while (*link) { 279 parent = *link; 280 myentry = rb_entry(parent, struct zswap_entry, rbnode); 281 if (myentry->offset > entry->offset) 282 link = &(*link)->rb_left; 283 else if (myentry->offset < entry->offset) 284 link = &(*link)->rb_right; 285 else { 286 *dupentry = myentry; 287 return -EEXIST; 288 } 289 } 290 rb_link_node(&entry->rbnode, parent, link); 291 rb_insert_color(&entry->rbnode, root); 292 return 0; 293 } 294 295 static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) 296 { 297 if (!RB_EMPTY_NODE(&entry->rbnode)) { 298 rb_erase(&entry->rbnode, root); 299 RB_CLEAR_NODE(&entry->rbnode); 300 } 301 } 302 303 /* 304 * Carries out the common pattern of freeing and entry's zpool allocation, 305 * freeing the entry itself, and decrementing the number of stored pages. 306 */ 307 static void zswap_free_entry(struct zswap_entry *entry) 308 { 309 zpool_free(entry->pool->zpool, entry->handle); 310 zswap_pool_put(entry->pool); 311 zswap_entry_cache_free(entry); 312 atomic_dec(&zswap_stored_pages); 313 zswap_update_total_size(); 314 } 315 316 /* caller must hold the tree lock */ 317 static void zswap_entry_get(struct zswap_entry *entry) 318 { 319 entry->refcount++; 320 } 321 322 /* caller must hold the tree lock 323 * remove from the tree and free it, if nobody reference the entry 324 */ 325 static void zswap_entry_put(struct zswap_tree *tree, 326 struct zswap_entry *entry) 327 { 328 int refcount = --entry->refcount; 329 330 BUG_ON(refcount < 0); 331 if (refcount == 0) { 332 zswap_rb_erase(&tree->rbroot, entry); 333 zswap_free_entry(entry); 334 } 335 } 336 337 /* caller must hold the tree lock */ 338 static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, 339 pgoff_t offset) 340 { 341 struct zswap_entry *entry; 342 343 entry = zswap_rb_search(root, offset); 344 if (entry) 345 zswap_entry_get(entry); 346 347 return entry; 348 } 349 350 /********************************* 351 * per-cpu code 352 **********************************/ 353 static DEFINE_PER_CPU(u8 *, zswap_dstmem); 354 355 static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu) 356 { 357 u8 *dst; 358 359 switch (action) { 360 case CPU_UP_PREPARE: 361 dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); 362 if (!dst) { 363 pr_err("can't allocate compressor buffer\n"); 364 return NOTIFY_BAD; 365 } 366 per_cpu(zswap_dstmem, cpu) = dst; 367 break; 368 case CPU_DEAD: 369 case CPU_UP_CANCELED: 370 dst = per_cpu(zswap_dstmem, cpu); 371 kfree(dst); 372 per_cpu(zswap_dstmem, cpu) = NULL; 373 break; 374 default: 375 break; 376 } 377 return NOTIFY_OK; 378 } 379 380 static int zswap_cpu_dstmem_notifier(struct notifier_block *nb, 381 unsigned long action, void *pcpu) 382 { 383 return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu); 384 } 385 386 static struct notifier_block zswap_dstmem_notifier = { 387 .notifier_call = zswap_cpu_dstmem_notifier, 388 }; 389 390 static int __init zswap_cpu_dstmem_init(void) 391 { 392 unsigned long cpu; 393 394 cpu_notifier_register_begin(); 395 for_each_online_cpu(cpu) 396 if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) == 397 NOTIFY_BAD) 398 goto cleanup; 399 __register_cpu_notifier(&zswap_dstmem_notifier); 400 cpu_notifier_register_done(); 401 return 0; 402 403 cleanup: 404 for_each_online_cpu(cpu) 405 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); 406 cpu_notifier_register_done(); 407 return -ENOMEM; 408 } 409 410 static void zswap_cpu_dstmem_destroy(void) 411 { 412 unsigned long cpu; 413 414 cpu_notifier_register_begin(); 415 for_each_online_cpu(cpu) 416 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); 417 __unregister_cpu_notifier(&zswap_dstmem_notifier); 418 cpu_notifier_register_done(); 419 } 420 421 static int __zswap_cpu_comp_notifier(struct zswap_pool *pool, 422 unsigned long action, unsigned long cpu) 423 { 424 struct crypto_comp *tfm; 425 426 switch (action) { 427 case CPU_UP_PREPARE: 428 if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu))) 429 break; 430 tfm = crypto_alloc_comp(pool->tfm_name, 0, 0); 431 if (IS_ERR_OR_NULL(tfm)) { 432 pr_err("could not alloc crypto comp %s : %ld\n", 433 pool->tfm_name, PTR_ERR(tfm)); 434 return NOTIFY_BAD; 435 } 436 *per_cpu_ptr(pool->tfm, cpu) = tfm; 437 break; 438 case CPU_DEAD: 439 case CPU_UP_CANCELED: 440 tfm = *per_cpu_ptr(pool->tfm, cpu); 441 if (!IS_ERR_OR_NULL(tfm)) 442 crypto_free_comp(tfm); 443 *per_cpu_ptr(pool->tfm, cpu) = NULL; 444 break; 445 default: 446 break; 447 } 448 return NOTIFY_OK; 449 } 450 451 static int zswap_cpu_comp_notifier(struct notifier_block *nb, 452 unsigned long action, void *pcpu) 453 { 454 unsigned long cpu = (unsigned long)pcpu; 455 struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier); 456 457 return __zswap_cpu_comp_notifier(pool, action, cpu); 458 } 459 460 static int zswap_cpu_comp_init(struct zswap_pool *pool) 461 { 462 unsigned long cpu; 463 464 memset(&pool->notifier, 0, sizeof(pool->notifier)); 465 pool->notifier.notifier_call = zswap_cpu_comp_notifier; 466 467 cpu_notifier_register_begin(); 468 for_each_online_cpu(cpu) 469 if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) == 470 NOTIFY_BAD) 471 goto cleanup; 472 __register_cpu_notifier(&pool->notifier); 473 cpu_notifier_register_done(); 474 return 0; 475 476 cleanup: 477 for_each_online_cpu(cpu) 478 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); 479 cpu_notifier_register_done(); 480 return -ENOMEM; 481 } 482 483 static void zswap_cpu_comp_destroy(struct zswap_pool *pool) 484 { 485 unsigned long cpu; 486 487 cpu_notifier_register_begin(); 488 for_each_online_cpu(cpu) 489 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); 490 __unregister_cpu_notifier(&pool->notifier); 491 cpu_notifier_register_done(); 492 } 493 494 /********************************* 495 * pool functions 496 **********************************/ 497 498 static struct zswap_pool *__zswap_pool_current(void) 499 { 500 struct zswap_pool *pool; 501 502 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); 503 WARN_ON(!pool); 504 505 return pool; 506 } 507 508 static struct zswap_pool *zswap_pool_current(void) 509 { 510 assert_spin_locked(&zswap_pools_lock); 511 512 return __zswap_pool_current(); 513 } 514 515 static struct zswap_pool *zswap_pool_current_get(void) 516 { 517 struct zswap_pool *pool; 518 519 rcu_read_lock(); 520 521 pool = __zswap_pool_current(); 522 if (!pool || !zswap_pool_get(pool)) 523 pool = NULL; 524 525 rcu_read_unlock(); 526 527 return pool; 528 } 529 530 static struct zswap_pool *zswap_pool_last_get(void) 531 { 532 struct zswap_pool *pool, *last = NULL; 533 534 rcu_read_lock(); 535 536 list_for_each_entry_rcu(pool, &zswap_pools, list) 537 last = pool; 538 if (!WARN_ON(!last) && !zswap_pool_get(last)) 539 last = NULL; 540 541 rcu_read_unlock(); 542 543 return last; 544 } 545 546 /* type and compressor must be null-terminated */ 547 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) 548 { 549 struct zswap_pool *pool; 550 551 assert_spin_locked(&zswap_pools_lock); 552 553 list_for_each_entry_rcu(pool, &zswap_pools, list) { 554 if (strcmp(pool->tfm_name, compressor)) 555 continue; 556 if (strcmp(zpool_get_type(pool->zpool), type)) 557 continue; 558 /* if we can't get it, it's about to be destroyed */ 559 if (!zswap_pool_get(pool)) 560 continue; 561 return pool; 562 } 563 564 return NULL; 565 } 566 567 static struct zswap_pool *zswap_pool_create(char *type, char *compressor) 568 { 569 struct zswap_pool *pool; 570 char name[38]; /* 'zswap' + 32 char (max) num + \0 */ 571 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; 572 573 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 574 if (!pool) { 575 pr_err("pool alloc failed\n"); 576 return NULL; 577 } 578 579 /* unique name for each pool specifically required by zsmalloc */ 580 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count)); 581 582 pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops); 583 if (!pool->zpool) { 584 pr_err("%s zpool not available\n", type); 585 goto error; 586 } 587 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool)); 588 589 strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name)); 590 pool->tfm = alloc_percpu(struct crypto_comp *); 591 if (!pool->tfm) { 592 pr_err("percpu alloc failed\n"); 593 goto error; 594 } 595 596 if (zswap_cpu_comp_init(pool)) 597 goto error; 598 pr_debug("using %s compressor\n", pool->tfm_name); 599 600 /* being the current pool takes 1 ref; this func expects the 601 * caller to always add the new pool as the current pool 602 */ 603 kref_init(&pool->kref); 604 INIT_LIST_HEAD(&pool->list); 605 606 zswap_pool_debug("created", pool); 607 608 return pool; 609 610 error: 611 free_percpu(pool->tfm); 612 if (pool->zpool) 613 zpool_destroy_pool(pool->zpool); 614 kfree(pool); 615 return NULL; 616 } 617 618 static __init struct zswap_pool *__zswap_pool_create_fallback(void) 619 { 620 if (!crypto_has_comp(zswap_compressor, 0, 0)) { 621 if (!strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) { 622 pr_err("default compressor %s not available\n", 623 zswap_compressor); 624 return NULL; 625 } 626 pr_err("compressor %s not available, using default %s\n", 627 zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT); 628 param_free_charp(&zswap_compressor); 629 zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT; 630 } 631 if (!zpool_has_pool(zswap_zpool_type)) { 632 if (!strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) { 633 pr_err("default zpool %s not available\n", 634 zswap_zpool_type); 635 return NULL; 636 } 637 pr_err("zpool %s not available, using default %s\n", 638 zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT); 639 param_free_charp(&zswap_zpool_type); 640 zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; 641 } 642 643 return zswap_pool_create(zswap_zpool_type, zswap_compressor); 644 } 645 646 static void zswap_pool_destroy(struct zswap_pool *pool) 647 { 648 zswap_pool_debug("destroying", pool); 649 650 zswap_cpu_comp_destroy(pool); 651 free_percpu(pool->tfm); 652 zpool_destroy_pool(pool->zpool); 653 kfree(pool); 654 } 655 656 static int __must_check zswap_pool_get(struct zswap_pool *pool) 657 { 658 return kref_get_unless_zero(&pool->kref); 659 } 660 661 static void __zswap_pool_release(struct rcu_head *head) 662 { 663 struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head); 664 665 /* nobody should have been able to get a kref... */ 666 WARN_ON(kref_get_unless_zero(&pool->kref)); 667 668 /* pool is now off zswap_pools list and has no references. */ 669 zswap_pool_destroy(pool); 670 } 671 672 static void __zswap_pool_empty(struct kref *kref) 673 { 674 struct zswap_pool *pool; 675 676 pool = container_of(kref, typeof(*pool), kref); 677 678 spin_lock(&zswap_pools_lock); 679 680 WARN_ON(pool == zswap_pool_current()); 681 682 list_del_rcu(&pool->list); 683 call_rcu(&pool->rcu_head, __zswap_pool_release); 684 685 spin_unlock(&zswap_pools_lock); 686 } 687 688 static void zswap_pool_put(struct zswap_pool *pool) 689 { 690 kref_put(&pool->kref, __zswap_pool_empty); 691 } 692 693 /********************************* 694 * param callbacks 695 **********************************/ 696 697 /* val must be a null-terminated string */ 698 static int __zswap_param_set(const char *val, const struct kernel_param *kp, 699 char *type, char *compressor) 700 { 701 struct zswap_pool *pool, *put_pool = NULL; 702 char *s = strstrip((char *)val); 703 int ret; 704 705 /* no change required */ 706 if (!strcmp(s, *(char **)kp->arg)) 707 return 0; 708 709 /* if this is load-time (pre-init) param setting, 710 * don't create a pool; that's done during init. 711 */ 712 if (!zswap_init_started) 713 return param_set_charp(s, kp); 714 715 if (!type) { 716 if (!zpool_has_pool(s)) { 717 pr_err("zpool %s not available\n", s); 718 return -ENOENT; 719 } 720 type = s; 721 } else if (!compressor) { 722 if (!crypto_has_comp(s, 0, 0)) { 723 pr_err("compressor %s not available\n", s); 724 return -ENOENT; 725 } 726 compressor = s; 727 } else { 728 WARN_ON(1); 729 return -EINVAL; 730 } 731 732 spin_lock(&zswap_pools_lock); 733 734 pool = zswap_pool_find_get(type, compressor); 735 if (pool) { 736 zswap_pool_debug("using existing", pool); 737 list_del_rcu(&pool->list); 738 } else { 739 spin_unlock(&zswap_pools_lock); 740 pool = zswap_pool_create(type, compressor); 741 spin_lock(&zswap_pools_lock); 742 } 743 744 if (pool) 745 ret = param_set_charp(s, kp); 746 else 747 ret = -EINVAL; 748 749 if (!ret) { 750 put_pool = zswap_pool_current(); 751 list_add_rcu(&pool->list, &zswap_pools); 752 } else if (pool) { 753 /* add the possibly pre-existing pool to the end of the pools 754 * list; if it's new (and empty) then it'll be removed and 755 * destroyed by the put after we drop the lock 756 */ 757 list_add_tail_rcu(&pool->list, &zswap_pools); 758 put_pool = pool; 759 } 760 761 spin_unlock(&zswap_pools_lock); 762 763 /* drop the ref from either the old current pool, 764 * or the new pool we failed to add 765 */ 766 if (put_pool) 767 zswap_pool_put(put_pool); 768 769 return ret; 770 } 771 772 static int zswap_compressor_param_set(const char *val, 773 const struct kernel_param *kp) 774 { 775 return __zswap_param_set(val, kp, zswap_zpool_type, NULL); 776 } 777 778 static int zswap_zpool_param_set(const char *val, 779 const struct kernel_param *kp) 780 { 781 return __zswap_param_set(val, kp, NULL, zswap_compressor); 782 } 783 784 /********************************* 785 * writeback code 786 **********************************/ 787 /* return enum for zswap_get_swap_cache_page */ 788 enum zswap_get_swap_ret { 789 ZSWAP_SWAPCACHE_NEW, 790 ZSWAP_SWAPCACHE_EXIST, 791 ZSWAP_SWAPCACHE_FAIL, 792 }; 793 794 /* 795 * zswap_get_swap_cache_page 796 * 797 * This is an adaption of read_swap_cache_async() 798 * 799 * This function tries to find a page with the given swap entry 800 * in the swapper_space address space (the swap cache). If the page 801 * is found, it is returned in retpage. Otherwise, a page is allocated, 802 * added to the swap cache, and returned in retpage. 803 * 804 * If success, the swap cache page is returned in retpage 805 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache 806 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated, 807 * the new page is added to swapcache and locked 808 * Returns ZSWAP_SWAPCACHE_FAIL on error 809 */ 810 static int zswap_get_swap_cache_page(swp_entry_t entry, 811 struct page **retpage) 812 { 813 bool page_was_allocated; 814 815 *retpage = __read_swap_cache_async(entry, GFP_KERNEL, 816 NULL, 0, &page_was_allocated); 817 if (page_was_allocated) 818 return ZSWAP_SWAPCACHE_NEW; 819 if (!*retpage) 820 return ZSWAP_SWAPCACHE_FAIL; 821 return ZSWAP_SWAPCACHE_EXIST; 822 } 823 824 /* 825 * Attempts to free an entry by adding a page to the swap cache, 826 * decompressing the entry data into the page, and issuing a 827 * bio write to write the page back to the swap device. 828 * 829 * This can be thought of as a "resumed writeback" of the page 830 * to the swap device. We are basically resuming the same swap 831 * writeback path that was intercepted with the frontswap_store() 832 * in the first place. After the page has been decompressed into 833 * the swap cache, the compressed version stored by zswap can be 834 * freed. 835 */ 836 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) 837 { 838 struct zswap_header *zhdr; 839 swp_entry_t swpentry; 840 struct zswap_tree *tree; 841 pgoff_t offset; 842 struct zswap_entry *entry; 843 struct page *page; 844 struct crypto_comp *tfm; 845 u8 *src, *dst; 846 unsigned int dlen; 847 int ret; 848 struct writeback_control wbc = { 849 .sync_mode = WB_SYNC_NONE, 850 }; 851 852 /* extract swpentry from data */ 853 zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO); 854 swpentry = zhdr->swpentry; /* here */ 855 zpool_unmap_handle(pool, handle); 856 tree = zswap_trees[swp_type(swpentry)]; 857 offset = swp_offset(swpentry); 858 859 /* find and ref zswap entry */ 860 spin_lock(&tree->lock); 861 entry = zswap_entry_find_get(&tree->rbroot, offset); 862 if (!entry) { 863 /* entry was invalidated */ 864 spin_unlock(&tree->lock); 865 return 0; 866 } 867 spin_unlock(&tree->lock); 868 BUG_ON(offset != entry->offset); 869 870 /* try to allocate swap cache page */ 871 switch (zswap_get_swap_cache_page(swpentry, &page)) { 872 case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */ 873 ret = -ENOMEM; 874 goto fail; 875 876 case ZSWAP_SWAPCACHE_EXIST: 877 /* page is already in the swap cache, ignore for now */ 878 put_page(page); 879 ret = -EEXIST; 880 goto fail; 881 882 case ZSWAP_SWAPCACHE_NEW: /* page is locked */ 883 /* decompress */ 884 dlen = PAGE_SIZE; 885 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle, 886 ZPOOL_MM_RO) + sizeof(struct zswap_header); 887 dst = kmap_atomic(page); 888 tfm = *get_cpu_ptr(entry->pool->tfm); 889 ret = crypto_comp_decompress(tfm, src, entry->length, 890 dst, &dlen); 891 put_cpu_ptr(entry->pool->tfm); 892 kunmap_atomic(dst); 893 zpool_unmap_handle(entry->pool->zpool, entry->handle); 894 BUG_ON(ret); 895 BUG_ON(dlen != PAGE_SIZE); 896 897 /* page is up to date */ 898 SetPageUptodate(page); 899 } 900 901 /* move it to the tail of the inactive list after end_writeback */ 902 SetPageReclaim(page); 903 904 /* start writeback */ 905 __swap_writepage(page, &wbc, end_swap_bio_write); 906 put_page(page); 907 zswap_written_back_pages++; 908 909 spin_lock(&tree->lock); 910 /* drop local reference */ 911 zswap_entry_put(tree, entry); 912 913 /* 914 * There are two possible situations for entry here: 915 * (1) refcount is 1(normal case), entry is valid and on the tree 916 * (2) refcount is 0, entry is freed and not on the tree 917 * because invalidate happened during writeback 918 * search the tree and free the entry if find entry 919 */ 920 if (entry == zswap_rb_search(&tree->rbroot, offset)) 921 zswap_entry_put(tree, entry); 922 spin_unlock(&tree->lock); 923 924 goto end; 925 926 /* 927 * if we get here due to ZSWAP_SWAPCACHE_EXIST 928 * a load may happening concurrently 929 * it is safe and okay to not free the entry 930 * if we free the entry in the following put 931 * it it either okay to return !0 932 */ 933 fail: 934 spin_lock(&tree->lock); 935 zswap_entry_put(tree, entry); 936 spin_unlock(&tree->lock); 937 938 end: 939 return ret; 940 } 941 942 static int zswap_shrink(void) 943 { 944 struct zswap_pool *pool; 945 int ret; 946 947 pool = zswap_pool_last_get(); 948 if (!pool) 949 return -ENOENT; 950 951 ret = zpool_shrink(pool->zpool, 1, NULL); 952 953 zswap_pool_put(pool); 954 955 return ret; 956 } 957 958 /********************************* 959 * frontswap hooks 960 **********************************/ 961 /* attempts to compress and store an single page */ 962 static int zswap_frontswap_store(unsigned type, pgoff_t offset, 963 struct page *page) 964 { 965 struct zswap_tree *tree = zswap_trees[type]; 966 struct zswap_entry *entry, *dupentry; 967 struct crypto_comp *tfm; 968 int ret; 969 unsigned int dlen = PAGE_SIZE, len; 970 unsigned long handle; 971 char *buf; 972 u8 *src, *dst; 973 struct zswap_header *zhdr; 974 975 if (!zswap_enabled || !tree) { 976 ret = -ENODEV; 977 goto reject; 978 } 979 980 /* reclaim space if needed */ 981 if (zswap_is_full()) { 982 zswap_pool_limit_hit++; 983 if (zswap_shrink()) { 984 zswap_reject_reclaim_fail++; 985 ret = -ENOMEM; 986 goto reject; 987 } 988 } 989 990 /* allocate entry */ 991 entry = zswap_entry_cache_alloc(GFP_KERNEL); 992 if (!entry) { 993 zswap_reject_kmemcache_fail++; 994 ret = -ENOMEM; 995 goto reject; 996 } 997 998 /* if entry is successfully added, it keeps the reference */ 999 entry->pool = zswap_pool_current_get(); 1000 if (!entry->pool) { 1001 ret = -EINVAL; 1002 goto freepage; 1003 } 1004 1005 /* compress */ 1006 dst = get_cpu_var(zswap_dstmem); 1007 tfm = *get_cpu_ptr(entry->pool->tfm); 1008 src = kmap_atomic(page); 1009 ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen); 1010 kunmap_atomic(src); 1011 put_cpu_ptr(entry->pool->tfm); 1012 if (ret) { 1013 ret = -EINVAL; 1014 goto put_dstmem; 1015 } 1016 1017 /* store */ 1018 len = dlen + sizeof(struct zswap_header); 1019 ret = zpool_malloc(entry->pool->zpool, len, 1020 __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM, 1021 &handle); 1022 if (ret == -ENOSPC) { 1023 zswap_reject_compress_poor++; 1024 goto put_dstmem; 1025 } 1026 if (ret) { 1027 zswap_reject_alloc_fail++; 1028 goto put_dstmem; 1029 } 1030 zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW); 1031 zhdr->swpentry = swp_entry(type, offset); 1032 buf = (u8 *)(zhdr + 1); 1033 memcpy(buf, dst, dlen); 1034 zpool_unmap_handle(entry->pool->zpool, handle); 1035 put_cpu_var(zswap_dstmem); 1036 1037 /* populate entry */ 1038 entry->offset = offset; 1039 entry->handle = handle; 1040 entry->length = dlen; 1041 1042 /* map */ 1043 spin_lock(&tree->lock); 1044 do { 1045 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry); 1046 if (ret == -EEXIST) { 1047 zswap_duplicate_entry++; 1048 /* remove from rbtree */ 1049 zswap_rb_erase(&tree->rbroot, dupentry); 1050 zswap_entry_put(tree, dupentry); 1051 } 1052 } while (ret == -EEXIST); 1053 spin_unlock(&tree->lock); 1054 1055 /* update stats */ 1056 atomic_inc(&zswap_stored_pages); 1057 zswap_update_total_size(); 1058 1059 return 0; 1060 1061 put_dstmem: 1062 put_cpu_var(zswap_dstmem); 1063 zswap_pool_put(entry->pool); 1064 freepage: 1065 zswap_entry_cache_free(entry); 1066 reject: 1067 return ret; 1068 } 1069 1070 /* 1071 * returns 0 if the page was successfully decompressed 1072 * return -1 on entry not found or error 1073 */ 1074 static int zswap_frontswap_load(unsigned type, pgoff_t offset, 1075 struct page *page) 1076 { 1077 struct zswap_tree *tree = zswap_trees[type]; 1078 struct zswap_entry *entry; 1079 struct crypto_comp *tfm; 1080 u8 *src, *dst; 1081 unsigned int dlen; 1082 int ret; 1083 1084 /* find */ 1085 spin_lock(&tree->lock); 1086 entry = zswap_entry_find_get(&tree->rbroot, offset); 1087 if (!entry) { 1088 /* entry was written back */ 1089 spin_unlock(&tree->lock); 1090 return -1; 1091 } 1092 spin_unlock(&tree->lock); 1093 1094 /* decompress */ 1095 dlen = PAGE_SIZE; 1096 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle, 1097 ZPOOL_MM_RO) + sizeof(struct zswap_header); 1098 dst = kmap_atomic(page); 1099 tfm = *get_cpu_ptr(entry->pool->tfm); 1100 ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen); 1101 put_cpu_ptr(entry->pool->tfm); 1102 kunmap_atomic(dst); 1103 zpool_unmap_handle(entry->pool->zpool, entry->handle); 1104 BUG_ON(ret); 1105 1106 spin_lock(&tree->lock); 1107 zswap_entry_put(tree, entry); 1108 spin_unlock(&tree->lock); 1109 1110 return 0; 1111 } 1112 1113 /* frees an entry in zswap */ 1114 static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset) 1115 { 1116 struct zswap_tree *tree = zswap_trees[type]; 1117 struct zswap_entry *entry; 1118 1119 /* find */ 1120 spin_lock(&tree->lock); 1121 entry = zswap_rb_search(&tree->rbroot, offset); 1122 if (!entry) { 1123 /* entry was written back */ 1124 spin_unlock(&tree->lock); 1125 return; 1126 } 1127 1128 /* remove from rbtree */ 1129 zswap_rb_erase(&tree->rbroot, entry); 1130 1131 /* drop the initial reference from entry creation */ 1132 zswap_entry_put(tree, entry); 1133 1134 spin_unlock(&tree->lock); 1135 } 1136 1137 /* frees all zswap entries for the given swap type */ 1138 static void zswap_frontswap_invalidate_area(unsigned type) 1139 { 1140 struct zswap_tree *tree = zswap_trees[type]; 1141 struct zswap_entry *entry, *n; 1142 1143 if (!tree) 1144 return; 1145 1146 /* walk the tree and free everything */ 1147 spin_lock(&tree->lock); 1148 rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) 1149 zswap_free_entry(entry); 1150 tree->rbroot = RB_ROOT; 1151 spin_unlock(&tree->lock); 1152 kfree(tree); 1153 zswap_trees[type] = NULL; 1154 } 1155 1156 static void zswap_frontswap_init(unsigned type) 1157 { 1158 struct zswap_tree *tree; 1159 1160 tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL); 1161 if (!tree) { 1162 pr_err("alloc failed, zswap disabled for swap type %d\n", type); 1163 return; 1164 } 1165 1166 tree->rbroot = RB_ROOT; 1167 spin_lock_init(&tree->lock); 1168 zswap_trees[type] = tree; 1169 } 1170 1171 static struct frontswap_ops zswap_frontswap_ops = { 1172 .store = zswap_frontswap_store, 1173 .load = zswap_frontswap_load, 1174 .invalidate_page = zswap_frontswap_invalidate_page, 1175 .invalidate_area = zswap_frontswap_invalidate_area, 1176 .init = zswap_frontswap_init 1177 }; 1178 1179 /********************************* 1180 * debugfs functions 1181 **********************************/ 1182 #ifdef CONFIG_DEBUG_FS 1183 #include <linux/debugfs.h> 1184 1185 static struct dentry *zswap_debugfs_root; 1186 1187 static int __init zswap_debugfs_init(void) 1188 { 1189 if (!debugfs_initialized()) 1190 return -ENODEV; 1191 1192 zswap_debugfs_root = debugfs_create_dir("zswap", NULL); 1193 if (!zswap_debugfs_root) 1194 return -ENOMEM; 1195 1196 debugfs_create_u64("pool_limit_hit", S_IRUGO, 1197 zswap_debugfs_root, &zswap_pool_limit_hit); 1198 debugfs_create_u64("reject_reclaim_fail", S_IRUGO, 1199 zswap_debugfs_root, &zswap_reject_reclaim_fail); 1200 debugfs_create_u64("reject_alloc_fail", S_IRUGO, 1201 zswap_debugfs_root, &zswap_reject_alloc_fail); 1202 debugfs_create_u64("reject_kmemcache_fail", S_IRUGO, 1203 zswap_debugfs_root, &zswap_reject_kmemcache_fail); 1204 debugfs_create_u64("reject_compress_poor", S_IRUGO, 1205 zswap_debugfs_root, &zswap_reject_compress_poor); 1206 debugfs_create_u64("written_back_pages", S_IRUGO, 1207 zswap_debugfs_root, &zswap_written_back_pages); 1208 debugfs_create_u64("duplicate_entry", S_IRUGO, 1209 zswap_debugfs_root, &zswap_duplicate_entry); 1210 debugfs_create_u64("pool_total_size", S_IRUGO, 1211 zswap_debugfs_root, &zswap_pool_total_size); 1212 debugfs_create_atomic_t("stored_pages", S_IRUGO, 1213 zswap_debugfs_root, &zswap_stored_pages); 1214 1215 return 0; 1216 } 1217 1218 static void __exit zswap_debugfs_exit(void) 1219 { 1220 debugfs_remove_recursive(zswap_debugfs_root); 1221 } 1222 #else 1223 static int __init zswap_debugfs_init(void) 1224 { 1225 return 0; 1226 } 1227 1228 static void __exit zswap_debugfs_exit(void) { } 1229 #endif 1230 1231 /********************************* 1232 * module init and exit 1233 **********************************/ 1234 static int __init init_zswap(void) 1235 { 1236 struct zswap_pool *pool; 1237 1238 zswap_init_started = true; 1239 1240 if (zswap_entry_cache_create()) { 1241 pr_err("entry cache creation failed\n"); 1242 goto cache_fail; 1243 } 1244 1245 if (zswap_cpu_dstmem_init()) { 1246 pr_err("dstmem alloc failed\n"); 1247 goto dstmem_fail; 1248 } 1249 1250 pool = __zswap_pool_create_fallback(); 1251 if (!pool) { 1252 pr_err("pool creation failed\n"); 1253 goto pool_fail; 1254 } 1255 pr_info("loaded using pool %s/%s\n", pool->tfm_name, 1256 zpool_get_type(pool->zpool)); 1257 1258 list_add(&pool->list, &zswap_pools); 1259 1260 frontswap_register_ops(&zswap_frontswap_ops); 1261 if (zswap_debugfs_init()) 1262 pr_warn("debugfs initialization failed\n"); 1263 return 0; 1264 1265 pool_fail: 1266 zswap_cpu_dstmem_destroy(); 1267 dstmem_fail: 1268 zswap_entry_cache_destroy(); 1269 cache_fail: 1270 return -ENOMEM; 1271 } 1272 /* must be late so crypto has time to come up */ 1273 late_initcall(init_zswap); 1274 1275 MODULE_LICENSE("GPL"); 1276 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>"); 1277 MODULE_DESCRIPTION("Compressed cache for swap pages"); 1278