1 /* 2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. 3 * Copyright (C) 2007 The Regents of the University of California. 4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>. 6 * UCRL-CODE-235197 7 * 8 * This file is part of the SPL, Solaris Porting Layer. 9 * 10 * The SPL is free software; you can redistribute it and/or modify it 11 * under the terms of the GNU General Public License as published by the 12 * Free Software Foundation; either version 2 of the License, or (at your 13 * option) any later version. 14 * 15 * The SPL is distributed in the hope that it will be useful, but WITHOUT 16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 * for more details. 19 * 20 * You should have received a copy of the GNU General Public License along 21 * with the SPL. If not, see <http://www.gnu.org/licenses/>. 22 */ 23 24 #include <linux/percpu_compat.h> 25 #include <sys/kmem.h> 26 #include <sys/kmem_cache.h> 27 #include <sys/taskq.h> 28 #include <sys/timer.h> 29 #include <sys/vmem.h> 30 #include <sys/wait.h> 31 #include <linux/slab.h> 32 #include <linux/swap.h> 33 #include <linux/prefetch.h> 34 35 /* 36 * Within the scope of spl-kmem.c file the kmem_cache_* definitions 37 * are removed to allow access to the real Linux slab allocator. 38 */ 39 #undef kmem_cache_destroy 40 #undef kmem_cache_create 41 #undef kmem_cache_alloc 42 #undef kmem_cache_free 43 44 45 /* 46 * Linux 3.16 replaced smp_mb__{before,after}_{atomic,clear}_{dec,inc,bit}() 47 * with smp_mb__{before,after}_atomic() because they were redundant. This is 48 * only used inside our SLAB allocator, so we implement an internal wrapper 49 * here to give us smp_mb__{before,after}_atomic() on older kernels. 50 */ 51 #ifndef smp_mb__before_atomic 52 #define smp_mb__before_atomic(x) smp_mb__before_clear_bit(x) 53 #endif 54 55 #ifndef smp_mb__after_atomic 56 #define smp_mb__after_atomic(x) smp_mb__after_clear_bit(x) 57 #endif 58 59 /* BEGIN CSTYLED */ 60 /* 61 * Cache magazines are an optimization designed to minimize the cost of 62 * allocating memory. They do this by keeping a per-cpu cache of recently 63 * freed objects, which can then be reallocated without taking a lock. This 64 * can improve performance on highly contended caches. However, because 65 * objects in magazines will prevent otherwise empty slabs from being 66 * immediately released this may not be ideal for low memory machines. 67 * 68 * For this reason spl_kmem_cache_magazine_size can be used to set a maximum 69 * magazine size. When this value is set to 0 the magazine size will be 70 * automatically determined based on the object size. Otherwise magazines 71 * will be limited to 2-256 objects per magazine (i.e per cpu). Magazines 72 * may never be entirely disabled in this implementation. 73 */ 74 static unsigned int spl_kmem_cache_magazine_size = 0; 75 module_param(spl_kmem_cache_magazine_size, uint, 0444); 76 MODULE_PARM_DESC(spl_kmem_cache_magazine_size, 77 "Default magazine size (2-256), set automatically (0)"); 78 79 /* 80 * The default behavior is to report the number of objects remaining in the 81 * cache. This allows the Linux VM to repeatedly reclaim objects from the 82 * cache when memory is low satisfy other memory allocations. Alternately, 83 * setting this value to KMC_RECLAIM_ONCE limits how aggressively the cache 84 * is reclaimed. This may increase the likelihood of out of memory events. 85 */ 86 static unsigned int spl_kmem_cache_reclaim = 0 /* KMC_RECLAIM_ONCE */; 87 module_param(spl_kmem_cache_reclaim, uint, 0644); 88 MODULE_PARM_DESC(spl_kmem_cache_reclaim, "Single reclaim pass (0x1)"); 89 90 static unsigned int spl_kmem_cache_obj_per_slab = SPL_KMEM_CACHE_OBJ_PER_SLAB; 91 module_param(spl_kmem_cache_obj_per_slab, uint, 0644); 92 MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab, "Number of objects per slab"); 93 94 static unsigned int spl_kmem_cache_max_size = SPL_KMEM_CACHE_MAX_SIZE; 95 module_param(spl_kmem_cache_max_size, uint, 0644); 96 MODULE_PARM_DESC(spl_kmem_cache_max_size, "Maximum size of slab in MB"); 97 98 /* 99 * For small objects the Linux slab allocator should be used to make the most 100 * efficient use of the memory. However, large objects are not supported by 101 * the Linux slab and therefore the SPL implementation is preferred. A cutoff 102 * of 16K was determined to be optimal for architectures using 4K pages and 103 * to also work well on architecutres using larger 64K page sizes. 104 */ 105 static unsigned int spl_kmem_cache_slab_limit = 16384; 106 module_param(spl_kmem_cache_slab_limit, uint, 0644); 107 MODULE_PARM_DESC(spl_kmem_cache_slab_limit, 108 "Objects less than N bytes use the Linux slab"); 109 110 /* 111 * The number of threads available to allocate new slabs for caches. This 112 * should not need to be tuned but it is available for performance analysis. 113 */ 114 static unsigned int spl_kmem_cache_kmem_threads = 4; 115 module_param(spl_kmem_cache_kmem_threads, uint, 0444); 116 MODULE_PARM_DESC(spl_kmem_cache_kmem_threads, 117 "Number of spl_kmem_cache threads"); 118 /* END CSTYLED */ 119 120 /* 121 * Slab allocation interfaces 122 * 123 * While the Linux slab implementation was inspired by the Solaris 124 * implementation I cannot use it to emulate the Solaris APIs. I 125 * require two features which are not provided by the Linux slab. 126 * 127 * 1) Constructors AND destructors. Recent versions of the Linux 128 * kernel have removed support for destructors. This is a deal 129 * breaker for the SPL which contains particularly expensive 130 * initializers for mutex's, condition variables, etc. We also 131 * require a minimal level of cleanup for these data types unlike 132 * many Linux data types which do need to be explicitly destroyed. 133 * 134 * 2) Virtual address space backed slab. Callers of the Solaris slab 135 * expect it to work well for both small are very large allocations. 136 * Because of memory fragmentation the Linux slab which is backed 137 * by kmalloc'ed memory performs very badly when confronted with 138 * large numbers of large allocations. Basing the slab on the 139 * virtual address space removes the need for contiguous pages 140 * and greatly improve performance for large allocations. 141 * 142 * For these reasons, the SPL has its own slab implementation with 143 * the needed features. It is not as highly optimized as either the 144 * Solaris or Linux slabs, but it should get me most of what is 145 * needed until it can be optimized or obsoleted by another approach. 146 * 147 * One serious concern I do have about this method is the relatively 148 * small virtual address space on 32bit arches. This will seriously 149 * constrain the size of the slab caches and their performance. 150 */ 151 152 struct list_head spl_kmem_cache_list; /* List of caches */ 153 struct rw_semaphore spl_kmem_cache_sem; /* Cache list lock */ 154 taskq_t *spl_kmem_cache_taskq; /* Task queue for aging / reclaim */ 155 156 static void spl_cache_shrink(spl_kmem_cache_t *skc, void *obj); 157 158 static void * 159 kv_alloc(spl_kmem_cache_t *skc, int size, int flags) 160 { 161 gfp_t lflags = kmem_flags_convert(flags); 162 void *ptr; 163 164 ptr = spl_vmalloc(size, lflags | __GFP_HIGHMEM); 165 166 /* Resulting allocated memory will be page aligned */ 167 ASSERT(IS_P2ALIGNED(ptr, PAGE_SIZE)); 168 169 return (ptr); 170 } 171 172 static void 173 kv_free(spl_kmem_cache_t *skc, void *ptr, int size) 174 { 175 ASSERT(IS_P2ALIGNED(ptr, PAGE_SIZE)); 176 177 /* 178 * The Linux direct reclaim path uses this out of band value to 179 * determine if forward progress is being made. Normally this is 180 * incremented by kmem_freepages() which is part of the various 181 * Linux slab implementations. However, since we are using none 182 * of that infrastructure we are responsible for incrementing it. 183 */ 184 if (current->reclaim_state) 185 current->reclaim_state->reclaimed_slab += size >> PAGE_SHIFT; 186 187 vfree(ptr); 188 } 189 190 /* 191 * Required space for each aligned sks. 192 */ 193 static inline uint32_t 194 spl_sks_size(spl_kmem_cache_t *skc) 195 { 196 return (P2ROUNDUP_TYPED(sizeof (spl_kmem_slab_t), 197 skc->skc_obj_align, uint32_t)); 198 } 199 200 /* 201 * Required space for each aligned object. 202 */ 203 static inline uint32_t 204 spl_obj_size(spl_kmem_cache_t *skc) 205 { 206 uint32_t align = skc->skc_obj_align; 207 208 return (P2ROUNDUP_TYPED(skc->skc_obj_size, align, uint32_t) + 209 P2ROUNDUP_TYPED(sizeof (spl_kmem_obj_t), align, uint32_t)); 210 } 211 212 uint64_t 213 spl_kmem_cache_inuse(kmem_cache_t *cache) 214 { 215 return (cache->skc_obj_total); 216 } 217 EXPORT_SYMBOL(spl_kmem_cache_inuse); 218 219 uint64_t 220 spl_kmem_cache_entry_size(kmem_cache_t *cache) 221 { 222 return (cache->skc_obj_size); 223 } 224 EXPORT_SYMBOL(spl_kmem_cache_entry_size); 225 226 /* 227 * Lookup the spl_kmem_object_t for an object given that object. 228 */ 229 static inline spl_kmem_obj_t * 230 spl_sko_from_obj(spl_kmem_cache_t *skc, void *obj) 231 { 232 return (obj + P2ROUNDUP_TYPED(skc->skc_obj_size, 233 skc->skc_obj_align, uint32_t)); 234 } 235 236 /* 237 * It's important that we pack the spl_kmem_obj_t structure and the 238 * actual objects in to one large address space to minimize the number 239 * of calls to the allocator. It is far better to do a few large 240 * allocations and then subdivide it ourselves. Now which allocator 241 * we use requires balancing a few trade offs. 242 * 243 * For small objects we use kmem_alloc() because as long as you are 244 * only requesting a small number of pages (ideally just one) its cheap. 245 * However, when you start requesting multiple pages with kmem_alloc() 246 * it gets increasingly expensive since it requires contiguous pages. 247 * For this reason we shift to vmem_alloc() for slabs of large objects 248 * which removes the need for contiguous pages. We do not use 249 * vmem_alloc() in all cases because there is significant locking 250 * overhead in __get_vm_area_node(). This function takes a single 251 * global lock when acquiring an available virtual address range which 252 * serializes all vmem_alloc()'s for all slab caches. Using slightly 253 * different allocation functions for small and large objects should 254 * give us the best of both worlds. 255 * 256 * +------------------------+ 257 * | spl_kmem_slab_t --+-+ | 258 * | skc_obj_size <-+ | | 259 * | spl_kmem_obj_t | | 260 * | skc_obj_size <---+ | 261 * | spl_kmem_obj_t | | 262 * | ... v | 263 * +------------------------+ 264 */ 265 static spl_kmem_slab_t * 266 spl_slab_alloc(spl_kmem_cache_t *skc, int flags) 267 { 268 spl_kmem_slab_t *sks; 269 void *base; 270 uint32_t obj_size; 271 272 base = kv_alloc(skc, skc->skc_slab_size, flags); 273 if (base == NULL) 274 return (NULL); 275 276 sks = (spl_kmem_slab_t *)base; 277 sks->sks_magic = SKS_MAGIC; 278 sks->sks_objs = skc->skc_slab_objs; 279 sks->sks_age = jiffies; 280 sks->sks_cache = skc; 281 INIT_LIST_HEAD(&sks->sks_list); 282 INIT_LIST_HEAD(&sks->sks_free_list); 283 sks->sks_ref = 0; 284 obj_size = spl_obj_size(skc); 285 286 for (int i = 0; i < sks->sks_objs; i++) { 287 void *obj = base + spl_sks_size(skc) + (i * obj_size); 288 289 ASSERT(IS_P2ALIGNED(obj, skc->skc_obj_align)); 290 spl_kmem_obj_t *sko = spl_sko_from_obj(skc, obj); 291 sko->sko_addr = obj; 292 sko->sko_magic = SKO_MAGIC; 293 sko->sko_slab = sks; 294 INIT_LIST_HEAD(&sko->sko_list); 295 list_add_tail(&sko->sko_list, &sks->sks_free_list); 296 } 297 298 return (sks); 299 } 300 301 /* 302 * Remove a slab from complete or partial list, it must be called with 303 * the 'skc->skc_lock' held but the actual free must be performed 304 * outside the lock to prevent deadlocking on vmem addresses. 305 */ 306 static void 307 spl_slab_free(spl_kmem_slab_t *sks, 308 struct list_head *sks_list, struct list_head *sko_list) 309 { 310 spl_kmem_cache_t *skc; 311 312 ASSERT(sks->sks_magic == SKS_MAGIC); 313 ASSERT(sks->sks_ref == 0); 314 315 skc = sks->sks_cache; 316 ASSERT(skc->skc_magic == SKC_MAGIC); 317 318 /* 319 * Update slab/objects counters in the cache, then remove the 320 * slab from the skc->skc_partial_list. Finally add the slab 321 * and all its objects in to the private work lists where the 322 * destructors will be called and the memory freed to the system. 323 */ 324 skc->skc_obj_total -= sks->sks_objs; 325 skc->skc_slab_total--; 326 list_del(&sks->sks_list); 327 list_add(&sks->sks_list, sks_list); 328 list_splice_init(&sks->sks_free_list, sko_list); 329 } 330 331 /* 332 * Reclaim empty slabs at the end of the partial list. 333 */ 334 static void 335 spl_slab_reclaim(spl_kmem_cache_t *skc) 336 { 337 spl_kmem_slab_t *sks = NULL, *m = NULL; 338 spl_kmem_obj_t *sko = NULL, *n = NULL; 339 LIST_HEAD(sks_list); 340 LIST_HEAD(sko_list); 341 342 /* 343 * Empty slabs and objects must be moved to a private list so they 344 * can be safely freed outside the spin lock. All empty slabs are 345 * at the end of skc->skc_partial_list, therefore once a non-empty 346 * slab is found we can stop scanning. 347 */ 348 spin_lock(&skc->skc_lock); 349 list_for_each_entry_safe_reverse(sks, m, 350 &skc->skc_partial_list, sks_list) { 351 352 if (sks->sks_ref > 0) 353 break; 354 355 spl_slab_free(sks, &sks_list, &sko_list); 356 } 357 spin_unlock(&skc->skc_lock); 358 359 /* 360 * The following two loops ensure all the object destructors are run, 361 * and the slabs themselves are freed. This is all done outside the 362 * skc->skc_lock since this allows the destructor to sleep, and 363 * allows us to perform a conditional reschedule when a freeing a 364 * large number of objects and slabs back to the system. 365 */ 366 367 list_for_each_entry_safe(sko, n, &sko_list, sko_list) { 368 ASSERT(sko->sko_magic == SKO_MAGIC); 369 } 370 371 list_for_each_entry_safe(sks, m, &sks_list, sks_list) { 372 ASSERT(sks->sks_magic == SKS_MAGIC); 373 kv_free(skc, sks, skc->skc_slab_size); 374 } 375 } 376 377 static spl_kmem_emergency_t * 378 spl_emergency_search(struct rb_root *root, void *obj) 379 { 380 struct rb_node *node = root->rb_node; 381 spl_kmem_emergency_t *ske; 382 unsigned long address = (unsigned long)obj; 383 384 while (node) { 385 ske = container_of(node, spl_kmem_emergency_t, ske_node); 386 387 if (address < ske->ske_obj) 388 node = node->rb_left; 389 else if (address > ske->ske_obj) 390 node = node->rb_right; 391 else 392 return (ske); 393 } 394 395 return (NULL); 396 } 397 398 static int 399 spl_emergency_insert(struct rb_root *root, spl_kmem_emergency_t *ske) 400 { 401 struct rb_node **new = &(root->rb_node), *parent = NULL; 402 spl_kmem_emergency_t *ske_tmp; 403 unsigned long address = ske->ske_obj; 404 405 while (*new) { 406 ske_tmp = container_of(*new, spl_kmem_emergency_t, ske_node); 407 408 parent = *new; 409 if (address < ske_tmp->ske_obj) 410 new = &((*new)->rb_left); 411 else if (address > ske_tmp->ske_obj) 412 new = &((*new)->rb_right); 413 else 414 return (0); 415 } 416 417 rb_link_node(&ske->ske_node, parent, new); 418 rb_insert_color(&ske->ske_node, root); 419 420 return (1); 421 } 422 423 /* 424 * Allocate a single emergency object and track it in a red black tree. 425 */ 426 static int 427 spl_emergency_alloc(spl_kmem_cache_t *skc, int flags, void **obj) 428 { 429 gfp_t lflags = kmem_flags_convert(flags); 430 spl_kmem_emergency_t *ske; 431 int order = get_order(skc->skc_obj_size); 432 int empty; 433 434 /* Last chance use a partial slab if one now exists */ 435 spin_lock(&skc->skc_lock); 436 empty = list_empty(&skc->skc_partial_list); 437 spin_unlock(&skc->skc_lock); 438 if (!empty) 439 return (-EEXIST); 440 441 ske = kmalloc(sizeof (*ske), lflags); 442 if (ske == NULL) 443 return (-ENOMEM); 444 445 ske->ske_obj = __get_free_pages(lflags, order); 446 if (ske->ske_obj == 0) { 447 kfree(ske); 448 return (-ENOMEM); 449 } 450 451 spin_lock(&skc->skc_lock); 452 empty = spl_emergency_insert(&skc->skc_emergency_tree, ske); 453 if (likely(empty)) { 454 skc->skc_obj_total++; 455 skc->skc_obj_emergency++; 456 if (skc->skc_obj_emergency > skc->skc_obj_emergency_max) 457 skc->skc_obj_emergency_max = skc->skc_obj_emergency; 458 } 459 spin_unlock(&skc->skc_lock); 460 461 if (unlikely(!empty)) { 462 free_pages(ske->ske_obj, order); 463 kfree(ske); 464 return (-EINVAL); 465 } 466 467 *obj = (void *)ske->ske_obj; 468 469 return (0); 470 } 471 472 /* 473 * Locate the passed object in the red black tree and free it. 474 */ 475 static int 476 spl_emergency_free(spl_kmem_cache_t *skc, void *obj) 477 { 478 spl_kmem_emergency_t *ske; 479 int order = get_order(skc->skc_obj_size); 480 481 spin_lock(&skc->skc_lock); 482 ske = spl_emergency_search(&skc->skc_emergency_tree, obj); 483 if (ske) { 484 rb_erase(&ske->ske_node, &skc->skc_emergency_tree); 485 skc->skc_obj_emergency--; 486 skc->skc_obj_total--; 487 } 488 spin_unlock(&skc->skc_lock); 489 490 if (ske == NULL) 491 return (-ENOENT); 492 493 free_pages(ske->ske_obj, order); 494 kfree(ske); 495 496 return (0); 497 } 498 499 /* 500 * Release objects from the per-cpu magazine back to their slab. The flush 501 * argument contains the max number of entries to remove from the magazine. 502 */ 503 static void 504 spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush) 505 { 506 spin_lock(&skc->skc_lock); 507 508 ASSERT(skc->skc_magic == SKC_MAGIC); 509 ASSERT(skm->skm_magic == SKM_MAGIC); 510 511 int count = MIN(flush, skm->skm_avail); 512 for (int i = 0; i < count; i++) 513 spl_cache_shrink(skc, skm->skm_objs[i]); 514 515 skm->skm_avail -= count; 516 memmove(skm->skm_objs, &(skm->skm_objs[count]), 517 sizeof (void *) * skm->skm_avail); 518 519 spin_unlock(&skc->skc_lock); 520 } 521 522 /* 523 * Size a slab based on the size of each aligned object plus spl_kmem_obj_t. 524 * When on-slab we want to target spl_kmem_cache_obj_per_slab. However, 525 * for very small objects we may end up with more than this so as not 526 * to waste space in the minimal allocation of a single page. 527 */ 528 static int 529 spl_slab_size(spl_kmem_cache_t *skc, uint32_t *objs, uint32_t *size) 530 { 531 uint32_t sks_size, obj_size, max_size, tgt_size, tgt_objs; 532 533 sks_size = spl_sks_size(skc); 534 obj_size = spl_obj_size(skc); 535 max_size = (spl_kmem_cache_max_size * 1024 * 1024); 536 tgt_size = (spl_kmem_cache_obj_per_slab * obj_size + sks_size); 537 538 if (tgt_size <= max_size) { 539 tgt_objs = (tgt_size - sks_size) / obj_size; 540 } else { 541 tgt_objs = (max_size - sks_size) / obj_size; 542 tgt_size = (tgt_objs * obj_size) + sks_size; 543 } 544 545 if (tgt_objs == 0) 546 return (-ENOSPC); 547 548 *objs = tgt_objs; 549 *size = tgt_size; 550 551 return (0); 552 } 553 554 /* 555 * Make a guess at reasonable per-cpu magazine size based on the size of 556 * each object and the cost of caching N of them in each magazine. Long 557 * term this should really adapt based on an observed usage heuristic. 558 */ 559 static int 560 spl_magazine_size(spl_kmem_cache_t *skc) 561 { 562 uint32_t obj_size = spl_obj_size(skc); 563 int size; 564 565 if (spl_kmem_cache_magazine_size > 0) 566 return (MAX(MIN(spl_kmem_cache_magazine_size, 256), 2)); 567 568 /* Per-magazine sizes below assume a 4Kib page size */ 569 if (obj_size > (PAGE_SIZE * 256)) 570 size = 4; /* Minimum 4Mib per-magazine */ 571 else if (obj_size > (PAGE_SIZE * 32)) 572 size = 16; /* Minimum 2Mib per-magazine */ 573 else if (obj_size > (PAGE_SIZE)) 574 size = 64; /* Minimum 256Kib per-magazine */ 575 else if (obj_size > (PAGE_SIZE / 4)) 576 size = 128; /* Minimum 128Kib per-magazine */ 577 else 578 size = 256; 579 580 return (size); 581 } 582 583 /* 584 * Allocate a per-cpu magazine to associate with a specific core. 585 */ 586 static spl_kmem_magazine_t * 587 spl_magazine_alloc(spl_kmem_cache_t *skc, int cpu) 588 { 589 spl_kmem_magazine_t *skm; 590 int size = sizeof (spl_kmem_magazine_t) + 591 sizeof (void *) * skc->skc_mag_size; 592 593 skm = kmalloc_node(size, GFP_KERNEL, cpu_to_node(cpu)); 594 if (skm) { 595 skm->skm_magic = SKM_MAGIC; 596 skm->skm_avail = 0; 597 skm->skm_size = skc->skc_mag_size; 598 skm->skm_refill = skc->skc_mag_refill; 599 skm->skm_cache = skc; 600 skm->skm_cpu = cpu; 601 } 602 603 return (skm); 604 } 605 606 /* 607 * Free a per-cpu magazine associated with a specific core. 608 */ 609 static void 610 spl_magazine_free(spl_kmem_magazine_t *skm) 611 { 612 ASSERT(skm->skm_magic == SKM_MAGIC); 613 ASSERT(skm->skm_avail == 0); 614 kfree(skm); 615 } 616 617 /* 618 * Create all pre-cpu magazines of reasonable sizes. 619 */ 620 static int 621 spl_magazine_create(spl_kmem_cache_t *skc) 622 { 623 int i = 0; 624 625 ASSERT((skc->skc_flags & KMC_SLAB) == 0); 626 627 skc->skc_mag = kzalloc(sizeof (spl_kmem_magazine_t *) * 628 num_possible_cpus(), kmem_flags_convert(KM_SLEEP)); 629 skc->skc_mag_size = spl_magazine_size(skc); 630 skc->skc_mag_refill = (skc->skc_mag_size + 1) / 2; 631 632 for_each_possible_cpu(i) { 633 skc->skc_mag[i] = spl_magazine_alloc(skc, i); 634 if (!skc->skc_mag[i]) { 635 for (i--; i >= 0; i--) 636 spl_magazine_free(skc->skc_mag[i]); 637 638 kfree(skc->skc_mag); 639 return (-ENOMEM); 640 } 641 } 642 643 return (0); 644 } 645 646 /* 647 * Destroy all pre-cpu magazines. 648 */ 649 static void 650 spl_magazine_destroy(spl_kmem_cache_t *skc) 651 { 652 spl_kmem_magazine_t *skm; 653 int i = 0; 654 655 ASSERT((skc->skc_flags & KMC_SLAB) == 0); 656 657 for_each_possible_cpu(i) { 658 skm = skc->skc_mag[i]; 659 spl_cache_flush(skc, skm, skm->skm_avail); 660 spl_magazine_free(skm); 661 } 662 663 kfree(skc->skc_mag); 664 } 665 666 /* 667 * Create a object cache based on the following arguments: 668 * name cache name 669 * size cache object size 670 * align cache object alignment 671 * ctor cache object constructor 672 * dtor cache object destructor 673 * reclaim cache object reclaim 674 * priv cache private data for ctor/dtor/reclaim 675 * vmp unused must be NULL 676 * flags 677 * KMC_KVMEM Force kvmem backed SPL cache 678 * KMC_SLAB Force Linux slab backed cache 679 * KMC_NODEBUG Disable debugging (unsupported) 680 */ 681 spl_kmem_cache_t * 682 spl_kmem_cache_create(const char *name, size_t size, size_t align, 683 spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, void *reclaim, 684 void *priv, void *vmp, int flags) 685 { 686 gfp_t lflags = kmem_flags_convert(KM_SLEEP); 687 spl_kmem_cache_t *skc; 688 int rc; 689 690 /* 691 * Unsupported flags 692 */ 693 ASSERT(vmp == NULL); 694 ASSERT(reclaim == NULL); 695 696 might_sleep(); 697 698 skc = kzalloc(sizeof (*skc), lflags); 699 if (skc == NULL) 700 return (NULL); 701 702 skc->skc_magic = SKC_MAGIC; 703 skc->skc_name_size = strlen(name) + 1; 704 skc->skc_name = (char *)kmalloc(skc->skc_name_size, lflags); 705 if (skc->skc_name == NULL) { 706 kfree(skc); 707 return (NULL); 708 } 709 strncpy(skc->skc_name, name, skc->skc_name_size); 710 711 skc->skc_ctor = ctor; 712 skc->skc_dtor = dtor; 713 skc->skc_private = priv; 714 skc->skc_vmp = vmp; 715 skc->skc_linux_cache = NULL; 716 skc->skc_flags = flags; 717 skc->skc_obj_size = size; 718 skc->skc_obj_align = SPL_KMEM_CACHE_ALIGN; 719 atomic_set(&skc->skc_ref, 0); 720 721 INIT_LIST_HEAD(&skc->skc_list); 722 INIT_LIST_HEAD(&skc->skc_complete_list); 723 INIT_LIST_HEAD(&skc->skc_partial_list); 724 skc->skc_emergency_tree = RB_ROOT; 725 spin_lock_init(&skc->skc_lock); 726 init_waitqueue_head(&skc->skc_waitq); 727 skc->skc_slab_fail = 0; 728 skc->skc_slab_create = 0; 729 skc->skc_slab_destroy = 0; 730 skc->skc_slab_total = 0; 731 skc->skc_slab_alloc = 0; 732 skc->skc_slab_max = 0; 733 skc->skc_obj_total = 0; 734 skc->skc_obj_alloc = 0; 735 skc->skc_obj_max = 0; 736 skc->skc_obj_deadlock = 0; 737 skc->skc_obj_emergency = 0; 738 skc->skc_obj_emergency_max = 0; 739 740 rc = percpu_counter_init_common(&skc->skc_linux_alloc, 0, 741 GFP_KERNEL); 742 if (rc != 0) { 743 kfree(skc); 744 return (NULL); 745 } 746 747 /* 748 * Verify the requested alignment restriction is sane. 749 */ 750 if (align) { 751 VERIFY(ISP2(align)); 752 VERIFY3U(align, >=, SPL_KMEM_CACHE_ALIGN); 753 VERIFY3U(align, <=, PAGE_SIZE); 754 skc->skc_obj_align = align; 755 } 756 757 /* 758 * When no specific type of slab is requested (kmem, vmem, or 759 * linuxslab) then select a cache type based on the object size 760 * and default tunables. 761 */ 762 if (!(skc->skc_flags & (KMC_SLAB | KMC_KVMEM))) { 763 if (spl_kmem_cache_slab_limit && 764 size <= (size_t)spl_kmem_cache_slab_limit) { 765 /* 766 * Objects smaller than spl_kmem_cache_slab_limit can 767 * use the Linux slab for better space-efficiency. 768 */ 769 skc->skc_flags |= KMC_SLAB; 770 } else { 771 /* 772 * All other objects are considered large and are 773 * placed on kvmem backed slabs. 774 */ 775 skc->skc_flags |= KMC_KVMEM; 776 } 777 } 778 779 /* 780 * Given the type of slab allocate the required resources. 781 */ 782 if (skc->skc_flags & KMC_KVMEM) { 783 rc = spl_slab_size(skc, 784 &skc->skc_slab_objs, &skc->skc_slab_size); 785 if (rc) 786 goto out; 787 788 rc = spl_magazine_create(skc); 789 if (rc) 790 goto out; 791 } else { 792 unsigned long slabflags = 0; 793 794 if (size > (SPL_MAX_KMEM_ORDER_NR_PAGES * PAGE_SIZE)) { 795 rc = EINVAL; 796 goto out; 797 } 798 799 #if defined(SLAB_USERCOPY) 800 /* 801 * Required for PAX-enabled kernels if the slab is to be 802 * used for copying between user and kernel space. 803 */ 804 slabflags |= SLAB_USERCOPY; 805 #endif 806 807 #if defined(HAVE_KMEM_CACHE_CREATE_USERCOPY) 808 /* 809 * Newer grsec patchset uses kmem_cache_create_usercopy() 810 * instead of SLAB_USERCOPY flag 811 */ 812 skc->skc_linux_cache = kmem_cache_create_usercopy( 813 skc->skc_name, size, align, slabflags, 0, size, NULL); 814 #else 815 skc->skc_linux_cache = kmem_cache_create( 816 skc->skc_name, size, align, slabflags, NULL); 817 #endif 818 if (skc->skc_linux_cache == NULL) { 819 rc = ENOMEM; 820 goto out; 821 } 822 } 823 824 down_write(&spl_kmem_cache_sem); 825 list_add_tail(&skc->skc_list, &spl_kmem_cache_list); 826 up_write(&spl_kmem_cache_sem); 827 828 return (skc); 829 out: 830 kfree(skc->skc_name); 831 percpu_counter_destroy(&skc->skc_linux_alloc); 832 kfree(skc); 833 return (NULL); 834 } 835 EXPORT_SYMBOL(spl_kmem_cache_create); 836 837 /* 838 * Register a move callback for cache defragmentation. 839 * XXX: Unimplemented but harmless to stub out for now. 840 */ 841 void 842 spl_kmem_cache_set_move(spl_kmem_cache_t *skc, 843 kmem_cbrc_t (move)(void *, void *, size_t, void *)) 844 { 845 ASSERT(move != NULL); 846 } 847 EXPORT_SYMBOL(spl_kmem_cache_set_move); 848 849 /* 850 * Destroy a cache and all objects associated with the cache. 851 */ 852 void 853 spl_kmem_cache_destroy(spl_kmem_cache_t *skc) 854 { 855 DECLARE_WAIT_QUEUE_HEAD(wq); 856 taskqid_t id; 857 858 ASSERT(skc->skc_magic == SKC_MAGIC); 859 ASSERT(skc->skc_flags & (KMC_KVMEM | KMC_SLAB)); 860 861 down_write(&spl_kmem_cache_sem); 862 list_del_init(&skc->skc_list); 863 up_write(&spl_kmem_cache_sem); 864 865 /* Cancel any and wait for any pending delayed tasks */ 866 VERIFY(!test_and_set_bit(KMC_BIT_DESTROY, &skc->skc_flags)); 867 868 spin_lock(&skc->skc_lock); 869 id = skc->skc_taskqid; 870 spin_unlock(&skc->skc_lock); 871 872 taskq_cancel_id(spl_kmem_cache_taskq, id); 873 874 /* 875 * Wait until all current callers complete, this is mainly 876 * to catch the case where a low memory situation triggers a 877 * cache reaping action which races with this destroy. 878 */ 879 wait_event(wq, atomic_read(&skc->skc_ref) == 0); 880 881 if (skc->skc_flags & KMC_KVMEM) { 882 spl_magazine_destroy(skc); 883 spl_slab_reclaim(skc); 884 } else { 885 ASSERT(skc->skc_flags & KMC_SLAB); 886 kmem_cache_destroy(skc->skc_linux_cache); 887 } 888 889 spin_lock(&skc->skc_lock); 890 891 /* 892 * Validate there are no objects in use and free all the 893 * spl_kmem_slab_t, spl_kmem_obj_t, and object buffers. 894 */ 895 ASSERT3U(skc->skc_slab_alloc, ==, 0); 896 ASSERT3U(skc->skc_obj_alloc, ==, 0); 897 ASSERT3U(skc->skc_slab_total, ==, 0); 898 ASSERT3U(skc->skc_obj_total, ==, 0); 899 ASSERT3U(skc->skc_obj_emergency, ==, 0); 900 ASSERT(list_empty(&skc->skc_complete_list)); 901 902 ASSERT3U(percpu_counter_sum(&skc->skc_linux_alloc), ==, 0); 903 percpu_counter_destroy(&skc->skc_linux_alloc); 904 905 spin_unlock(&skc->skc_lock); 906 907 kfree(skc->skc_name); 908 kfree(skc); 909 } 910 EXPORT_SYMBOL(spl_kmem_cache_destroy); 911 912 /* 913 * Allocate an object from a slab attached to the cache. This is used to 914 * repopulate the per-cpu magazine caches in batches when they run low. 915 */ 916 static void * 917 spl_cache_obj(spl_kmem_cache_t *skc, spl_kmem_slab_t *sks) 918 { 919 spl_kmem_obj_t *sko; 920 921 ASSERT(skc->skc_magic == SKC_MAGIC); 922 ASSERT(sks->sks_magic == SKS_MAGIC); 923 924 sko = list_entry(sks->sks_free_list.next, spl_kmem_obj_t, sko_list); 925 ASSERT(sko->sko_magic == SKO_MAGIC); 926 ASSERT(sko->sko_addr != NULL); 927 928 /* Remove from sks_free_list */ 929 list_del_init(&sko->sko_list); 930 931 sks->sks_age = jiffies; 932 sks->sks_ref++; 933 skc->skc_obj_alloc++; 934 935 /* Track max obj usage statistics */ 936 if (skc->skc_obj_alloc > skc->skc_obj_max) 937 skc->skc_obj_max = skc->skc_obj_alloc; 938 939 /* Track max slab usage statistics */ 940 if (sks->sks_ref == 1) { 941 skc->skc_slab_alloc++; 942 943 if (skc->skc_slab_alloc > skc->skc_slab_max) 944 skc->skc_slab_max = skc->skc_slab_alloc; 945 } 946 947 return (sko->sko_addr); 948 } 949 950 /* 951 * Generic slab allocation function to run by the global work queues. 952 * It is responsible for allocating a new slab, linking it in to the list 953 * of partial slabs, and then waking any waiters. 954 */ 955 static int 956 __spl_cache_grow(spl_kmem_cache_t *skc, int flags) 957 { 958 spl_kmem_slab_t *sks; 959 960 fstrans_cookie_t cookie = spl_fstrans_mark(); 961 sks = spl_slab_alloc(skc, flags); 962 spl_fstrans_unmark(cookie); 963 964 spin_lock(&skc->skc_lock); 965 if (sks) { 966 skc->skc_slab_total++; 967 skc->skc_obj_total += sks->sks_objs; 968 list_add_tail(&sks->sks_list, &skc->skc_partial_list); 969 970 smp_mb__before_atomic(); 971 clear_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags); 972 smp_mb__after_atomic(); 973 } 974 spin_unlock(&skc->skc_lock); 975 976 return (sks == NULL ? -ENOMEM : 0); 977 } 978 979 static void 980 spl_cache_grow_work(void *data) 981 { 982 spl_kmem_alloc_t *ska = (spl_kmem_alloc_t *)data; 983 spl_kmem_cache_t *skc = ska->ska_cache; 984 985 int error = __spl_cache_grow(skc, ska->ska_flags); 986 987 atomic_dec(&skc->skc_ref); 988 smp_mb__before_atomic(); 989 clear_bit(KMC_BIT_GROWING, &skc->skc_flags); 990 smp_mb__after_atomic(); 991 if (error == 0) 992 wake_up_all(&skc->skc_waitq); 993 994 kfree(ska); 995 } 996 997 /* 998 * Returns non-zero when a new slab should be available. 999 */ 1000 static int 1001 spl_cache_grow_wait(spl_kmem_cache_t *skc) 1002 { 1003 return (!test_bit(KMC_BIT_GROWING, &skc->skc_flags)); 1004 } 1005 1006 /* 1007 * No available objects on any slabs, create a new slab. Note that this 1008 * functionality is disabled for KMC_SLAB caches which are backed by the 1009 * Linux slab. 1010 */ 1011 static int 1012 spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj) 1013 { 1014 int remaining, rc = 0; 1015 1016 ASSERT0(flags & ~KM_PUBLIC_MASK); 1017 ASSERT(skc->skc_magic == SKC_MAGIC); 1018 ASSERT((skc->skc_flags & KMC_SLAB) == 0); 1019 might_sleep(); 1020 *obj = NULL; 1021 1022 /* 1023 * Before allocating a new slab wait for any reaping to complete and 1024 * then return so the local magazine can be rechecked for new objects. 1025 */ 1026 if (test_bit(KMC_BIT_REAPING, &skc->skc_flags)) { 1027 rc = spl_wait_on_bit(&skc->skc_flags, KMC_BIT_REAPING, 1028 TASK_UNINTERRUPTIBLE); 1029 return (rc ? rc : -EAGAIN); 1030 } 1031 1032 /* 1033 * Note: It would be nice to reduce the overhead of context switch 1034 * and improve NUMA locality, by trying to allocate a new slab in the 1035 * current process context with KM_NOSLEEP flag. 1036 * 1037 * However, this can't be applied to vmem/kvmem due to a bug that 1038 * spl_vmalloc() doesn't honor gfp flags in page table allocation. 1039 */ 1040 1041 /* 1042 * This is handled by dispatching a work request to the global work 1043 * queue. This allows us to asynchronously allocate a new slab while 1044 * retaining the ability to safely fall back to a smaller synchronous 1045 * allocations to ensure forward progress is always maintained. 1046 */ 1047 if (test_and_set_bit(KMC_BIT_GROWING, &skc->skc_flags) == 0) { 1048 spl_kmem_alloc_t *ska; 1049 1050 ska = kmalloc(sizeof (*ska), kmem_flags_convert(flags)); 1051 if (ska == NULL) { 1052 clear_bit_unlock(KMC_BIT_GROWING, &skc->skc_flags); 1053 smp_mb__after_atomic(); 1054 wake_up_all(&skc->skc_waitq); 1055 return (-ENOMEM); 1056 } 1057 1058 atomic_inc(&skc->skc_ref); 1059 ska->ska_cache = skc; 1060 ska->ska_flags = flags; 1061 taskq_init_ent(&ska->ska_tqe); 1062 taskq_dispatch_ent(spl_kmem_cache_taskq, 1063 spl_cache_grow_work, ska, 0, &ska->ska_tqe); 1064 } 1065 1066 /* 1067 * The goal here is to only detect the rare case where a virtual slab 1068 * allocation has deadlocked. We must be careful to minimize the use 1069 * of emergency objects which are more expensive to track. Therefore, 1070 * we set a very long timeout for the asynchronous allocation and if 1071 * the timeout is reached the cache is flagged as deadlocked. From 1072 * this point only new emergency objects will be allocated until the 1073 * asynchronous allocation completes and clears the deadlocked flag. 1074 */ 1075 if (test_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags)) { 1076 rc = spl_emergency_alloc(skc, flags, obj); 1077 } else { 1078 remaining = wait_event_timeout(skc->skc_waitq, 1079 spl_cache_grow_wait(skc), HZ / 10); 1080 1081 if (!remaining) { 1082 spin_lock(&skc->skc_lock); 1083 if (test_bit(KMC_BIT_GROWING, &skc->skc_flags)) { 1084 set_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags); 1085 skc->skc_obj_deadlock++; 1086 } 1087 spin_unlock(&skc->skc_lock); 1088 } 1089 1090 rc = -ENOMEM; 1091 } 1092 1093 return (rc); 1094 } 1095 1096 /* 1097 * Refill a per-cpu magazine with objects from the slabs for this cache. 1098 * Ideally the magazine can be repopulated using existing objects which have 1099 * been released, however if we are unable to locate enough free objects new 1100 * slabs of objects will be created. On success NULL is returned, otherwise 1101 * the address of a single emergency object is returned for use by the caller. 1102 */ 1103 static void * 1104 spl_cache_refill(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flags) 1105 { 1106 spl_kmem_slab_t *sks; 1107 int count = 0, rc, refill; 1108 void *obj = NULL; 1109 1110 ASSERT(skc->skc_magic == SKC_MAGIC); 1111 ASSERT(skm->skm_magic == SKM_MAGIC); 1112 1113 refill = MIN(skm->skm_refill, skm->skm_size - skm->skm_avail); 1114 spin_lock(&skc->skc_lock); 1115 1116 while (refill > 0) { 1117 /* No slabs available we may need to grow the cache */ 1118 if (list_empty(&skc->skc_partial_list)) { 1119 spin_unlock(&skc->skc_lock); 1120 1121 local_irq_enable(); 1122 rc = spl_cache_grow(skc, flags, &obj); 1123 local_irq_disable(); 1124 1125 /* Emergency object for immediate use by caller */ 1126 if (rc == 0 && obj != NULL) 1127 return (obj); 1128 1129 if (rc) 1130 goto out; 1131 1132 /* Rescheduled to different CPU skm is not local */ 1133 if (skm != skc->skc_mag[smp_processor_id()]) 1134 goto out; 1135 1136 /* 1137 * Potentially rescheduled to the same CPU but 1138 * allocations may have occurred from this CPU while 1139 * we were sleeping so recalculate max refill. 1140 */ 1141 refill = MIN(refill, skm->skm_size - skm->skm_avail); 1142 1143 spin_lock(&skc->skc_lock); 1144 continue; 1145 } 1146 1147 /* Grab the next available slab */ 1148 sks = list_entry((&skc->skc_partial_list)->next, 1149 spl_kmem_slab_t, sks_list); 1150 ASSERT(sks->sks_magic == SKS_MAGIC); 1151 ASSERT(sks->sks_ref < sks->sks_objs); 1152 ASSERT(!list_empty(&sks->sks_free_list)); 1153 1154 /* 1155 * Consume as many objects as needed to refill the requested 1156 * cache. We must also be careful not to overfill it. 1157 */ 1158 while (sks->sks_ref < sks->sks_objs && refill-- > 0 && 1159 ++count) { 1160 ASSERT(skm->skm_avail < skm->skm_size); 1161 ASSERT(count < skm->skm_size); 1162 skm->skm_objs[skm->skm_avail++] = 1163 spl_cache_obj(skc, sks); 1164 } 1165 1166 /* Move slab to skc_complete_list when full */ 1167 if (sks->sks_ref == sks->sks_objs) { 1168 list_del(&sks->sks_list); 1169 list_add(&sks->sks_list, &skc->skc_complete_list); 1170 } 1171 } 1172 1173 spin_unlock(&skc->skc_lock); 1174 out: 1175 return (NULL); 1176 } 1177 1178 /* 1179 * Release an object back to the slab from which it came. 1180 */ 1181 static void 1182 spl_cache_shrink(spl_kmem_cache_t *skc, void *obj) 1183 { 1184 spl_kmem_slab_t *sks = NULL; 1185 spl_kmem_obj_t *sko = NULL; 1186 1187 ASSERT(skc->skc_magic == SKC_MAGIC); 1188 1189 sko = spl_sko_from_obj(skc, obj); 1190 ASSERT(sko->sko_magic == SKO_MAGIC); 1191 sks = sko->sko_slab; 1192 ASSERT(sks->sks_magic == SKS_MAGIC); 1193 ASSERT(sks->sks_cache == skc); 1194 list_add(&sko->sko_list, &sks->sks_free_list); 1195 1196 sks->sks_age = jiffies; 1197 sks->sks_ref--; 1198 skc->skc_obj_alloc--; 1199 1200 /* 1201 * Move slab to skc_partial_list when no longer full. Slabs 1202 * are added to the head to keep the partial list is quasi-full 1203 * sorted order. Fuller at the head, emptier at the tail. 1204 */ 1205 if (sks->sks_ref == (sks->sks_objs - 1)) { 1206 list_del(&sks->sks_list); 1207 list_add(&sks->sks_list, &skc->skc_partial_list); 1208 } 1209 1210 /* 1211 * Move empty slabs to the end of the partial list so 1212 * they can be easily found and freed during reclamation. 1213 */ 1214 if (sks->sks_ref == 0) { 1215 list_del(&sks->sks_list); 1216 list_add_tail(&sks->sks_list, &skc->skc_partial_list); 1217 skc->skc_slab_alloc--; 1218 } 1219 } 1220 1221 /* 1222 * Allocate an object from the per-cpu magazine, or if the magazine 1223 * is empty directly allocate from a slab and repopulate the magazine. 1224 */ 1225 void * 1226 spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags) 1227 { 1228 spl_kmem_magazine_t *skm; 1229 void *obj = NULL; 1230 1231 ASSERT0(flags & ~KM_PUBLIC_MASK); 1232 ASSERT(skc->skc_magic == SKC_MAGIC); 1233 ASSERT(!test_bit(KMC_BIT_DESTROY, &skc->skc_flags)); 1234 1235 /* 1236 * Allocate directly from a Linux slab. All optimizations are left 1237 * to the underlying cache we only need to guarantee that KM_SLEEP 1238 * callers will never fail. 1239 */ 1240 if (skc->skc_flags & KMC_SLAB) { 1241 struct kmem_cache *slc = skc->skc_linux_cache; 1242 do { 1243 obj = kmem_cache_alloc(slc, kmem_flags_convert(flags)); 1244 } while ((obj == NULL) && !(flags & KM_NOSLEEP)); 1245 1246 if (obj != NULL) { 1247 /* 1248 * Even though we leave everything up to the 1249 * underlying cache we still keep track of 1250 * how many objects we've allocated in it for 1251 * better debuggability. 1252 */ 1253 percpu_counter_inc(&skc->skc_linux_alloc); 1254 } 1255 goto ret; 1256 } 1257 1258 local_irq_disable(); 1259 1260 restart: 1261 /* 1262 * Safe to update per-cpu structure without lock, but 1263 * in the restart case we must be careful to reacquire 1264 * the local magazine since this may have changed 1265 * when we need to grow the cache. 1266 */ 1267 skm = skc->skc_mag[smp_processor_id()]; 1268 ASSERT(skm->skm_magic == SKM_MAGIC); 1269 1270 if (likely(skm->skm_avail)) { 1271 /* Object available in CPU cache, use it */ 1272 obj = skm->skm_objs[--skm->skm_avail]; 1273 } else { 1274 obj = spl_cache_refill(skc, skm, flags); 1275 if ((obj == NULL) && !(flags & KM_NOSLEEP)) 1276 goto restart; 1277 1278 local_irq_enable(); 1279 goto ret; 1280 } 1281 1282 local_irq_enable(); 1283 ASSERT(obj); 1284 ASSERT(IS_P2ALIGNED(obj, skc->skc_obj_align)); 1285 1286 ret: 1287 /* Pre-emptively migrate object to CPU L1 cache */ 1288 if (obj) { 1289 if (obj && skc->skc_ctor) 1290 skc->skc_ctor(obj, skc->skc_private, flags); 1291 else 1292 prefetchw(obj); 1293 } 1294 1295 return (obj); 1296 } 1297 EXPORT_SYMBOL(spl_kmem_cache_alloc); 1298 1299 /* 1300 * Free an object back to the local per-cpu magazine, there is no 1301 * guarantee that this is the same magazine the object was originally 1302 * allocated from. We may need to flush entire from the magazine 1303 * back to the slabs to make space. 1304 */ 1305 void 1306 spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj) 1307 { 1308 spl_kmem_magazine_t *skm; 1309 unsigned long flags; 1310 int do_reclaim = 0; 1311 int do_emergency = 0; 1312 1313 ASSERT(skc->skc_magic == SKC_MAGIC); 1314 ASSERT(!test_bit(KMC_BIT_DESTROY, &skc->skc_flags)); 1315 1316 /* 1317 * Run the destructor 1318 */ 1319 if (skc->skc_dtor) 1320 skc->skc_dtor(obj, skc->skc_private); 1321 1322 /* 1323 * Free the object from the Linux underlying Linux slab. 1324 */ 1325 if (skc->skc_flags & KMC_SLAB) { 1326 kmem_cache_free(skc->skc_linux_cache, obj); 1327 percpu_counter_dec(&skc->skc_linux_alloc); 1328 return; 1329 } 1330 1331 /* 1332 * While a cache has outstanding emergency objects all freed objects 1333 * must be checked. However, since emergency objects will never use 1334 * a virtual address these objects can be safely excluded as an 1335 * optimization. 1336 */ 1337 if (!is_vmalloc_addr(obj)) { 1338 spin_lock(&skc->skc_lock); 1339 do_emergency = (skc->skc_obj_emergency > 0); 1340 spin_unlock(&skc->skc_lock); 1341 1342 if (do_emergency && (spl_emergency_free(skc, obj) == 0)) 1343 return; 1344 } 1345 1346 local_irq_save(flags); 1347 1348 /* 1349 * Safe to update per-cpu structure without lock, but 1350 * no remote memory allocation tracking is being performed 1351 * it is entirely possible to allocate an object from one 1352 * CPU cache and return it to another. 1353 */ 1354 skm = skc->skc_mag[smp_processor_id()]; 1355 ASSERT(skm->skm_magic == SKM_MAGIC); 1356 1357 /* 1358 * Per-CPU cache full, flush it to make space for this object, 1359 * this may result in an empty slab which can be reclaimed once 1360 * interrupts are re-enabled. 1361 */ 1362 if (unlikely(skm->skm_avail >= skm->skm_size)) { 1363 spl_cache_flush(skc, skm, skm->skm_refill); 1364 do_reclaim = 1; 1365 } 1366 1367 /* Available space in cache, use it */ 1368 skm->skm_objs[skm->skm_avail++] = obj; 1369 1370 local_irq_restore(flags); 1371 1372 if (do_reclaim) 1373 spl_slab_reclaim(skc); 1374 } 1375 EXPORT_SYMBOL(spl_kmem_cache_free); 1376 1377 /* 1378 * Depending on how many and which objects are released it may simply 1379 * repopulate the local magazine which will then need to age-out. Objects 1380 * which cannot fit in the magazine will be released back to their slabs 1381 * which will also need to age out before being released. This is all just 1382 * best effort and we do not want to thrash creating and destroying slabs. 1383 */ 1384 void 1385 spl_kmem_cache_reap_now(spl_kmem_cache_t *skc) 1386 { 1387 ASSERT(skc->skc_magic == SKC_MAGIC); 1388 ASSERT(!test_bit(KMC_BIT_DESTROY, &skc->skc_flags)); 1389 1390 if (skc->skc_flags & KMC_SLAB) 1391 return; 1392 1393 atomic_inc(&skc->skc_ref); 1394 1395 /* 1396 * Prevent concurrent cache reaping when contended. 1397 */ 1398 if (test_and_set_bit(KMC_BIT_REAPING, &skc->skc_flags)) 1399 goto out; 1400 1401 /* Reclaim from the magazine and free all now empty slabs. */ 1402 unsigned long irq_flags; 1403 local_irq_save(irq_flags); 1404 spl_kmem_magazine_t *skm = skc->skc_mag[smp_processor_id()]; 1405 spl_cache_flush(skc, skm, skm->skm_avail); 1406 local_irq_restore(irq_flags); 1407 1408 spl_slab_reclaim(skc); 1409 clear_bit_unlock(KMC_BIT_REAPING, &skc->skc_flags); 1410 smp_mb__after_atomic(); 1411 wake_up_bit(&skc->skc_flags, KMC_BIT_REAPING); 1412 out: 1413 atomic_dec(&skc->skc_ref); 1414 } 1415 EXPORT_SYMBOL(spl_kmem_cache_reap_now); 1416 1417 /* 1418 * This is stubbed out for code consistency with other platforms. There 1419 * is existing logic to prevent concurrent reaping so while this is ugly 1420 * it should do no harm. 1421 */ 1422 int 1423 spl_kmem_cache_reap_active(void) 1424 { 1425 return (0); 1426 } 1427 EXPORT_SYMBOL(spl_kmem_cache_reap_active); 1428 1429 /* 1430 * Reap all free slabs from all registered caches. 1431 */ 1432 void 1433 spl_kmem_reap(void) 1434 { 1435 spl_kmem_cache_t *skc = NULL; 1436 1437 down_read(&spl_kmem_cache_sem); 1438 list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) { 1439 spl_kmem_cache_reap_now(skc); 1440 } 1441 up_read(&spl_kmem_cache_sem); 1442 } 1443 EXPORT_SYMBOL(spl_kmem_reap); 1444 1445 int 1446 spl_kmem_cache_init(void) 1447 { 1448 init_rwsem(&spl_kmem_cache_sem); 1449 INIT_LIST_HEAD(&spl_kmem_cache_list); 1450 spl_kmem_cache_taskq = taskq_create("spl_kmem_cache", 1451 spl_kmem_cache_kmem_threads, maxclsyspri, 1452 spl_kmem_cache_kmem_threads * 8, INT_MAX, 1453 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 1454 1455 if (spl_kmem_cache_taskq == NULL) 1456 return (-ENOMEM); 1457 1458 return (0); 1459 } 1460 1461 void 1462 spl_kmem_cache_fini(void) 1463 { 1464 taskq_destroy(spl_kmem_cache_taskq); 1465 } 1466