1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2018, Joyent, Inc. 24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. 26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 #include <sys/spa.h> 30 #include <sys/zio.h> 31 #include <sys/spa_impl.h> 32 #include <sys/zio_compress.h> 33 #include <sys/zio_checksum.h> 34 #include <sys/zfs_context.h> 35 #include <sys/arc.h> 36 #include <sys/zfs_refcount.h> 37 #include <sys/vdev.h> 38 #include <sys/vdev_trim.h> 39 #include <sys/vdev_impl.h> 40 #include <sys/dsl_pool.h> 41 #include <sys/zio_checksum.h> 42 #include <sys/multilist.h> 43 #include <sys/abd.h> 44 #include <sys/zil.h> 45 #include <sys/fm/fs/zfs.h> 46 #ifdef _KERNEL 47 #include <sys/shrinker.h> 48 #include <sys/vmsystm.h> 49 #include <sys/zpl.h> 50 #include <linux/page_compat.h> 51 #include <linux/notifier.h> 52 #include <linux/memory.h> 53 #endif 54 #include <sys/callb.h> 55 #include <sys/kstat.h> 56 #include <sys/zthr.h> 57 #include <zfs_fletcher.h> 58 #include <sys/arc_impl.h> 59 #include <sys/trace_zfs.h> 60 #include <sys/aggsum.h> 61 62 /* 63 * This is a limit on how many pages the ARC shrinker makes available for 64 * eviction in response to one page allocation attempt. Note that in 65 * practice, the kernel's shrinker can ask us to evict up to about 4x this 66 * for one allocation attempt. 67 * 68 * The default limit of 10,000 (in practice, 160MB per allocation attempt 69 * with 4K pages) limits the amount of time spent attempting to reclaim ARC 70 * memory to less than 100ms per allocation attempt, even with a small 71 * average compressed block size of ~8KB. 72 * 73 * See also the comment in arc_shrinker_count(). 74 * Set to 0 to disable limit. 75 */ 76 int zfs_arc_shrinker_limit = 10000; 77 78 #ifdef CONFIG_MEMORY_HOTPLUG 79 static struct notifier_block arc_hotplug_callback_mem_nb; 80 #endif 81 82 /* 83 * Return a default max arc size based on the amount of physical memory. 84 */ 85 uint64_t 86 arc_default_max(uint64_t min, uint64_t allmem) 87 { 88 /* Default to 1/2 of all memory. */ 89 return (MAX(allmem / 2, min)); 90 } 91 92 #ifdef _KERNEL 93 /* 94 * Return maximum amount of memory that we could possibly use. Reduced 95 * to half of all memory in user space which is primarily used for testing. 96 */ 97 uint64_t 98 arc_all_memory(void) 99 { 100 #ifdef CONFIG_HIGHMEM 101 return (ptob(zfs_totalram_pages - zfs_totalhigh_pages)); 102 #else 103 return (ptob(zfs_totalram_pages)); 104 #endif /* CONFIG_HIGHMEM */ 105 } 106 107 /* 108 * Return the amount of memory that is considered free. In user space 109 * which is primarily used for testing we pretend that free memory ranges 110 * from 0-20% of all memory. 111 */ 112 uint64_t 113 arc_free_memory(void) 114 { 115 #ifdef CONFIG_HIGHMEM 116 struct sysinfo si; 117 si_meminfo(&si); 118 return (ptob(si.freeram - si.freehigh)); 119 #else 120 return (ptob(nr_free_pages() + 121 nr_inactive_file_pages())); 122 #endif /* CONFIG_HIGHMEM */ 123 } 124 125 /* 126 * Return the amount of memory that can be consumed before reclaim will be 127 * needed. Positive if there is sufficient free memory, negative indicates 128 * the amount of memory that needs to be freed up. 129 */ 130 int64_t 131 arc_available_memory(void) 132 { 133 return (arc_free_memory() - arc_sys_free); 134 } 135 136 static uint64_t 137 arc_evictable_memory(void) 138 { 139 int64_t asize = aggsum_value(&arc_size); 140 uint64_t arc_clean = 141 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_DATA]) + 142 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) + 143 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_DATA]) + 144 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 145 uint64_t arc_dirty = MAX((int64_t)asize - (int64_t)arc_clean, 0); 146 147 /* 148 * Scale reported evictable memory in proportion to page cache, cap 149 * at specified min/max. 150 */ 151 uint64_t min = (ptob(nr_file_pages()) / 100) * zfs_arc_pc_percent; 152 min = MAX(arc_c_min, MIN(arc_c_max, min)); 153 154 if (arc_dirty >= min) 155 return (arc_clean); 156 157 return (MAX((int64_t)asize - (int64_t)min, 0)); 158 } 159 160 /* 161 * The _count() function returns the number of free-able objects. 162 * The _scan() function returns the number of objects that were freed. 163 */ 164 static unsigned long 165 arc_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 166 { 167 /* 168 * __GFP_FS won't be set if we are called from ZFS code (see 169 * kmem_flags_convert(), which removes it). To avoid a deadlock, we 170 * don't allow evicting in this case. We return 0 rather than 171 * SHRINK_STOP so that the shrinker logic doesn't accumulate a 172 * deficit against us. 173 */ 174 if (!(sc->gfp_mask & __GFP_FS)) { 175 return (0); 176 } 177 178 /* 179 * This code is reached in the "direct reclaim" case, where the 180 * kernel (outside ZFS) is trying to allocate a page, and the system 181 * is low on memory. 182 * 183 * The kernel's shrinker code doesn't understand how many pages the 184 * ARC's callback actually frees, so it may ask the ARC to shrink a 185 * lot for one page allocation. This is problematic because it may 186 * take a long time, thus delaying the page allocation, and because 187 * it may force the ARC to unnecessarily shrink very small. 188 * 189 * Therefore, we limit the amount of data that we say is evictable, 190 * which limits the amount that the shrinker will ask us to evict for 191 * one page allocation attempt. 192 * 193 * In practice, we may be asked to shrink 4x the limit to satisfy one 194 * page allocation, before the kernel's shrinker code gives up on us. 195 * When that happens, we rely on the kernel code to find the pages 196 * that we freed before invoking the OOM killer. This happens in 197 * __alloc_pages_slowpath(), which retries and finds the pages we 198 * freed when it calls get_page_from_freelist(). 199 * 200 * See also the comment above zfs_arc_shrinker_limit. 201 */ 202 int64_t limit = zfs_arc_shrinker_limit != 0 ? 203 zfs_arc_shrinker_limit : INT64_MAX; 204 return (MIN(limit, btop((int64_t)arc_evictable_memory()))); 205 } 206 207 static unsigned long 208 arc_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 209 { 210 ASSERT((sc->gfp_mask & __GFP_FS) != 0); 211 212 /* The arc is considered warm once reclaim has occurred */ 213 if (unlikely(arc_warm == B_FALSE)) 214 arc_warm = B_TRUE; 215 216 /* 217 * Evict the requested number of pages by reducing arc_c and waiting 218 * for the requested amount of data to be evicted. 219 */ 220 arc_reduce_target_size(ptob(sc->nr_to_scan)); 221 arc_wait_for_eviction(ptob(sc->nr_to_scan)); 222 if (current->reclaim_state != NULL) 223 current->reclaim_state->reclaimed_slab += sc->nr_to_scan; 224 225 /* 226 * We are experiencing memory pressure which the arc_evict_zthr was 227 * unable to keep up with. Set arc_no_grow to briefly pause arc 228 * growth to avoid compounding the memory pressure. 229 */ 230 arc_no_grow = B_TRUE; 231 232 /* 233 * When direct reclaim is observed it usually indicates a rapid 234 * increase in memory pressure. This occurs because the kswapd 235 * threads were unable to asynchronously keep enough free memory 236 * available. 237 */ 238 if (current_is_kswapd()) { 239 ARCSTAT_BUMP(arcstat_memory_indirect_count); 240 } else { 241 ARCSTAT_BUMP(arcstat_memory_direct_count); 242 } 243 244 return (sc->nr_to_scan); 245 } 246 247 SPL_SHRINKER_DECLARE(arc_shrinker, 248 arc_shrinker_count, arc_shrinker_scan, DEFAULT_SEEKS); 249 250 int 251 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg) 252 { 253 uint64_t free_memory = arc_free_memory(); 254 255 if (free_memory > arc_all_memory() * arc_lotsfree_percent / 100) 256 return (0); 257 258 if (txg > spa->spa_lowmem_last_txg) { 259 spa->spa_lowmem_last_txg = txg; 260 spa->spa_lowmem_page_load = 0; 261 } 262 /* 263 * If we are in pageout, we know that memory is already tight, 264 * the arc is already going to be evicting, so we just want to 265 * continue to let page writes occur as quickly as possible. 266 */ 267 if (current_is_kswapd()) { 268 if (spa->spa_lowmem_page_load > 269 MAX(arc_sys_free / 4, free_memory) / 4) { 270 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim); 271 return (SET_ERROR(ERESTART)); 272 } 273 /* Note: reserve is inflated, so we deflate */ 274 atomic_add_64(&spa->spa_lowmem_page_load, reserve / 8); 275 return (0); 276 } else if (spa->spa_lowmem_page_load > 0 && arc_reclaim_needed()) { 277 /* memory is low, delay before restarting */ 278 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 279 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim); 280 return (SET_ERROR(EAGAIN)); 281 } 282 spa->spa_lowmem_page_load = 0; 283 return (0); 284 } 285 286 static void 287 arc_set_sys_free(uint64_t allmem) 288 { 289 /* 290 * The ARC tries to keep at least this much memory available for the 291 * system. This gives the ARC time to shrink in response to memory 292 * pressure, before running completely out of memory and invoking the 293 * direct-reclaim ARC shrinker. 294 * 295 * This should be more than twice high_wmark_pages(), so that 296 * arc_wait_for_eviction() will wait until at least the 297 * high_wmark_pages() are free (see arc_evict_state_impl()). 298 * 299 * Note: Even when the system is very low on memory, the kernel's 300 * shrinker code may only ask for one "batch" of pages (512KB) to be 301 * evicted. If concurrent allocations consume these pages, there may 302 * still be insufficient free pages, and the OOM killer takes action. 303 * 304 * By setting arc_sys_free large enough, and having 305 * arc_wait_for_eviction() wait until there is at least arc_sys_free/2 306 * free memory, it is much less likely that concurrent allocations can 307 * consume all the memory that was evicted before checking for 308 * OOM. 309 * 310 * It's hard to iterate the zones from a linux kernel module, which 311 * makes it difficult to determine the watermark dynamically. Instead 312 * we compute the maximum high watermark for this system, based 313 * on the amount of memory, assuming default parameters on Linux kernel 314 * 5.3. 315 */ 316 317 /* 318 * Base wmark_low is 4 * the square root of Kbytes of RAM. 319 */ 320 long wmark = 4 * int_sqrt(allmem/1024) * 1024; 321 322 /* 323 * Clamp to between 128K and 64MB. 324 */ 325 wmark = MAX(wmark, 128 * 1024); 326 wmark = MIN(wmark, 64 * 1024 * 1024); 327 328 /* 329 * watermark_boost can increase the wmark by up to 150%. 330 */ 331 wmark += wmark * 150 / 100; 332 333 /* 334 * arc_sys_free needs to be more than 2x the watermark, because 335 * arc_wait_for_eviction() waits for half of arc_sys_free. Bump this up 336 * to 3x to ensure we're above it. 337 */ 338 arc_sys_free = wmark * 3 + allmem / 32; 339 } 340 341 void 342 arc_lowmem_init(void) 343 { 344 uint64_t allmem = arc_all_memory(); 345 346 /* 347 * Register a shrinker to support synchronous (direct) memory 348 * reclaim from the arc. This is done to prevent kswapd from 349 * swapping out pages when it is preferable to shrink the arc. 350 */ 351 spl_register_shrinker(&arc_shrinker); 352 arc_set_sys_free(allmem); 353 } 354 355 void 356 arc_lowmem_fini(void) 357 { 358 spl_unregister_shrinker(&arc_shrinker); 359 } 360 361 int 362 param_set_arc_long(const char *buf, zfs_kernel_param_t *kp) 363 { 364 int error; 365 366 error = param_set_long(buf, kp); 367 if (error < 0) 368 return (SET_ERROR(error)); 369 370 arc_tuning_update(B_TRUE); 371 372 return (0); 373 } 374 375 int 376 param_set_arc_int(const char *buf, zfs_kernel_param_t *kp) 377 { 378 int error; 379 380 error = param_set_int(buf, kp); 381 if (error < 0) 382 return (SET_ERROR(error)); 383 384 arc_tuning_update(B_TRUE); 385 386 return (0); 387 } 388 389 #ifdef CONFIG_MEMORY_HOTPLUG 390 /* ARGSUSED */ 391 static int 392 arc_hotplug_callback(struct notifier_block *self, unsigned long action, 393 void *arg) 394 { 395 uint64_t allmem = arc_all_memory(); 396 if (action != MEM_ONLINE) 397 return (NOTIFY_OK); 398 399 arc_set_limits(allmem); 400 401 #ifdef __LP64__ 402 if (zfs_dirty_data_max_max == 0) 403 zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024, 404 allmem * zfs_dirty_data_max_max_percent / 100); 405 #else 406 if (zfs_dirty_data_max_max == 0) 407 zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024, 408 allmem * zfs_dirty_data_max_max_percent / 100); 409 #endif 410 411 arc_set_sys_free(allmem); 412 return (NOTIFY_OK); 413 } 414 #endif 415 416 void 417 arc_register_hotplug(void) 418 { 419 #ifdef CONFIG_MEMORY_HOTPLUG 420 arc_hotplug_callback_mem_nb.notifier_call = arc_hotplug_callback; 421 /* There is no significance to the value 100 */ 422 arc_hotplug_callback_mem_nb.priority = 100; 423 register_memory_notifier(&arc_hotplug_callback_mem_nb); 424 #endif 425 } 426 427 void 428 arc_unregister_hotplug(void) 429 { 430 #ifdef CONFIG_MEMORY_HOTPLUG 431 unregister_memory_notifier(&arc_hotplug_callback_mem_nb); 432 #endif 433 } 434 #else /* _KERNEL */ 435 int64_t 436 arc_available_memory(void) 437 { 438 int64_t lowest = INT64_MAX; 439 440 /* Every 100 calls, free a small amount */ 441 if (spa_get_random(100) == 0) 442 lowest = -1024; 443 444 return (lowest); 445 } 446 447 int 448 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg) 449 { 450 return (0); 451 } 452 453 uint64_t 454 arc_all_memory(void) 455 { 456 return (ptob(physmem) / 2); 457 } 458 459 uint64_t 460 arc_free_memory(void) 461 { 462 return (spa_get_random(arc_all_memory() * 20 / 100)); 463 } 464 465 void 466 arc_register_hotplug(void) 467 { 468 } 469 470 void 471 arc_unregister_hotplug(void) 472 { 473 } 474 #endif /* _KERNEL */ 475 476 /* 477 * Helper function for arc_prune_async() it is responsible for safely 478 * handling the execution of a registered arc_prune_func_t. 479 */ 480 static void 481 arc_prune_task(void *ptr) 482 { 483 arc_prune_t *ap = (arc_prune_t *)ptr; 484 arc_prune_func_t *func = ap->p_pfunc; 485 486 if (func != NULL) 487 func(ap->p_adjust, ap->p_private); 488 489 zfs_refcount_remove(&ap->p_refcnt, func); 490 } 491 492 /* 493 * Notify registered consumers they must drop holds on a portion of the ARC 494 * buffered they reference. This provides a mechanism to ensure the ARC can 495 * honor the arc_meta_limit and reclaim otherwise pinned ARC buffers. This 496 * is analogous to dnlc_reduce_cache() but more generic. 497 * 498 * This operation is performed asynchronously so it may be safely called 499 * in the context of the arc_reclaim_thread(). A reference is taken here 500 * for each registered arc_prune_t and the arc_prune_task() is responsible 501 * for releasing it once the registered arc_prune_func_t has completed. 502 */ 503 void 504 arc_prune_async(int64_t adjust) 505 { 506 arc_prune_t *ap; 507 508 mutex_enter(&arc_prune_mtx); 509 for (ap = list_head(&arc_prune_list); ap != NULL; 510 ap = list_next(&arc_prune_list, ap)) { 511 512 if (zfs_refcount_count(&ap->p_refcnt) >= 2) 513 continue; 514 515 zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc); 516 ap->p_adjust = adjust; 517 if (taskq_dispatch(arc_prune_taskq, arc_prune_task, 518 ap, TQ_SLEEP) == TASKQID_INVALID) { 519 zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc); 520 continue; 521 } 522 ARCSTAT_BUMP(arcstat_prune); 523 } 524 mutex_exit(&arc_prune_mtx); 525 } 526 527 /* BEGIN CSTYLED */ 528 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, shrinker_limit, INT, ZMOD_RW, 529 "Limit on number of pages that ARC shrinker can reclaim at once"); 530 /* END CSTYLED */ 531