1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2018, Joyent, Inc. 24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. 26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 #include <sys/spa.h> 30 #include <sys/zio.h> 31 #include <sys/spa_impl.h> 32 #include <sys/zio_compress.h> 33 #include <sys/zio_checksum.h> 34 #include <sys/zfs_context.h> 35 #include <sys/arc.h> 36 #include <sys/zfs_refcount.h> 37 #include <sys/vdev.h> 38 #include <sys/vdev_trim.h> 39 #include <sys/vdev_impl.h> 40 #include <sys/dsl_pool.h> 41 #include <sys/multilist.h> 42 #include <sys/abd.h> 43 #include <sys/zil.h> 44 #include <sys/fm/fs/zfs.h> 45 #include <sys/shrinker.h> 46 #include <sys/vmsystm.h> 47 #include <sys/zpl.h> 48 #include <linux/page_compat.h> 49 #include <linux/notifier.h> 50 #include <linux/memory.h> 51 #include <linux/version.h> 52 #include <sys/callb.h> 53 #include <sys/kstat.h> 54 #include <sys/zthr.h> 55 #include <zfs_fletcher.h> 56 #include <sys/arc_impl.h> 57 #include <sys/trace_zfs.h> 58 #include <sys/aggsum.h> 59 60 /* 61 * This is a limit on how many pages the ARC shrinker makes available for 62 * eviction in response to one page allocation attempt. Note that in 63 * practice, the kernel's shrinker can ask us to evict up to about 4x this 64 * for one allocation attempt. 65 * 66 * The default limit of 10,000 (in practice, 160MB per allocation attempt 67 * with 4K pages) limits the amount of time spent attempting to reclaim ARC 68 * memory to less than 100ms per allocation attempt, even with a small 69 * average compressed block size of ~8KB. 70 * 71 * See also the comment in arc_shrinker_count(). 72 * Set to 0 to disable limit. 73 */ 74 static int zfs_arc_shrinker_limit = 10000; 75 76 /* 77 * Relative cost of ARC eviction, AKA number of seeks needed to restore evicted 78 * page. Bigger values make ARC more precious and evictions smaller comparing 79 * to other kernel subsystems. Value of 4 means parity with page cache, 80 * according to my reading of kernel's do_shrink_slab() and other code. 81 */ 82 static int zfs_arc_shrinker_seeks = DEFAULT_SEEKS; 83 84 #ifdef CONFIG_MEMORY_HOTPLUG 85 static struct notifier_block arc_hotplug_callback_mem_nb; 86 #endif 87 88 /* 89 * Return a default max arc size based on the amount of physical memory. 90 * This may be overridden by tuning the zfs_arc_max module parameter. 91 */ 92 uint64_t 93 arc_default_max(uint64_t min, uint64_t allmem) 94 { 95 uint64_t size; 96 97 if (allmem >= 1 << 30) 98 size = allmem - (1 << 30); 99 else 100 size = min; 101 return (MAX(allmem * 5 / 8, size)); 102 } 103 104 /* 105 * Return maximum amount of memory that we could possibly use. Reduced 106 * to half of all memory in user space which is primarily used for testing. 107 */ 108 uint64_t 109 arc_all_memory(void) 110 { 111 #ifdef CONFIG_HIGHMEM 112 return (ptob(zfs_totalram_pages - zfs_totalhigh_pages)); 113 #else 114 return (ptob(zfs_totalram_pages)); 115 #endif /* CONFIG_HIGHMEM */ 116 } 117 118 /* 119 * Return the amount of memory that is considered free. In user space 120 * which is primarily used for testing we pretend that free memory ranges 121 * from 0-20% of all memory. 122 */ 123 uint64_t 124 arc_free_memory(void) 125 { 126 #ifdef CONFIG_HIGHMEM 127 struct sysinfo si; 128 si_meminfo(&si); 129 return (ptob(si.freeram - si.freehigh)); 130 #else 131 return (ptob(nr_free_pages() + 132 nr_inactive_file_pages())); 133 #endif /* CONFIG_HIGHMEM */ 134 } 135 136 /* 137 * Return the amount of memory that can be consumed before reclaim will be 138 * needed. Positive if there is sufficient free memory, negative indicates 139 * the amount of memory that needs to be freed up. 140 */ 141 int64_t 142 arc_available_memory(void) 143 { 144 return (arc_free_memory() - arc_sys_free); 145 } 146 147 static uint64_t 148 arc_evictable_memory(void) 149 { 150 int64_t asize = aggsum_value(&arc_sums.arcstat_size); 151 uint64_t arc_clean = 152 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_DATA]) + 153 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) + 154 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_DATA]) + 155 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 156 uint64_t arc_dirty = MAX((int64_t)asize - (int64_t)arc_clean, 0); 157 158 /* 159 * Scale reported evictable memory in proportion to page cache, cap 160 * at specified min/max. 161 */ 162 uint64_t min = (ptob(nr_file_pages()) / 100) * zfs_arc_pc_percent; 163 min = MAX(arc_c_min, MIN(arc_c_max, min)); 164 165 if (arc_dirty >= min) 166 return (arc_clean); 167 168 return (MAX((int64_t)asize - (int64_t)min, 0)); 169 } 170 171 /* 172 * The _count() function returns the number of free-able objects. 173 * The _scan() function returns the number of objects that were freed. 174 */ 175 static unsigned long 176 arc_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) 177 { 178 /* 179 * The kernel's shrinker code may not understand how many pages the 180 * ARC's callback actually frees, so it may ask the ARC to shrink a 181 * lot for one page allocation. This is problematic because it may 182 * take a long time, thus delaying the page allocation, and because 183 * it may force the ARC to unnecessarily shrink very small. 184 * 185 * Therefore, we limit the amount of data that we say is evictable, 186 * which limits the amount that the shrinker will ask us to evict for 187 * one page allocation attempt. 188 * 189 * In practice, we may be asked to shrink 4x the limit to satisfy one 190 * page allocation, before the kernel's shrinker code gives up on us. 191 * When that happens, we rely on the kernel code to find the pages 192 * that we freed before invoking the OOM killer. This happens in 193 * __alloc_pages_slowpath(), which retries and finds the pages we 194 * freed when it calls get_page_from_freelist(). 195 * 196 * See also the comment above zfs_arc_shrinker_limit. 197 */ 198 int64_t can_free = btop(arc_evictable_memory()); 199 if (current_is_kswapd() && zfs_arc_shrinker_limit) 200 can_free = MIN(can_free, zfs_arc_shrinker_limit); 201 return (can_free); 202 } 203 204 static unsigned long 205 arc_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) 206 { 207 /* The arc is considered warm once reclaim has occurred */ 208 if (unlikely(arc_warm == B_FALSE)) 209 arc_warm = B_TRUE; 210 211 /* 212 * We are experiencing memory pressure which the arc_evict_zthr was 213 * unable to keep up with. Set arc_no_grow to briefly pause ARC 214 * growth to avoid compounding the memory pressure. 215 */ 216 arc_no_grow = B_TRUE; 217 218 /* 219 * Evict the requested number of pages by reducing arc_c and waiting 220 * for the requested amount of data to be evicted. To avoid deadlock 221 * do not wait for eviction if we may be called from ZFS itself (see 222 * kmem_flags_convert() removing __GFP_FS). It may cause excessive 223 * eviction later if many evictions are accumulated, but just skipping 224 * the eviction is not good either if most of memory is used by ARC. 225 */ 226 uint64_t to_free = arc_reduce_target_size(ptob(sc->nr_to_scan)); 227 if (sc->gfp_mask & __GFP_FS) 228 arc_wait_for_eviction(to_free, B_FALSE, B_FALSE); 229 if (current->reclaim_state != NULL) 230 #ifdef HAVE_RECLAIM_STATE_RECLAIMED 231 current->reclaim_state->reclaimed += btop(to_free); 232 #else 233 current->reclaim_state->reclaimed_slab += btop(to_free); 234 #endif 235 236 /* 237 * When direct reclaim is observed it usually indicates a rapid 238 * increase in memory pressure. This occurs because the kswapd 239 * threads were unable to asynchronously keep enough free memory 240 * available. 241 */ 242 if (current_is_kswapd()) { 243 ARCSTAT_BUMP(arcstat_memory_indirect_count); 244 } else { 245 ARCSTAT_BUMP(arcstat_memory_direct_count); 246 } 247 248 return (btop(to_free)); 249 } 250 251 static struct shrinker *arc_shrinker = NULL; 252 253 int 254 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg) 255 { 256 uint64_t free_memory = arc_free_memory(); 257 258 if (free_memory > arc_all_memory() * arc_lotsfree_percent / 100) 259 return (0); 260 261 if (txg > spa->spa_lowmem_last_txg) { 262 spa->spa_lowmem_last_txg = txg; 263 spa->spa_lowmem_page_load = 0; 264 } 265 /* 266 * If we are in pageout, we know that memory is already tight, 267 * the arc is already going to be evicting, so we just want to 268 * continue to let page writes occur as quickly as possible. 269 */ 270 if (current_is_kswapd()) { 271 if (spa->spa_lowmem_page_load > 272 MAX(arc_sys_free / 4, free_memory) / 4) { 273 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim); 274 return (SET_ERROR(ERESTART)); 275 } 276 /* Note: reserve is inflated, so we deflate */ 277 atomic_add_64(&spa->spa_lowmem_page_load, reserve / 8); 278 return (0); 279 } else if (spa->spa_lowmem_page_load > 0 && arc_reclaim_needed()) { 280 /* memory is low, delay before restarting */ 281 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 282 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim); 283 return (SET_ERROR(EAGAIN)); 284 } 285 spa->spa_lowmem_page_load = 0; 286 return (0); 287 } 288 289 static void 290 arc_set_sys_free(uint64_t allmem) 291 { 292 /* 293 * The ARC tries to keep at least this much memory available for the 294 * system. This gives the ARC time to shrink in response to memory 295 * pressure, before running completely out of memory and invoking the 296 * direct-reclaim ARC shrinker. 297 * 298 * This should be more than twice high_wmark_pages(), so that 299 * arc_wait_for_eviction() will wait until at least the 300 * high_wmark_pages() are free (see arc_evict_state_impl()). 301 * 302 * Note: If concurrent allocations consume these pages, there may 303 * still be insufficient free pages, and the OOM killer takes action. 304 * 305 * By setting arc_sys_free large enough, and having 306 * arc_wait_for_eviction() wait until there is at least arc_sys_free/2 307 * free memory, it is much less likely that concurrent allocations can 308 * consume all the memory that was evicted before checking for 309 * OOM. 310 * 311 * It's hard to iterate the zones from a linux kernel module, which 312 * makes it difficult to determine the watermark dynamically. Instead 313 * we compute the maximum high watermark for this system, based 314 * on the amount of memory, using the same method as the kernel uses 315 * to calculate its internal `min_free_kbytes` variable. See 316 * torvalds/linux@ee8eb9a5fe86 for the change in the upper clamp value 317 * from 64M to 256M. 318 */ 319 320 /* 321 * Base wmark_low is 4 * the square root of Kbytes of RAM. 322 */ 323 long wmark = int_sqrt(allmem / 1024 * 16) * 1024; 324 325 /* 326 * Clamp to between 128K and 256/64MB. 327 */ 328 wmark = MAX(wmark, 128 * 1024); 329 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0) 330 wmark = MIN(wmark, 256 * 1024 * 1024); 331 #else 332 wmark = MIN(wmark, 64 * 1024 * 1024); 333 #endif 334 335 /* 336 * watermark_boost can increase the wmark by up to 150%. 337 */ 338 wmark += wmark * 150 / 100; 339 340 /* 341 * arc_sys_free needs to be more than 2x the watermark, because 342 * arc_wait_for_eviction() waits for half of arc_sys_free. Bump this up 343 * to 3x to ensure we're above it. 344 */ 345 arc_sys_free = wmark * 3 + allmem / 32; 346 } 347 348 void 349 arc_lowmem_init(void) 350 { 351 uint64_t allmem = arc_all_memory(); 352 353 /* 354 * Register a shrinker to support synchronous (direct) memory 355 * reclaim from the arc. This is done to prevent kswapd from 356 * swapping out pages when it is preferable to shrink the arc. 357 */ 358 arc_shrinker = spl_register_shrinker("zfs-arc-shrinker", 359 arc_shrinker_count, arc_shrinker_scan, zfs_arc_shrinker_seeks); 360 VERIFY(arc_shrinker); 361 362 arc_set_sys_free(allmem); 363 } 364 365 void 366 arc_lowmem_fini(void) 367 { 368 spl_unregister_shrinker(arc_shrinker); 369 arc_shrinker = NULL; 370 } 371 372 int 373 param_set_arc_u64(const char *buf, zfs_kernel_param_t *kp) 374 { 375 int error; 376 377 error = spl_param_set_u64(buf, kp); 378 if (error < 0) 379 return (SET_ERROR(error)); 380 381 arc_tuning_update(B_TRUE); 382 383 return (0); 384 } 385 386 int 387 param_set_arc_min(const char *buf, zfs_kernel_param_t *kp) 388 { 389 return (param_set_arc_u64(buf, kp)); 390 } 391 392 int 393 param_set_arc_max(const char *buf, zfs_kernel_param_t *kp) 394 { 395 return (param_set_arc_u64(buf, kp)); 396 } 397 398 int 399 param_set_arc_int(const char *buf, zfs_kernel_param_t *kp) 400 { 401 int error; 402 403 error = param_set_int(buf, kp); 404 if (error < 0) 405 return (SET_ERROR(error)); 406 407 arc_tuning_update(B_TRUE); 408 409 return (0); 410 } 411 412 #ifdef CONFIG_MEMORY_HOTPLUG 413 static int 414 arc_hotplug_callback(struct notifier_block *self, unsigned long action, 415 void *arg) 416 { 417 (void) self, (void) arg; 418 uint64_t allmem = arc_all_memory(); 419 if (action != MEM_ONLINE) 420 return (NOTIFY_OK); 421 422 arc_set_limits(allmem); 423 424 #ifdef __LP64__ 425 if (zfs_dirty_data_max_max == 0) 426 zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024, 427 allmem * zfs_dirty_data_max_max_percent / 100); 428 #else 429 if (zfs_dirty_data_max_max == 0) 430 zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024, 431 allmem * zfs_dirty_data_max_max_percent / 100); 432 #endif 433 434 arc_set_sys_free(allmem); 435 return (NOTIFY_OK); 436 } 437 #endif 438 439 void 440 arc_register_hotplug(void) 441 { 442 #ifdef CONFIG_MEMORY_HOTPLUG 443 arc_hotplug_callback_mem_nb.notifier_call = arc_hotplug_callback; 444 /* There is no significance to the value 100 */ 445 arc_hotplug_callback_mem_nb.priority = 100; 446 register_memory_notifier(&arc_hotplug_callback_mem_nb); 447 #endif 448 } 449 450 void 451 arc_unregister_hotplug(void) 452 { 453 #ifdef CONFIG_MEMORY_HOTPLUG 454 unregister_memory_notifier(&arc_hotplug_callback_mem_nb); 455 #endif 456 } 457 458 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, shrinker_limit, INT, ZMOD_RW, 459 "Limit on number of pages that ARC shrinker can reclaim at once"); 460 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, shrinker_seeks, INT, ZMOD_RD, 461 "Relative cost of ARC eviction vs other kernel subsystems"); 462