1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <sys/spa.h>
30 #include <sys/zio.h>
31 #include <sys/spa_impl.h>
32 #include <sys/zio_compress.h>
33 #include <sys/zio_checksum.h>
34 #include <sys/zfs_context.h>
35 #include <sys/arc.h>
36 #include <sys/zfs_refcount.h>
37 #include <sys/vdev.h>
38 #include <sys/vdev_trim.h>
39 #include <sys/vdev_impl.h>
40 #include <sys/dsl_pool.h>
41 #include <sys/multilist.h>
42 #include <sys/abd.h>
43 #include <sys/zil.h>
44 #include <sys/fm/fs/zfs.h>
45 #include <sys/shrinker.h>
46 #include <sys/vmsystm.h>
47 #include <sys/zpl.h>
48 #include <linux/page_compat.h>
49 #include <linux/notifier.h>
50 #include <linux/memory.h>
51 #include <linux/version.h>
52 #include <sys/callb.h>
53 #include <sys/kstat.h>
54 #include <sys/zthr.h>
55 #include <zfs_fletcher.h>
56 #include <sys/arc_impl.h>
57 #include <sys/trace_zfs.h>
58 #include <sys/aggsum.h>
59
60 /*
61 * This is a limit on how many pages the ARC shrinker makes available for
62 * eviction in response to one page allocation attempt. Note that in
63 * practice, the kernel's shrinker can ask us to evict up to about 4x this
64 * for one allocation attempt.
65 *
66 * The default limit of 10,000 (in practice, 160MB per allocation attempt
67 * with 4K pages) limits the amount of time spent attempting to reclaim ARC
68 * memory to less than 100ms per allocation attempt, even with a small
69 * average compressed block size of ~8KB.
70 *
71 * See also the comment in arc_shrinker_count().
72 * Set to 0 to disable limit.
73 */
74 static int zfs_arc_shrinker_limit = 10000;
75
76 /*
77 * Relative cost of ARC eviction, AKA number of seeks needed to restore evicted
78 * page. Bigger values make ARC more precious and evictions smaller comparing
79 * to other kernel subsystems. Value of 4 means parity with page cache,
80 * according to my reading of kernel's do_shrink_slab() and other code.
81 */
82 static int zfs_arc_shrinker_seeks = DEFAULT_SEEKS;
83
84 #ifdef CONFIG_MEMORY_HOTPLUG
85 static struct notifier_block arc_hotplug_callback_mem_nb;
86 #endif
87
88 /*
89 * Return a default max arc size based on the amount of physical memory.
90 * This may be overridden by tuning the zfs_arc_max module parameter.
91 */
92 uint64_t
arc_default_max(uint64_t min,uint64_t allmem)93 arc_default_max(uint64_t min, uint64_t allmem)
94 {
95 uint64_t size;
96
97 if (allmem >= 1 << 30)
98 size = allmem - (1 << 30);
99 else
100 size = min;
101 return (MAX(allmem * 5 / 8, size));
102 }
103
104 /*
105 * Return maximum amount of memory that we could possibly use. Reduced
106 * to half of all memory in user space which is primarily used for testing.
107 */
108 uint64_t
arc_all_memory(void)109 arc_all_memory(void)
110 {
111 #ifdef CONFIG_HIGHMEM
112 return (ptob(zfs_totalram_pages - zfs_totalhigh_pages));
113 #else
114 return (ptob(zfs_totalram_pages));
115 #endif /* CONFIG_HIGHMEM */
116 }
117
118 /*
119 * Return the amount of memory that is considered free. In user space
120 * which is primarily used for testing we pretend that free memory ranges
121 * from 0-20% of all memory.
122 */
123 uint64_t
arc_free_memory(void)124 arc_free_memory(void)
125 {
126 #ifdef CONFIG_HIGHMEM
127 struct sysinfo si;
128 si_meminfo(&si);
129 return (ptob(si.freeram - si.freehigh));
130 #else
131 return (ptob(nr_free_pages() +
132 nr_inactive_file_pages()));
133 #endif /* CONFIG_HIGHMEM */
134 }
135
136 /*
137 * Return the amount of memory that can be consumed before reclaim will be
138 * needed. Positive if there is sufficient free memory, negative indicates
139 * the amount of memory that needs to be freed up.
140 */
141 int64_t
arc_available_memory(void)142 arc_available_memory(void)
143 {
144 return (arc_free_memory() - arc_sys_free);
145 }
146
147 static uint64_t
arc_evictable_memory(void)148 arc_evictable_memory(void)
149 {
150 int64_t asize = aggsum_value(&arc_sums.arcstat_size);
151 uint64_t arc_clean =
152 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_DATA]) +
153 zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) +
154 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_DATA]) +
155 zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
156 uint64_t arc_dirty = MAX((int64_t)asize - (int64_t)arc_clean, 0);
157
158 /*
159 * Scale reported evictable memory in proportion to page cache, cap
160 * at specified min/max.
161 */
162 uint64_t min = (ptob(nr_file_pages()) / 100) * zfs_arc_pc_percent;
163 min = MAX(arc_c_min, MIN(arc_c_max, min));
164
165 if (arc_dirty >= min)
166 return (arc_clean);
167
168 return (MAX((int64_t)asize - (int64_t)min, 0));
169 }
170
171 /*
172 * The _count() function returns the number of free-able objects.
173 * The _scan() function returns the number of objects that were freed.
174 */
175 static unsigned long
arc_shrinker_count(struct shrinker * shrink,struct shrink_control * sc)176 arc_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
177 {
178 /*
179 * The kernel's shrinker code may not understand how many pages the
180 * ARC's callback actually frees, so it may ask the ARC to shrink a
181 * lot for one page allocation. This is problematic because it may
182 * take a long time, thus delaying the page allocation, and because
183 * it may force the ARC to unnecessarily shrink very small.
184 *
185 * Therefore, we limit the amount of data that we say is evictable,
186 * which limits the amount that the shrinker will ask us to evict for
187 * one page allocation attempt.
188 *
189 * In practice, we may be asked to shrink 4x the limit to satisfy one
190 * page allocation, before the kernel's shrinker code gives up on us.
191 * When that happens, we rely on the kernel code to find the pages
192 * that we freed before invoking the OOM killer. This happens in
193 * __alloc_pages_slowpath(), which retries and finds the pages we
194 * freed when it calls get_page_from_freelist().
195 *
196 * See also the comment above zfs_arc_shrinker_limit.
197 */
198 int64_t can_free = btop(arc_evictable_memory());
199 if (current_is_kswapd() && zfs_arc_shrinker_limit)
200 can_free = MIN(can_free, zfs_arc_shrinker_limit);
201 return (can_free);
202 }
203
204 static unsigned long
arc_shrinker_scan(struct shrinker * shrink,struct shrink_control * sc)205 arc_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
206 {
207 /* The arc is considered warm once reclaim has occurred */
208 if (unlikely(arc_warm == B_FALSE))
209 arc_warm = B_TRUE;
210
211 /*
212 * We are experiencing memory pressure which the arc_evict_zthr was
213 * unable to keep up with. Set arc_no_grow to briefly pause ARC
214 * growth to avoid compounding the memory pressure.
215 */
216 arc_no_grow = B_TRUE;
217
218 /*
219 * Evict the requested number of pages by reducing arc_c and waiting
220 * for the requested amount of data to be evicted. To avoid deadlock
221 * do not wait for eviction if we may be called from ZFS itself (see
222 * kmem_flags_convert() removing __GFP_FS). It may cause excessive
223 * eviction later if many evictions are accumulated, but just skipping
224 * the eviction is not good either if most of memory is used by ARC.
225 */
226 uint64_t to_free = arc_reduce_target_size(ptob(sc->nr_to_scan));
227 if (sc->gfp_mask & __GFP_FS)
228 arc_wait_for_eviction(to_free, B_FALSE, B_FALSE);
229 if (current->reclaim_state != NULL)
230 #ifdef HAVE_RECLAIM_STATE_RECLAIMED
231 current->reclaim_state->reclaimed += btop(to_free);
232 #else
233 current->reclaim_state->reclaimed_slab += btop(to_free);
234 #endif
235
236 /*
237 * When direct reclaim is observed it usually indicates a rapid
238 * increase in memory pressure. This occurs because the kswapd
239 * threads were unable to asynchronously keep enough free memory
240 * available.
241 */
242 if (current_is_kswapd()) {
243 ARCSTAT_BUMP(arcstat_memory_indirect_count);
244 } else {
245 ARCSTAT_BUMP(arcstat_memory_direct_count);
246 }
247
248 return (btop(to_free));
249 }
250
251 static struct shrinker *arc_shrinker = NULL;
252
253 int
arc_memory_throttle(spa_t * spa,uint64_t reserve,uint64_t txg)254 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg)
255 {
256 uint64_t free_memory = arc_free_memory();
257
258 if (free_memory > arc_all_memory() * arc_lotsfree_percent / 100)
259 return (0);
260
261 if (txg > spa->spa_lowmem_last_txg) {
262 spa->spa_lowmem_last_txg = txg;
263 spa->spa_lowmem_page_load = 0;
264 }
265 /*
266 * If we are in pageout, we know that memory is already tight,
267 * the arc is already going to be evicting, so we just want to
268 * continue to let page writes occur as quickly as possible.
269 */
270 if (current_is_kswapd()) {
271 if (spa->spa_lowmem_page_load >
272 MAX(arc_sys_free / 4, free_memory) / 4) {
273 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim);
274 return (SET_ERROR(ERESTART));
275 }
276 /* Note: reserve is inflated, so we deflate */
277 atomic_add_64(&spa->spa_lowmem_page_load, reserve / 8);
278 return (0);
279 } else if (spa->spa_lowmem_page_load > 0 && arc_reclaim_needed()) {
280 /* memory is low, delay before restarting */
281 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
282 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim);
283 return (SET_ERROR(EAGAIN));
284 }
285 spa->spa_lowmem_page_load = 0;
286 return (0);
287 }
288
289 static void
arc_set_sys_free(uint64_t allmem)290 arc_set_sys_free(uint64_t allmem)
291 {
292 /*
293 * The ARC tries to keep at least this much memory available for the
294 * system. This gives the ARC time to shrink in response to memory
295 * pressure, before running completely out of memory and invoking the
296 * direct-reclaim ARC shrinker.
297 *
298 * This should be more than twice high_wmark_pages(), so that
299 * arc_wait_for_eviction() will wait until at least the
300 * high_wmark_pages() are free (see arc_evict_state_impl()).
301 *
302 * Note: If concurrent allocations consume these pages, there may
303 * still be insufficient free pages, and the OOM killer takes action.
304 *
305 * By setting arc_sys_free large enough, and having
306 * arc_wait_for_eviction() wait until there is at least arc_sys_free/2
307 * free memory, it is much less likely that concurrent allocations can
308 * consume all the memory that was evicted before checking for
309 * OOM.
310 *
311 * It's hard to iterate the zones from a linux kernel module, which
312 * makes it difficult to determine the watermark dynamically. Instead
313 * we compute the maximum high watermark for this system, based
314 * on the amount of memory, using the same method as the kernel uses
315 * to calculate its internal `min_free_kbytes` variable. See
316 * torvalds/linux@ee8eb9a5fe86 for the change in the upper clamp value
317 * from 64M to 256M.
318 */
319
320 /*
321 * Base wmark_low is 4 * the square root of Kbytes of RAM.
322 */
323 long wmark = int_sqrt(allmem / 1024 * 16) * 1024;
324
325 /*
326 * Clamp to between 128K and 256/64MB.
327 */
328 wmark = MAX(wmark, 128 * 1024);
329 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
330 wmark = MIN(wmark, 256 * 1024 * 1024);
331 #else
332 wmark = MIN(wmark, 64 * 1024 * 1024);
333 #endif
334
335 /*
336 * watermark_boost can increase the wmark by up to 150%.
337 */
338 wmark += wmark * 150 / 100;
339
340 /*
341 * arc_sys_free needs to be more than 2x the watermark, because
342 * arc_wait_for_eviction() waits for half of arc_sys_free. Bump this up
343 * to 3x to ensure we're above it.
344 */
345 arc_sys_free = wmark * 3 + allmem / 32;
346 }
347
348 void
arc_lowmem_init(void)349 arc_lowmem_init(void)
350 {
351 uint64_t allmem = arc_all_memory();
352
353 /*
354 * Register a shrinker to support synchronous (direct) memory
355 * reclaim from the arc. This is done to prevent kswapd from
356 * swapping out pages when it is preferable to shrink the arc.
357 */
358 arc_shrinker = spl_register_shrinker("zfs-arc-shrinker",
359 arc_shrinker_count, arc_shrinker_scan, zfs_arc_shrinker_seeks);
360 VERIFY(arc_shrinker);
361
362 arc_set_sys_free(allmem);
363 }
364
365 void
arc_lowmem_fini(void)366 arc_lowmem_fini(void)
367 {
368 spl_unregister_shrinker(arc_shrinker);
369 arc_shrinker = NULL;
370 }
371
372 int
param_set_arc_u64(const char * buf,zfs_kernel_param_t * kp)373 param_set_arc_u64(const char *buf, zfs_kernel_param_t *kp)
374 {
375 int error;
376
377 error = spl_param_set_u64(buf, kp);
378 if (error < 0)
379 return (SET_ERROR(error));
380
381 arc_tuning_update(B_TRUE);
382
383 return (0);
384 }
385
386 int
param_set_arc_min(const char * buf,zfs_kernel_param_t * kp)387 param_set_arc_min(const char *buf, zfs_kernel_param_t *kp)
388 {
389 return (param_set_arc_u64(buf, kp));
390 }
391
392 int
param_set_arc_max(const char * buf,zfs_kernel_param_t * kp)393 param_set_arc_max(const char *buf, zfs_kernel_param_t *kp)
394 {
395 return (param_set_arc_u64(buf, kp));
396 }
397
398 int
param_set_arc_int(const char * buf,zfs_kernel_param_t * kp)399 param_set_arc_int(const char *buf, zfs_kernel_param_t *kp)
400 {
401 int error;
402
403 error = param_set_int(buf, kp);
404 if (error < 0)
405 return (SET_ERROR(error));
406
407 arc_tuning_update(B_TRUE);
408
409 return (0);
410 }
411
412 #ifdef CONFIG_MEMORY_HOTPLUG
413 static int
arc_hotplug_callback(struct notifier_block * self,unsigned long action,void * arg)414 arc_hotplug_callback(struct notifier_block *self, unsigned long action,
415 void *arg)
416 {
417 (void) self, (void) arg;
418 uint64_t allmem = arc_all_memory();
419 if (action != MEM_ONLINE)
420 return (NOTIFY_OK);
421
422 arc_set_limits(allmem);
423
424 #ifdef __LP64__
425 if (zfs_dirty_data_max_max == 0)
426 zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
427 allmem * zfs_dirty_data_max_max_percent / 100);
428 #else
429 if (zfs_dirty_data_max_max == 0)
430 zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
431 allmem * zfs_dirty_data_max_max_percent / 100);
432 #endif
433
434 arc_set_sys_free(allmem);
435 return (NOTIFY_OK);
436 }
437 #endif
438
439 void
arc_register_hotplug(void)440 arc_register_hotplug(void)
441 {
442 #ifdef CONFIG_MEMORY_HOTPLUG
443 arc_hotplug_callback_mem_nb.notifier_call = arc_hotplug_callback;
444 /* There is no significance to the value 100 */
445 arc_hotplug_callback_mem_nb.priority = 100;
446 register_memory_notifier(&arc_hotplug_callback_mem_nb);
447 #endif
448 }
449
450 void
arc_unregister_hotplug(void)451 arc_unregister_hotplug(void)
452 {
453 #ifdef CONFIG_MEMORY_HOTPLUG
454 unregister_memory_notifier(&arc_hotplug_callback_mem_nb);
455 #endif
456 }
457
458 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, shrinker_limit, INT, ZMOD_RW,
459 "Limit on number of pages that ARC shrinker can reclaim at once");
460 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, shrinker_seeks, INT, ZMOD_RD,
461 "Relative cost of ARC eviction vs other kernel subsystems");
462