/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * sun4u Memory Scrubbing * * On detection of a correctable memory ECC error, the sun4u kernel * returns the corrected data to the requester and re-writes it * to memory (DRAM). So if the correctable error was transient, * the read has effectively been cleaned (scrubbed) from memory. * * Scrubbing thus reduces the likelyhood that multiple transient errors * will occur in the same memory word, making uncorrectable errors due * to transients less likely. * * Thus is born the desire that every memory location be periodically * accessed. * * This file implements a memory scrubbing thread. This scrubber * guarantees that all of physical memory is accessed periodically * (memscrub_period_sec -- 12 hours). * * It attempts to do this as unobtrusively as possible. The thread * schedules itself to wake up at an interval such that if it reads * memscrub_span_pages (32MB) on each wakeup, it will read all of physical * memory in in memscrub_period_sec (12 hours). * * The scrubber uses the block load and prefetch hardware to read memory * @ 1300MB/s, so it reads spans of 32MB in 0.025 seconds. Unlike the * original sun4d scrubber the sun4u scrubber does not read ahead if the * system is idle because we can read memory very efficently. * * The scrubber maintains a private copy of the phys_install memory list * to keep track of what memory should be scrubbed. * * The global routines memscrub_add_span() and memscrub_delete_span() are * used to add and delete from this list. If hotplug memory is later * supported these two routines can be used to notify the scrubber of * memory configuration changes. * * The following parameters can be set via /etc/system * * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (8MB) * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours) * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI) * memscrub_delay_start_sec = (5 minutes) * memscrub_verbose = (0) * memscrub_override_ticks = (1 tick) * disable_memscrub = (0) * pause_memscrub = (0) * read_all_memscrub = (0) * * The scrubber will print NOTICE messages of what it is doing if * "memscrub_verbose" is set. * * If the scrubber's sleep time calculation drops to zero ticks, * memscrub_override_ticks will be used as the sleep time instead. The * sleep time should only drop to zero on a system with over 131.84 * terabytes of memory, or where the default scrubber parameters have * been adjusted. For example, reducing memscrub_span_pages or * memscrub_period_sec causes the sleep time to drop to zero with less * memory. Note that since the sleep time is calculated in clock ticks, * using hires clock ticks allows for more memory before the sleep time * becomes zero. * * The scrubber will exit (or never be started) if it finds the variable * "disable_memscrub" set. * * The scrubber will pause (not read memory) when "pause_memscrub" * is set. It will check the state of pause_memscrub at each wakeup * period. The scrubber will not make up for lost time. If you * pause the scrubber for a prolonged period of time you can use * the "read_all_memscrub" switch (see below) to catch up. In addition, * pause_memscrub is used internally by the post memory DR callbacks. * It is set for the small period of time during which the callbacks * are executing. This ensures "memscrub_lock" will be released, * allowing the callbacks to finish. * * The scrubber will read all memory if "read_all_memscrub" is set. * The normal span read will also occur during the wakeup. * * MEMSCRUB_MIN_PAGES (32MB) is the minimum amount of memory a system * must have before we'll start the scrubber. * * MEMSCRUB_DFL_SPAN_PAGES (32MB) is based on the guess that 0.025 sec * is a "good" amount of minimum time for the thread to run at a time. * * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess -- * twice the frequency the hardware folk estimated would be necessary. * * MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI) is based on the assumption * that the scurbber should get its fair share of time (since it * is short). At a priority of 0 the scrubber will be starved. */ #include <sys/systm.h> /* timeout, types, t_lock */ #include <sys/cmn_err.h> #include <sys/sysmacros.h> /* MIN */ #include <sys/memlist.h> /* memlist */ #include <sys/mem_config.h> /* memory add/delete */ #include <sys/kmem.h> /* KMEM_NOSLEEP */ #include <sys/cpuvar.h> /* ncpus_online */ #include <sys/debug.h> /* ASSERTs */ #include <sys/machsystm.h> /* lddphys */ #include <sys/cpu_module.h> /* vtag_flushpage */ #include <sys/kstat.h> #include <sys/atomic.h> /* atomic_add_32 */ #include <vm/hat.h> #include <vm/seg_kmem.h> #include <vm/hat_sfmmu.h> /* XXX FIXME - delete */ #include <sys/time.h> #include <sys/callb.h> /* CPR callback */ #include <sys/ontrap.h> /* * Should really have paddr_t defined, but it is broken. Use * ms_paddr_t in the meantime to make the code cleaner */ typedef uint64_t ms_paddr_t; /* * Global Routines: */ int memscrub_add_span(pfn_t pfn, pgcnt_t pages); int memscrub_delete_span(pfn_t pfn, pgcnt_t pages); int memscrub_init(void); void memscrub_induced_error(void); /* * Global Data: */ /* * scrub if we have at least this many pages */ #define MEMSCRUB_MIN_PAGES (32 * 1024 * 1024 / PAGESIZE) /* * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC */ #define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */ /* * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration */ #define MEMSCRUB_DFL_SPAN_PAGES ((32 * 1024 * 1024) / PAGESIZE) /* * almost anything is higher priority than scrubbing */ #define MEMSCRUB_DFL_THREAD_PRI MINCLSYSPRI /* * size used when scanning memory */ #define MEMSCRUB_BLOCK_SIZE 256 #define MEMSCRUB_BLOCK_SIZE_SHIFT 8 /* log2(MEMSCRUB_BLOCK_SIZE) */ #define MEMSCRUB_BLOCKS_PER_PAGE (PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT) #define MEMSCRUB_BPP4M MMU_PAGESIZE4M >> MEMSCRUB_BLOCK_SIZE_SHIFT #define MEMSCRUB_BPP512K MMU_PAGESIZE512K >> MEMSCRUB_BLOCK_SIZE_SHIFT #define MEMSCRUB_BPP64K MMU_PAGESIZE64K >> MEMSCRUB_BLOCK_SIZE_SHIFT #define MEMSCRUB_BPP MMU_PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT /* * This message indicates that we have exceeded the limitations of * the memscrubber. See the comments above regarding what would * cause the sleep time to become zero. In DEBUG mode, this message * is logged on the console and in the messages file. In non-DEBUG * mode, it is only logged in the messages file. */ #ifdef DEBUG #define MEMSCRUB_OVERRIDE_MSG "Memory scrubber sleep time is zero " \ "seconds, consuming entire CPU." #else #define MEMSCRUB_OVERRIDE_MSG "!Memory scrubber sleep time is zero " \ "seconds, consuming entire CPU." #endif /* DEBUG */ /* * we can patch these defaults in /etc/system if necessary */ uint_t disable_memscrub = 0; uint_t pause_memscrub = 0; uint_t read_all_memscrub = 0; uint_t memscrub_verbose = 0; uint_t memscrub_all_idle = 0; uint_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES; uint_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC; uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI; uint_t memscrub_delay_start_sec = 5 * 60; uint_t memscrub_override_ticks = 1; /* * Static Routines */ static void memscrubber(void); static void memscrub_cleanup(void); static int memscrub_add_span_gen(pfn_t, pgcnt_t, struct memlist **, uint_t *); static int memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp); static void memscrub_scan(uint_t blks, ms_paddr_t src); /* * Static Data */ static struct memlist *memscrub_memlist; static uint_t memscrub_phys_pages; static kcondvar_t memscrub_cv; static kmutex_t memscrub_lock; /* * memscrub_lock protects memscrub_memlist, interval_ticks, cprinfo, ... */ static void memscrub_init_mem_config(void); static void memscrub_uninit_mem_config(void); /* * Linked list of memscrub aware spans having retired pages. * Currently enabled only on sun4u USIII-based platforms. */ typedef struct memscrub_page_retire_span { ms_paddr_t address; struct memscrub_page_retire_span *next; } memscrub_page_retire_span_t; static memscrub_page_retire_span_t *memscrub_page_retire_span_list = NULL; static void memscrub_page_retire_span_add(ms_paddr_t); static void memscrub_page_retire_span_delete(ms_paddr_t); static int memscrub_page_retire_span_search(ms_paddr_t); static void memscrub_page_retire_span_list_update(void); /* * add_to_page_retire_list: Set by cpu_async_log_err() routine * by calling memscrub_induced_error() when CE/UE occurs on a retired * page due to memscrub reading. Cleared by memscrub after updating * global page retire span list. Piggybacking on protection of * memscrub_lock, which is held during set and clear. * Note: When cpu_async_log_err() calls memscrub_induced_error(), it is running * on softint context, which gets fired on a cpu memscrub thread currently * running. Memscrub thread has affinity set during memscrub_read(), hence * migration to new cpu not expected. */ static int add_to_page_retire_list = 0; /* * Keep track of some interesting statistics */ static struct memscrub_kstats { kstat_named_t done_early; /* ahead of schedule */ kstat_named_t early_sec; /* by cumulative num secs */ kstat_named_t done_late; /* behind schedule */ kstat_named_t late_sec; /* by cumulative num secs */ kstat_named_t interval_ticks; /* num ticks between intervals */ kstat_named_t force_run; /* forced to run, non-timeout */ kstat_named_t errors_found; /* num errors found by memscrub */ } memscrub_counts = { { "done_early", KSTAT_DATA_UINT32 }, { "early_sec", KSTAT_DATA_UINT32 }, { "done_late", KSTAT_DATA_UINT32 }, { "late_sec", KSTAT_DATA_UINT32 }, { "interval_ticks", KSTAT_DATA_UINT32 }, { "force_run", KSTAT_DATA_UINT32 }, { "errors_found", KSTAT_DATA_UINT32 }, }; #define MEMSCRUB_STAT_INC(stat) memscrub_counts.stat.value.ui32++ #define MEMSCRUB_STAT_SET(stat, val) memscrub_counts.stat.value.ui32 = (val) #define MEMSCRUB_STAT_NINC(stat, val) memscrub_counts.stat.value.ui32 += (val) static struct kstat *memscrub_ksp = (struct kstat *)NULL; static timeout_id_t memscrub_tid = 0; /* keep track of timeout id */ /* * create memscrub_memlist from phys_install list * initialize locks, set memscrub_phys_pages. */ int memscrub_init(void) { struct memlist *src; /* * only startup the scrubber if we have a minimum * number of pages */ if (physinstalled >= MEMSCRUB_MIN_PAGES) { /* * initialize locks */ mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL); cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL); /* * copy phys_install to memscrub_memlist */ for (src = phys_install; src; src = src->ml_next) { if (memscrub_add_span( (pfn_t)(src->ml_address >> PAGESHIFT), (pgcnt_t)(src->ml_size >> PAGESHIFT))) { memscrub_cleanup(); return (-1); } } /* * initialize kstats */ memscrub_ksp = kstat_create("unix", 0, "memscrub_kstat", "misc", KSTAT_TYPE_NAMED, sizeof (memscrub_counts) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE); if (memscrub_ksp) { memscrub_ksp->ks_data = (void *)&memscrub_counts; kstat_install(memscrub_ksp); } else { cmn_err(CE_NOTE, "Memscrubber cannot create kstats\n"); } /* * create memscrubber thread */ (void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0, TS_RUN, memscrub_thread_pri); /* * We don't want call backs changing the list * if there is no thread running. We do not * attempt to deal with stopping/starting scrubbing * on memory size changes. */ memscrub_init_mem_config(); } return (0); } static void memscrub_cleanup(void) { memscrub_uninit_mem_config(); while (memscrub_memlist) { (void) memscrub_delete_span( (pfn_t)(memscrub_memlist->ml_address >> PAGESHIFT), (pgcnt_t)(memscrub_memlist->ml_size >> PAGESHIFT)); } if (memscrub_ksp) kstat_delete(memscrub_ksp); cv_destroy(&memscrub_cv); mutex_destroy(&memscrub_lock); } #ifdef MEMSCRUB_DEBUG static void memscrub_printmemlist(char *title, struct memlist *listp) { struct memlist *list; cmn_err(CE_CONT, "%s:\n", title); for (list = listp; list; list = list->ml_next) { cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n", list->ml_address, list->ml_size); } } #endif /* MEMSCRUB_DEBUG */ /* ARGSUSED */ static void memscrub_wakeup(void *c) { /* * grab mutex to guarantee that our wakeup call * arrives after we go to sleep -- so we can't sleep forever. */ mutex_enter(&memscrub_lock); cv_signal(&memscrub_cv); mutex_exit(&memscrub_lock); } /* * provide an interface external to the memscrubber * which will force the memscrub thread to run vs. * waiting for the timeout, if one is set */ void memscrub_run(void) { MEMSCRUB_STAT_INC(force_run); if (memscrub_tid) { (void) untimeout(memscrub_tid); memscrub_wakeup((void *)NULL); } } /* * this calculation doesn't account for the time * that the actual scan consumes -- so we'd fall * slightly behind schedule with this interval. * It's very small. */ static uint_t compute_interval_ticks(void) { /* * We use msp_safe mpp_safe below to insure somebody * doesn't set memscrub_span_pages or memscrub_phys_pages * to 0 on us. */ static uint_t msp_safe, mpp_safe; static uint_t interval_ticks, period_ticks; msp_safe = memscrub_span_pages; mpp_safe = memscrub_phys_pages; period_ticks = memscrub_period_sec * hz; interval_ticks = period_ticks; ASSERT(mutex_owned(&memscrub_lock)); if ((msp_safe != 0) && (mpp_safe != 0)) { if (memscrub_phys_pages <= msp_safe) { interval_ticks = period_ticks; } else { interval_ticks = (period_ticks / (mpp_safe / msp_safe)); } } return (interval_ticks); } void memscrubber(void) { ms_paddr_t address, addr; time_t deadline; pgcnt_t pages; uint_t reached_end = 1; uint_t paused_message = 0; uint_t interval_ticks = 0; uint_t sleep_warn_printed = 0; callb_cpr_t cprinfo; /* * notify CPR of our existence */ CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub"); mutex_enter(&memscrub_lock); if (memscrub_memlist == NULL) { cmn_err(CE_WARN, "memscrub_memlist not initialized."); goto memscrub_exit; } address = memscrub_memlist->ml_address; deadline = gethrestime_sec() + memscrub_delay_start_sec; for (;;) { if (disable_memscrub) break; /* * compute interval_ticks */ interval_ticks = compute_interval_ticks(); /* * If the calculated sleep time is zero, and pause_memscrub * has been set, make sure we sleep so that another thread * can acquire memscrub_lock. */ if (interval_ticks == 0 && pause_memscrub) { interval_ticks = hz; } /* * And as a fail safe, under normal non-paused operation, do * not allow the sleep time to be zero. */ if (interval_ticks == 0) { interval_ticks = memscrub_override_ticks; if (!sleep_warn_printed) { cmn_err(CE_NOTE, MEMSCRUB_OVERRIDE_MSG); sleep_warn_printed = 1; } } MEMSCRUB_STAT_SET(interval_ticks, interval_ticks); /* * Did we just reach the end of memory? If we are at the * end of memory, delay end of memory processing until * pause_memscrub is not set. */ if (reached_end && !pause_memscrub) { time_t now = gethrestime_sec(); if (now >= deadline) { MEMSCRUB_STAT_INC(done_late); MEMSCRUB_STAT_NINC(late_sec, now - deadline); /* * past deadline, start right away */ interval_ticks = 0; deadline = now + memscrub_period_sec; } else { /* * we finished ahead of schedule. * wait till previous deadline before re-start. */ interval_ticks = (deadline - now) * hz; MEMSCRUB_STAT_INC(done_early); MEMSCRUB_STAT_NINC(early_sec, deadline - now); deadline += memscrub_period_sec; } reached_end = 0; sleep_warn_printed = 0; } if (interval_ticks != 0) { /* * it is safe from our standpoint for CPR to * suspend the system */ CALLB_CPR_SAFE_BEGIN(&cprinfo); /* * hit the snooze bar */ memscrub_tid = timeout(memscrub_wakeup, NULL, interval_ticks); /* * go to sleep */ cv_wait(&memscrub_cv, &memscrub_lock); /* * at this point, no timeout should be set */ memscrub_tid = 0; /* * we need to goto work and will be modifying * our internal state and mapping/unmapping * TTEs */ CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock); } if (memscrub_phys_pages == 0) { cmn_err(CE_WARN, "Memory scrubber has 0 pages to read"); goto memscrub_exit; } if (!pause_memscrub) { if (paused_message) { paused_message = 0; if (memscrub_verbose) cmn_err(CE_NOTE, "Memory scrubber " "resuming"); } if (read_all_memscrub) { if (memscrub_verbose) cmn_err(CE_NOTE, "Memory scrubber " "reading all memory per request"); addr = memscrub_memlist->ml_address; reached_end = 0; while (!reached_end) { if (disable_memscrub) break; pages = memscrub_phys_pages; reached_end = memscrub_verify_span( &addr, &pages); memscrub_scan(pages * MEMSCRUB_BLOCKS_PER_PAGE, addr); addr += ((uint64_t)pages * PAGESIZE); } read_all_memscrub = 0; } /* * read 1 span */ pages = memscrub_span_pages; if (disable_memscrub) break; /* * determine physical address range */ reached_end = memscrub_verify_span(&address, &pages); memscrub_scan(pages * MEMSCRUB_BLOCKS_PER_PAGE, address); address += ((uint64_t)pages * PAGESIZE); } if (pause_memscrub && !paused_message) { paused_message = 1; if (memscrub_verbose) cmn_err(CE_NOTE, "Memory scrubber paused"); } } memscrub_exit: cmn_err(CE_NOTE, "Memory scrubber exiting"); CALLB_CPR_EXIT(&cprinfo); memscrub_cleanup(); thread_exit(); /* NOTREACHED */ } /* * condition address and size * such that they span legal physical addresses. * * when appropriate, address will be rounded up to start of next * struct memlist, and pages will be rounded down to the end of the * memlist size. * * returns 1 if reached end of list, else returns 0. */ static int memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp) { struct memlist *mlp; ms_paddr_t address = *addrp; uint64_t bytes = (uint64_t)*pagesp * PAGESIZE; uint64_t bytes_remaining; int reached_end = 0; ASSERT(mutex_owned(&memscrub_lock)); /* * find memlist struct that contains addrp * assumes memlist is sorted by ascending address. */ for (mlp = memscrub_memlist; mlp != NULL; mlp = mlp->ml_next) { /* * if before this chunk, round up to beginning */ if (address < mlp->ml_address) { address = mlp->ml_address; break; } /* * if before end of chunk, then we found it */ if (address < (mlp->ml_address + mlp->ml_size)) break; /* else go to next struct memlist */ } /* * if we hit end of list, start at beginning */ if (mlp == NULL) { mlp = memscrub_memlist; address = mlp->ml_address; } /* * now we have legal address, and its mlp, condition bytes */ bytes_remaining = (mlp->ml_address + mlp->ml_size) - address; if (bytes > bytes_remaining) bytes = bytes_remaining; /* * will this span take us to end of list? */ if ((mlp->ml_next == NULL) && ((mlp->ml_address + mlp->ml_size) == (address + bytes))) reached_end = 1; /* return values */ *addrp = address; *pagesp = bytes / PAGESIZE; return (reached_end); } /* * add a span to the memscrub list * add to memscrub_phys_pages */ int memscrub_add_span(pfn_t pfn, pgcnt_t pages) { #ifdef MEMSCRUB_DEBUG ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT; uint64_t bytes = (uint64_t)pages << PAGESHIFT; #endif /* MEMSCRUB_DEBUG */ int retval; mutex_enter(&memscrub_lock); #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist before", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx" " size: 0x%llx\n", address, bytes); #endif /* MEMSCRUB_DEBUG */ retval = memscrub_add_span_gen(pfn, pages, &memscrub_memlist, &memscrub_phys_pages); #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist after", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); #endif /* MEMSCRUB_DEBUG */ mutex_exit(&memscrub_lock); return (retval); } static int memscrub_add_span_gen( pfn_t pfn, pgcnt_t pages, struct memlist **list, uint_t *npgs) { ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT; uint64_t bytes = (uint64_t)pages << PAGESHIFT; struct memlist *dst; struct memlist *prev, *next; int retval = 0; /* * allocate a new struct memlist */ dst = (struct memlist *) kmem_alloc(sizeof (struct memlist), KM_NOSLEEP); if (dst == NULL) { retval = -1; goto add_done; } dst->ml_address = address; dst->ml_size = bytes; /* * first insert */ if (*list == NULL) { dst->ml_prev = NULL; dst->ml_next = NULL; *list = dst; goto add_done; } /* * insert into sorted list */ for (prev = NULL, next = *list; next != NULL; prev = next, next = next->ml_next) { if (address > (next->ml_address + next->ml_size)) continue; /* * else insert here */ /* * prepend to next */ if ((address + bytes) == next->ml_address) { kmem_free(dst, sizeof (struct memlist)); next->ml_address = address; next->ml_size += bytes; goto add_done; } /* * append to next */ if (address == (next->ml_address + next->ml_size)) { kmem_free(dst, sizeof (struct memlist)); if (next->ml_next) { /* * don't overlap with next->ml_next */ if ((address + bytes) > next->ml_next->ml_address) { retval = -1; goto add_done; } /* * concatenate next and next->ml_next */ if ((address + bytes) == next->ml_next->ml_address) { struct memlist *mlp = next->ml_next; if (next == *list) *list = next->ml_next; mlp->ml_address = next->ml_address; mlp->ml_size += next->ml_size; mlp->ml_size += bytes; if (next->ml_prev) next->ml_prev->ml_next = mlp; mlp->ml_prev = next->ml_prev; kmem_free(next, sizeof (struct memlist)); goto add_done; } } next->ml_size += bytes; goto add_done; } /* don't overlap with next */ if ((address + bytes) > next->ml_address) { retval = -1; kmem_free(dst, sizeof (struct memlist)); goto add_done; } /* * insert before next */ dst->ml_prev = prev; dst->ml_next = next; next->ml_prev = dst; if (prev == NULL) { *list = dst; } else { prev->ml_next = dst; } goto add_done; } /* end for */ /* * end of list, prev is valid and next is NULL */ prev->ml_next = dst; dst->ml_prev = prev; dst->ml_next = NULL; add_done: if (retval != -1) *npgs += pages; return (retval); } /* * delete a span from the memscrub list * subtract from memscrub_phys_pages */ int memscrub_delete_span(pfn_t pfn, pgcnt_t pages) { ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT; uint64_t bytes = (uint64_t)pages << PAGESHIFT; struct memlist *dst, *next; int retval = 0; mutex_enter(&memscrub_lock); #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist Before", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); cmn_err(CE_CONT, "memscrub_delete_span: 0x%llx 0x%llx\n", address, bytes); #endif /* MEMSCRUB_DEBUG */ /* * find struct memlist containing page */ for (next = memscrub_memlist; next != NULL; next = next->ml_next) { if ((address >= next->ml_address) && (address < next->ml_address + next->ml_size)) break; } /* * if start address not in list */ if (next == NULL) { retval = -1; goto delete_done; } /* * error if size goes off end of this struct memlist */ if (address + bytes > next->ml_address + next->ml_size) { retval = -1; goto delete_done; } /* * pages at beginning of struct memlist */ if (address == next->ml_address) { /* * if start & size match, delete from list */ if (bytes == next->ml_size) { if (next == memscrub_memlist) memscrub_memlist = next->ml_next; if (next->ml_prev != NULL) next->ml_prev->ml_next = next->ml_next; if (next->ml_next != NULL) next->ml_next->ml_prev = next->ml_prev; kmem_free(next, sizeof (struct memlist)); } else { /* * increment start address by bytes */ next->ml_address += bytes; next->ml_size -= bytes; } goto delete_done; } /* * pages at end of struct memlist */ if (address + bytes == next->ml_address + next->ml_size) { /* * decrement size by bytes */ next->ml_size -= bytes; goto delete_done; } /* * delete a span in the middle of the struct memlist */ { /* * create a new struct memlist */ dst = (struct memlist *) kmem_alloc(sizeof (struct memlist), KM_NOSLEEP); if (dst == NULL) { retval = -1; goto delete_done; } /* * existing struct memlist gets address * and size up to pfn */ dst->ml_address = address + bytes; dst->ml_size = (next->ml_address + next->ml_size) - dst->ml_address; next->ml_size = address - next->ml_address; /* * new struct memlist gets address starting * after pfn, until end */ /* * link in new memlist after old */ dst->ml_next = next->ml_next; dst->ml_prev = next; if (next->ml_next != NULL) next->ml_next->ml_prev = dst; next->ml_next = dst; } delete_done: if (retval != -1) { memscrub_phys_pages -= pages; if (memscrub_phys_pages == 0) disable_memscrub = 1; } #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist After", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); #endif /* MEMSCRUB_DEBUG */ mutex_exit(&memscrub_lock); return (retval); } static void memscrub_scan(uint_t blks, ms_paddr_t src) { uint_t psz, bpp, pgsread; pfn_t pfn; ms_paddr_t pa; caddr_t va; on_trap_data_t otd; int scan_mmu_pagesize = 0; int retired_pages = 0; extern void memscrub_read(caddr_t src, uint_t blks); ASSERT(mutex_owned(&memscrub_lock)); pgsread = 0; pa = src; if (memscrub_page_retire_span_list != NULL) { if (memscrub_page_retire_span_search(src)) { /* retired pages in current span */ scan_mmu_pagesize = 1; } } #ifdef MEMSCRUB_DEBUG cmn_err(CE_NOTE, "scan_mmu_pagesize = %d\n" scan_mmu_pagesize); #endif /* MEMSCRUB_DEBUG */ while (blks != 0) { /* Ensure the PA is properly aligned */ if (((pa & MMU_PAGEMASK4M) == pa) && (blks >= MEMSCRUB_BPP4M)) { psz = MMU_PAGESIZE4M; bpp = MEMSCRUB_BPP4M; } else if (((pa & MMU_PAGEMASK512K) == pa) && (blks >= MEMSCRUB_BPP512K)) { psz = MMU_PAGESIZE512K; bpp = MEMSCRUB_BPP512K; } else if (((pa & MMU_PAGEMASK64K) == pa) && (blks >= MEMSCRUB_BPP64K)) { psz = MMU_PAGESIZE64K; bpp = MEMSCRUB_BPP64K; } else if ((pa & MMU_PAGEMASK) == pa) { psz = MMU_PAGESIZE; bpp = MEMSCRUB_BPP; } else { if (memscrub_verbose) { cmn_err(CE_NOTE, "Memory scrubber ignoring " "non-page aligned block starting at 0x%" PRIx64, src); } return; } if (blks < bpp) bpp = blks; #ifdef MEMSCRUB_DEBUG cmn_err(CE_NOTE, "Going to run psz=%x, " "bpp=%x pa=%llx\n", psz, bpp, pa); #endif /* MEMSCRUB_DEBUG */ /* * MEMSCRUBBASE is a 4MB aligned page in the * kernel so that we can quickly map the PA * to a VA for the block loads performed in * memscrub_read. */ pfn = mmu_btop(pa); va = (caddr_t)MEMSCRUBBASE; hat_devload(kas.a_hat, va, psz, pfn, PROT_READ, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); /* * Can't allow the memscrubber to migrate across CPUs as * we need to know whether CEEN is enabled for the current * CPU to enable us to scrub the memory. Don't use * kpreempt_disable as the time we take to scan a span (even * without cpu_check_ce having to manually cpu_check_block) * is too long to hold a higher priority thread (eg, RT) * off cpu. */ thread_affinity_set(curthread, CPU_CURRENT); /* * Protect read scrub from async faults. For now, we simply * maintain a count of such faults caught. */ if (!on_trap(&otd, OT_DATA_EC) && !scan_mmu_pagesize) { memscrub_read(va, bpp); /* * Check if CEs require logging */ cpu_check_ce(SCRUBBER_CEEN_CHECK, (uint64_t)pa, va, psz); no_trap(); thread_affinity_clear(curthread); } else { no_trap(); thread_affinity_clear(curthread); /* * Got an async error.. * Try rescanning it at MMU_PAGESIZE * granularity if we were trying to * read at a larger page size. * This is to ensure we continue to * scan the rest of the span. * OR scanning MMU_PAGESIZE granularity to avoid * reading retired pages memory when scan_mmu_pagesize * is set. */ if (psz > MMU_PAGESIZE || scan_mmu_pagesize) { caddr_t vaddr = va; ms_paddr_t paddr = pa; int tmp = 0; for (; tmp < bpp; tmp += MEMSCRUB_BPP) { /* Don't scrub retired pages */ if (page_retire_check(paddr, NULL) == 0) { vaddr += MMU_PAGESIZE; paddr += MMU_PAGESIZE; retired_pages++; continue; } thread_affinity_set(curthread, CPU_CURRENT); if (!on_trap(&otd, OT_DATA_EC)) { memscrub_read(vaddr, MEMSCRUB_BPP); cpu_check_ce( SCRUBBER_CEEN_CHECK, (uint64_t)paddr, vaddr, MMU_PAGESIZE); no_trap(); } else { no_trap(); MEMSCRUB_STAT_INC(errors_found); } thread_affinity_clear(curthread); vaddr += MMU_PAGESIZE; paddr += MMU_PAGESIZE; } } } hat_unload(kas.a_hat, va, psz, HAT_UNLOAD_UNLOCK); blks -= bpp; pa += psz; pgsread++; } /* * If just finished scrubbing MMU_PAGESIZE at a time, but no retired * pages found so delete span from global list. */ if (scan_mmu_pagesize && retired_pages == 0) memscrub_page_retire_span_delete(src); /* * Encountered CE/UE on a retired page during memscrub read of current * span. Adding span to global list to enable avoid reading further. */ if (add_to_page_retire_list) { if (!memscrub_page_retire_span_search(src)) memscrub_page_retire_span_add(src); add_to_page_retire_list = 0; } if (memscrub_verbose) { cmn_err(CE_NOTE, "Memory scrubber read 0x%x pages starting " "at 0x%" PRIx64, pgsread, src); } } /* * Called by cpu_async_log_err() when memscrub read causes * CE/UE on a retired page. */ void memscrub_induced_error(void) { add_to_page_retire_list = 1; } /* * Called by page_retire() when toxic pages cannot be retired * immediately and are scheduled for retire. Memscrubber stops * scrubbing them to avoid further CE/UEs. */ void memscrub_notify(ms_paddr_t pa) { mutex_enter(&memscrub_lock); if (!memscrub_page_retire_span_search(pa)) memscrub_page_retire_span_add(pa); mutex_exit(&memscrub_lock); } /* * Called by memscrub_scan() and memscrub_notify(). * pa: physical address of span with CE/UE, add to global list. */ static void memscrub_page_retire_span_add(ms_paddr_t pa) { memscrub_page_retire_span_t *new_span; new_span = (memscrub_page_retire_span_t *) kmem_zalloc(sizeof (memscrub_page_retire_span_t), KM_NOSLEEP); if (new_span == NULL) { #ifdef MEMSCRUB_DEBUG cmn_err(CE_NOTE, "failed to allocate new span - span with" " retired page/s not tracked.\n"); #endif /* MEMSCRUB_DEBUG */ return; } new_span->address = pa; new_span->next = memscrub_page_retire_span_list; memscrub_page_retire_span_list = new_span; } /* * Called by memscrub_scan(). * pa: physical address of span to be removed from global list. */ static void memscrub_page_retire_span_delete(ms_paddr_t pa) { memscrub_page_retire_span_t *prev_span, *next_span; prev_span = memscrub_page_retire_span_list; next_span = memscrub_page_retire_span_list->next; if (pa == prev_span->address) { memscrub_page_retire_span_list = next_span; kmem_free(prev_span, sizeof (memscrub_page_retire_span_t)); return; } while (next_span) { if (pa == next_span->address) { prev_span->next = next_span->next; kmem_free(next_span, sizeof (memscrub_page_retire_span_t)); return; } prev_span = next_span; next_span = next_span->next; } } /* * Called by memscrub_scan() and memscrub_notify(). * pa: physical address of span to be searched in global list. */ static int memscrub_page_retire_span_search(ms_paddr_t pa) { memscrub_page_retire_span_t *next_span = memscrub_page_retire_span_list; while (next_span) { if (pa == next_span->address) return (1); next_span = next_span->next; } return (0); } /* * Called from new_memscrub() as a result of memory delete. * Using page_numtopp_nolock() to determine if we have valid PA. */ static void memscrub_page_retire_span_list_update(void) { memscrub_page_retire_span_t *prev, *cur, *next; if (memscrub_page_retire_span_list == NULL) return; prev = cur = memscrub_page_retire_span_list; next = cur->next; while (cur) { if (page_numtopp_nolock(mmu_btop(cur->address)) == NULL) { if (cur == memscrub_page_retire_span_list) { memscrub_page_retire_span_list = next; kmem_free(cur, sizeof (memscrub_page_retire_span_t)); prev = cur = memscrub_page_retire_span_list; } else { prev->next = cur->next; kmem_free(cur, sizeof (memscrub_page_retire_span_t)); cur = next; } } else { prev = cur; cur = next; } if (cur != NULL) next = cur->next; } } /* * The memory add/delete callback mechanism does not pass in the * page ranges. The phys_install list has been updated though, so * create a new scrub list from it. */ static int new_memscrub(int update_page_retire_list) { struct memlist *src, *list, *old_list; uint_t npgs; /* * copy phys_install to memscrub_memlist */ list = NULL; npgs = 0; memlist_read_lock(); for (src = phys_install; src; src = src->ml_next) { if (memscrub_add_span_gen((pfn_t)(src->ml_address >> PAGESHIFT), (pgcnt_t)(src->ml_size >> PAGESHIFT), &list, &npgs)) { memlist_read_unlock(); while (list) { struct memlist *el; el = list; list = list->ml_next; kmem_free(el, sizeof (struct memlist)); } return (-1); } } memlist_read_unlock(); mutex_enter(&memscrub_lock); memscrub_phys_pages = npgs; old_list = memscrub_memlist; memscrub_memlist = list; if (update_page_retire_list) memscrub_page_retire_span_list_update(); mutex_exit(&memscrub_lock); while (old_list) { struct memlist *el; el = old_list; old_list = old_list->ml_next; kmem_free(el, sizeof (struct memlist)); } return (0); } /*ARGSUSED*/ static void memscrub_mem_config_post_add( void *arg, pgcnt_t delta_pages) { /* * We increment pause_memscrub before entering new_memscrub(). This * will force the memscrubber to sleep, allowing the DR callback * thread to acquire memscrub_lock in new_memscrub(). The use of * atomic_add_32() allows concurrent memory DR operations to use the * callbacks safely. */ atomic_inc_32(&pause_memscrub); ASSERT(pause_memscrub != 0); /* * "Don't care" if we are not scrubbing new memory. */ (void) new_memscrub(0); /* retain page retire list */ /* Restore the pause setting. */ atomic_dec_32(&pause_memscrub); } /*ARGSUSED*/ static int memscrub_mem_config_pre_del( void *arg, pgcnt_t delta_pages) { /* Nothing to do. */ return (0); } /*ARGSUSED*/ static void memscrub_mem_config_post_del( void *arg, pgcnt_t delta_pages, int cancelled) { /* * We increment pause_memscrub before entering new_memscrub(). This * will force the memscrubber to sleep, allowing the DR callback * thread to acquire memscrub_lock in new_memscrub(). The use of * atomic_add_32() allows concurrent memory DR operations to use the * callbacks safely. */ atomic_inc_32(&pause_memscrub); ASSERT(pause_memscrub != 0); /* * Must stop scrubbing deleted memory as it may be disconnected. */ if (new_memscrub(1)) { /* update page retire list */ disable_memscrub = 1; } /* Restore the pause setting. */ atomic_dec_32(&pause_memscrub); } static kphysm_setup_vector_t memscrub_mem_config_vec = { KPHYSM_SETUP_VECTOR_VERSION, memscrub_mem_config_post_add, memscrub_mem_config_pre_del, memscrub_mem_config_post_del, }; static void memscrub_init_mem_config() { int ret; ret = kphysm_setup_func_register(&memscrub_mem_config_vec, (void *)NULL); ASSERT(ret == 0); } static void memscrub_uninit_mem_config() { /* This call is OK if the register call was not done. */ kphysm_setup_func_unregister(&memscrub_mem_config_vec, (void *)NULL); }