xref: /titanic_51/usr/src/uts/sun4/os/memnode.c (revision a0261a438a8e91bafb5c9fc9a8c06a2cf682ab37)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/systm.h>
29 #include <sys/platform_module.h>
30 #include <sys/sysmacros.h>
31 #include <sys/atomic.h>
32 #include <sys/memlist.h>
33 #include <sys/memnode.h>
34 #include <vm/vm_dep.h>
35 
36 int max_mem_nodes = 1;		/* max memory nodes on this system */
37 
38 struct mem_node_conf mem_node_config[MAX_MEM_NODES];
39 int mem_node_pfn_shift;
40 /*
41  * num_memnodes should be updated atomically and always >=
42  * the number of bits in memnodes_mask or the algorithm may fail.
43  */
44 uint16_t num_memnodes;
45 mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
46 
47 /*
48  * If set, mem_node_physalign should be a power of two, and
49  * should reflect the minimum address alignment of each node.
50  */
51 uint64_t mem_node_physalign;
52 
53 /*
54  * Platform hooks we will need.
55  */
56 
57 #pragma weak plat_build_mem_nodes
58 #pragma weak plat_slice_add
59 #pragma weak plat_slice_del
60 
61 /*
62  * Adjust the memnode config after a DR operation.
63  *
64  * It is rather tricky to do these updates since we can't
65  * protect the memnode structures with locks, so we must
66  * be mindful of the order in which updates and reads to
67  * these values can occur.
68  */
69 void
70 mem_node_add_slice(pfn_t start, pfn_t end)
71 {
72 	int mnode;
73 	mnodeset_t newmask, oldmask;
74 
75 	/*
76 	 * DR will pass us the first pfn that is allocatable.
77 	 * We need to round down to get the real start of
78 	 * the slice.
79 	 */
80 	if (mem_node_physalign) {
81 		start &= ~(btop(mem_node_physalign) - 1);
82 		end = roundup(end, btop(mem_node_physalign)) - 1;
83 	}
84 
85 	if (&plat_slice_add != NULL)
86 		plat_slice_add(start, end);
87 
88 	mnode = PFN_2_MEM_NODE(start);
89 	ASSERT(mnode < max_mem_nodes);
90 
91 	if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
92 		/*
93 		 * Add slice to existing node.
94 		 */
95 		if (start < mem_node_config[mnode].physbase)
96 			mem_node_config[mnode].physbase = start;
97 		if (end > mem_node_config[mnode].physmax)
98 			mem_node_config[mnode].physmax = end;
99 	} else {
100 		mem_node_config[mnode].physbase = start;
101 		mem_node_config[mnode].physmax = end;
102 		atomic_add_16(&num_memnodes, 1);
103 		do {
104 			oldmask = memnodes_mask;
105 			newmask = memnodes_mask | (1ull << mnode);
106 		} while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
107 	}
108 	/*
109 	 * Let the common lgrp framework know about the new memory
110 	 */
111 	lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
112 }
113 
114 /* ARGSUSED */
115 void
116 mem_node_pre_del_slice(pfn_t start, pfn_t end)
117 {
118 	int mnode = PFN_2_MEM_NODE(start);
119 
120 	ASSERT(mnode < max_mem_nodes);
121 	ASSERT(mem_node_config[mnode].exists == 1);
122 }
123 
124 /*
125  * Remove a PFN range from a memnode.  On some platforms,
126  * the memnode will be created with physbase at the first
127  * allocatable PFN, but later deleted with the MC slice
128  * base address converted to a PFN, in which case we need
129  * to assume physbase and up.
130  */
131 void
132 mem_node_post_del_slice(pfn_t start, pfn_t end, int cancelled)
133 {
134 	int mnode;
135 	pgcnt_t delta_pgcnt, node_size;
136 	mnodeset_t omask, nmask;
137 
138 	if (mem_node_physalign) {
139 		start &= ~(btop(mem_node_physalign) - 1);
140 		end = roundup(end, btop(mem_node_physalign)) - 1;
141 	}
142 	mnode = PFN_2_MEM_NODE(start);
143 
144 	ASSERT(mnode < max_mem_nodes);
145 	ASSERT(mem_node_config[mnode].exists == 1);
146 
147 	if (!cancelled) {
148 		delta_pgcnt = end - start;
149 		node_size = mem_node_config[mnode].physmax -
150 		    mem_node_config[mnode].physbase;
151 
152 		if (node_size > delta_pgcnt) {
153 			/*
154 			 * Subtract the slice from the memnode.
155 			 */
156 			if (start <= mem_node_config[mnode].physbase)
157 				mem_node_config[mnode].physbase = end + 1;
158 			ASSERT(end <= mem_node_config[mnode].physmax);
159 			if (end == mem_node_config[mnode].physmax)
160 				mem_node_config[mnode].physmax = start - 1;
161 		} else {
162 
163 			/*
164 			 * Let the common lgrp framework know the mnode is
165 			 * leaving
166 			 */
167 			lgrp_config(LGRP_CONFIG_MEM_DEL, mnode,
168 			    MEM_NODE_2_LGRPHAND(mnode));
169 
170 			/*
171 			 * Delete the whole node.
172 			 */
173 			ASSERT(MNODE_PGCNT(mnode) == 0);
174 			do {
175 				omask = memnodes_mask;
176 				nmask = omask & ~(1ull << mnode);
177 			} while (cas64(&memnodes_mask, omask, nmask) != omask);
178 			atomic_add_16(&num_memnodes, -1);
179 			mem_node_config[mnode].exists = 0;
180 		}
181 
182 		if (&plat_slice_del != NULL)
183 			plat_slice_del(start, end);
184 	}
185 }
186 
187 void
188 startup_build_mem_nodes(prom_memlist_t *list, size_t nelems)
189 {
190 	size_t	elem;
191 	pfn_t	basepfn;
192 	pgcnt_t	npgs;
193 
194 	/* LINTED: ASSERT will always true or false */
195 	ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
196 
197 	if (&plat_build_mem_nodes != NULL) {
198 		plat_build_mem_nodes(list, nelems);
199 	} else {
200 		/*
201 		 * Boot install lists are arranged <addr, len>, ...
202 		 */
203 		for (elem = 0; elem < nelems; list++, elem++) {
204 			basepfn = btop(list->addr);
205 			npgs = btop(list->size);
206 			mem_node_add_slice(basepfn, basepfn + npgs - 1);
207 		}
208 	}
209 }
210 
211 /*
212  * Allocate an unassigned memnode.
213  */
214 int
215 mem_node_alloc()
216 {
217 	int mnode;
218 	mnodeset_t newmask, oldmask;
219 
220 	/*
221 	 * Find an unused memnode.  Update it atomically to prevent
222 	 * a first time memnode creation race.
223 	 */
224 	for (mnode = 0; mnode < max_mem_nodes; mnode++)
225 		if (cas32((uint32_t *)&mem_node_config[mnode].exists,
226 		    0, 1) == 0)
227 			break;
228 
229 	if (mnode >= max_mem_nodes)
230 			panic("Out of free memnodes\n");
231 
232 	mem_node_config[mnode].physbase = (uint64_t)-1;
233 	mem_node_config[mnode].physmax = 0;
234 	atomic_add_16(&num_memnodes, 1);
235 	do {
236 		oldmask = memnodes_mask;
237 		newmask = memnodes_mask | (1ull << mnode);
238 	} while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
239 
240 	return (mnode);
241 }
242 
243 /*
244  * Find the intersection between a memnode and a memlist
245  * and returns the number of pages that overlap.
246  *
247  * Grab the memlist lock to protect the list from DR operations.
248  */
249 pgcnt_t
250 mem_node_memlist_pages(int mnode, struct memlist *mlist)
251 {
252 	pfn_t		base, end;
253 	pfn_t		cur_base, cur_end;
254 	pgcnt_t		npgs = 0;
255 	pgcnt_t		pages;
256 	struct memlist	*pmem;
257 
258 	if (&plat_mem_node_intersect_range != NULL) {
259 		memlist_read_lock();
260 
261 		for (pmem = mlist; pmem; pmem = pmem->next) {
262 			plat_mem_node_intersect_range(btop(pmem->address),
263 			    btop(pmem->size), mnode, &pages);
264 			npgs += pages;
265 		}
266 
267 		memlist_read_unlock();
268 		return (npgs);
269 	}
270 
271 	base = mem_node_config[mnode].physbase;
272 	end = mem_node_config[mnode].physmax;
273 
274 	memlist_read_lock();
275 
276 	for (pmem = mlist; pmem; pmem = pmem->next) {
277 		cur_base = btop(pmem->address);
278 		cur_end = cur_base + btop(pmem->size) - 1;
279 		if (end < cur_base || base > cur_end)
280 			continue;
281 		npgs = npgs + (MIN(cur_end, end) -
282 		    MAX(cur_base, base)) + 1;
283 	}
284 
285 	memlist_read_unlock();
286 
287 	return (npgs);
288 }
289 
290 /*
291  * Find MIN(physbase) and MAX(physmax) over all mnodes
292  *
293  * Called during startup and DR to find hpm_counters limits when
294  * interleaved_mnodes is set.
295  * NOTE: there is a race condition with DR if it tries to change more than
296  * one mnode in parallel. Sizing shared hpm_counters depends on finding the
297  * min(physbase) and max(physmax) across all mnodes. Therefore, the caller of
298  * page_ctrs_adjust must ensure that mem_node_config does not change while it
299  * is running.
300  */
301 void
302 mem_node_max_range(pfn_t *basep, pfn_t *maxp)
303 {
304 	int mnode;
305 	pfn_t max = 0;
306 	pfn_t base = (pfn_t)-1;
307 
308 	for (mnode = 0; mnode < max_mem_nodes; mnode++) {
309 		if (mem_node_config[mnode].exists == 0)
310 			continue;
311 		if (max < mem_node_config[mnode].physmax)
312 			max = mem_node_config[mnode].physmax;
313 		if (base > mem_node_config[mnode].physbase)
314 			base = mem_node_config[mnode].physbase;
315 	}
316 	ASSERT(base != (pfn_t)-1 && max != 0);
317 	*basep = base;
318 	*maxp = max;
319 }
320