xref: /titanic_50/usr/src/uts/common/os/lgrp_topo.c (revision 0ed5c46e82c989cfa9726d9dae452e3d24ef83be)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*
26  * lgroup topology
27  */
28 
29 #include <sys/cpupart.h>
30 #include <sys/lgrp.h>
31 #include <sys/promif.h>
32 #include <sys/types.h>
33 
34 
35 #define	LGRP_TOPO_LEVELS	4	/* default height limit */
36 #define	LGRP_TOPO_LEVELS_MAX	4	/* max height limit */
37 
38 
39 /*
40  * Only collapse lgroups which have same latency (and resources)
41  */
42 int		lgrp_collapse_equidist = 1;
43 
44 int		lgrp_collapse_off = 1;	/* disable collapsing of duplicates */
45 
46 /*
47  * Height to limit lgroup topology
48  */
49 unsigned int	lgrp_topo_levels = LGRP_TOPO_LEVELS;
50 
51 int		lgrp_split_off = 1;	/* disable splitting lgroups */
52 
53 #ifdef	DEBUG
54 /*
55  * Debugging output
56  * - 0: off
57  * - >0: on and bigger means more
58  */
59 int	lgrp_topo_debug = 0;
60 
61 
62 void
klgrpset_print(klgrpset_t lgrpset)63 klgrpset_print(klgrpset_t lgrpset)
64 {
65 	int	i;
66 
67 
68 	prom_printf("0x%llx(", (u_longlong_t)lgrpset);
69 	for (i = 0; i <= lgrp_alloc_max; i++)
70 		if (klgrpset_ismember(lgrpset, i))
71 			prom_printf("%d ", i);
72 	prom_printf(")\n");
73 }
74 
75 
76 void
lgrp_rsets_print(char * string,klgrpset_t * rsets)77 lgrp_rsets_print(char *string, klgrpset_t *rsets)
78 {
79 	int	i;
80 
81 	prom_printf("%s\n", string);
82 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
83 		klgrpset_print(rsets[i]);
84 }
85 #endif	/* DEBUG */
86 
87 
88 /*
89  * Add "from" lgroup resources to "to" lgroup resources
90  */
91 void
lgrp_rsets_add(klgrpset_t * from,klgrpset_t * to)92 lgrp_rsets_add(klgrpset_t *from, klgrpset_t *to)
93 {
94 	int	i;
95 
96 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
97 		klgrpset_or(to[i], from[i]);
98 }
99 
100 
101 /*
102  * Copy "from" lgroup resources to "to" lgroup resources
103  */
104 void
lgrp_rsets_copy(klgrpset_t * from,klgrpset_t * to)105 lgrp_rsets_copy(klgrpset_t *from, klgrpset_t *to)
106 {
107 	int	i;
108 
109 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
110 		to[i] = from[i];
111 }
112 
113 
114 /*
115  * Delete given lgroup ID from lgroup resource set of specified lgroup
116  * and its ancestors if "follow_parent" is set
117  */
118 void
lgrp_rsets_delete(lgrp_t * lgrp,lgrp_id_t lgrpid,int follow_parent)119 lgrp_rsets_delete(lgrp_t *lgrp, lgrp_id_t lgrpid, int follow_parent)
120 {
121 	int	i;
122 
123 	while (lgrp != NULL) {
124 		for (i = 0; i < LGRP_RSRC_COUNT; i++)
125 			klgrpset_del(lgrp->lgrp_set[i], lgrpid);
126 		if (!follow_parent)
127 			break;
128 		lgrp = lgrp->lgrp_parent;
129 	}
130 }
131 
132 
133 /*
134  * Return whether given lgroup resource set empty
135  */
136 int
lgrp_rsets_empty(klgrpset_t * rset)137 lgrp_rsets_empty(klgrpset_t *rset)
138 {
139 	int	i;
140 
141 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
142 		if (!klgrpset_isempty(rset[i]))
143 			return (0);
144 
145 	return (1);
146 }
147 
148 
149 /*
150  * Return whether given lgroup resource sets are same
151  */
152 int
lgrp_rsets_equal(klgrpset_t * rset1,klgrpset_t * rset2)153 lgrp_rsets_equal(klgrpset_t *rset1, klgrpset_t *rset2)
154 {
155 	int	i;
156 
157 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
158 		if (rset1[i] != rset2[i])
159 			return (0);
160 
161 	return (1);
162 }
163 
164 
165 /*
166  * Return whether specified lgroup ID is in given lgroup resource set
167  */
168 int
lgrp_rsets_member(klgrpset_t * rset,lgrp_id_t lgrpid)169 lgrp_rsets_member(klgrpset_t *rset, lgrp_id_t lgrpid)
170 {
171 	int	i;
172 
173 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
174 		if (klgrpset_ismember(rset[i], lgrpid))
175 			return (1);
176 
177 	return (0);
178 }
179 
180 
181 /*
182  * Return whether specified lgroup ID is in all lgroup resources
183  */
184 int
lgrp_rsets_member_all(klgrpset_t * rset,lgrp_id_t lgrpid)185 lgrp_rsets_member_all(klgrpset_t *rset, lgrp_id_t lgrpid)
186 {
187 	int	i;
188 
189 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
190 		if (!klgrpset_ismember(rset[i], lgrpid))
191 			return (0);
192 
193 	return (1);
194 }
195 
196 
197 /*
198  * Replace resources for given lgroup with specified resources at given
199  * latency and shift its old resources to its parent and its parent's resources
200  * to its parent, etc. until root lgroup reached
201  */
202 void
lgrp_rsets_replace(klgrpset_t * rset,int latency,lgrp_t * lgrp,int shift)203 lgrp_rsets_replace(klgrpset_t *rset, int latency, lgrp_t *lgrp, int shift)
204 {
205 	lgrp_t		*cur;
206 	int		lat_new;
207 	int		lat_saved;
208 	klgrpset_t	rset_new[LGRP_RSRC_COUNT];
209 	klgrpset_t	rset_saved[LGRP_RSRC_COUNT];
210 
211 	cur = lgrp;
212 	lat_saved = latency;
213 	lgrp_rsets_copy(rset, rset_saved);
214 	while (cur && cur != lgrp_root) {
215 		/*
216 		 * Save current resources and latency to insert in parent and
217 		 * then replace with new resources and latency
218 		 */
219 		lgrp_rsets_copy(rset_saved, rset_new);
220 		lgrp_rsets_copy(cur->lgrp_set, rset_saved);
221 		lgrp_rsets_copy(rset_new, cur->lgrp_set);
222 
223 		lat_new = lat_saved;
224 		lat_saved = cur->lgrp_latency;
225 		cur->lgrp_latency = lat_new;
226 		if (!shift)
227 			break;
228 		cur = cur->lgrp_parent;
229 	}
230 }
231 
232 
233 /*
234  * Set "to" lgroup resource set with given lgroup ID
235  */
236 void
lgrp_rsets_set(klgrpset_t * to,lgrp_id_t lgrpid)237 lgrp_rsets_set(klgrpset_t *to, lgrp_id_t lgrpid)
238 {
239 	klgrpset_t	from;
240 	int		i;
241 
242 	klgrpset_clear(from);
243 	klgrpset_add(from, lgrpid);
244 	for (i = 0; i < LGRP_RSRC_COUNT; i++) {
245 		klgrpset_clear(to[i]);
246 		klgrpset_or(to[i], from);
247 	}
248 }
249 
250 
251 /*
252  * Delete any ancestors of given child lgroup which don't have any other
253  * children
254  */
255 int
lgrp_ancestor_delete(lgrp_t * child,klgrpset_t * changed)256 lgrp_ancestor_delete(lgrp_t *child, klgrpset_t *changed)
257 {
258 	int		count;
259 	lgrp_t		*current;
260 	lgrp_id_t	lgrpid;
261 	lgrp_t		*parent;
262 
263 #ifdef	DEBUG
264 	if (lgrp_topo_debug > 1) {
265 		prom_printf("lgrp_ancestor_delete(0x%p[%d],0x%p)\n",
266 		    (void *)child, child->lgrp_id, (void *)changed);
267 	}
268 #endif	/* DEBUG */
269 
270 	count = 0;
271 	if (changed)
272 		klgrpset_clear(*changed);
273 
274 	/*
275 	 * Visit ancestors, decrement child count for each, and remove any
276 	 * that don't have any children left until we reach an ancestor that
277 	 * has multiple children
278 	 */
279 	current = child;
280 	parent = child->lgrp_parent;
281 	lgrpid = current->lgrp_id;
282 	while (parent != NULL) {
283 #ifdef	DEBUG
284 		if (lgrp_topo_debug > 1)
285 			prom_printf("lgrp_ancestor_delete: parent %d,"
286 			    " current %d\n",
287 			    parent->lgrp_id, lgrpid);
288 #endif	/* DEBUG */
289 
290 		klgrpset_del(parent->lgrp_leaves, lgrpid);
291 		klgrpset_del(parent->lgrp_children, lgrpid);
292 		parent->lgrp_childcnt--;
293 		if (changed)
294 			klgrpset_add(*changed, parent->lgrp_id);
295 		count++;
296 		if (parent->lgrp_childcnt != 0)
297 			break;
298 
299 		current = parent;
300 		parent = current->lgrp_parent;
301 		lgrpid = current->lgrp_id;
302 
303 #ifdef	DEBUG
304 		if (lgrp_topo_debug > 0)
305 			prom_printf("lgrp_ancestor_delete: destroy"
306 			    " lgrp %d at 0x%p\n",
307 			    current->lgrp_id, (void *)current);
308 #endif	/* DEBUG */
309 		lgrp_destroy(current);
310 	}
311 
312 #ifdef	DEBUG
313 	if (lgrp_topo_debug > 1 && changed)
314 		prom_printf("lgrp_ancestor_delete: changed %d lgrps: 0x%llx\n",
315 		    count, (u_longlong_t)*changed);
316 #endif	/* DEBUG */
317 
318 	return (count);
319 }
320 
321 
322 /*
323  * Consolidate lgrp1 into lgrp2
324  */
325 int
lgrp_consolidate(lgrp_t * lgrp1,lgrp_t * lgrp2,klgrpset_t * changed)326 lgrp_consolidate(lgrp_t *lgrp1, lgrp_t *lgrp2, klgrpset_t *changed)
327 {
328 	klgrpset_t	changes;
329 	lgrp_t		*child;
330 	int		count;
331 	int		i;
332 	lgrp_t		*parent;
333 
334 	/*
335 	 * Leaf lgroups should never need to be consolidated
336 	 */
337 	if (lgrp1 == NULL || lgrp2 == NULL || lgrp1->lgrp_childcnt < 1 ||
338 	    lgrp2->lgrp_childcnt < 1)
339 		return (0);
340 
341 #ifdef	DEBUG
342 	if (lgrp_topo_debug > 0)
343 		prom_printf("lgrp_consolidate(0x%p[%d],0x%p[%d],0x%p)\n",
344 		    (void *)lgrp1, lgrp1->lgrp_id, (void *)lgrp2,
345 		    lgrp2->lgrp_id, (void *)changed);
346 #endif	/* DEBUG */
347 
348 	count = 0;
349 	if (changed)
350 		klgrpset_clear(*changed);
351 
352 	/*
353 	 * Lgroup represents resources within certain latency, so need to keep
354 	 * biggest latency value of lgroups being consolidated
355 	 */
356 	if (lgrp1->lgrp_latency > lgrp2->lgrp_latency)
357 		lgrp2->lgrp_latency = lgrp1->lgrp_latency;
358 
359 	/*
360 	 * Delete ancestors of lgrp1 that don't have any other children
361 	 */
362 #ifdef	DEBUG
363 	if (lgrp_topo_debug > 1)
364 		prom_printf("lgrp_consolidate: delete ancestors\n");
365 #endif	/* DEBUG */
366 	count += lgrp_ancestor_delete(lgrp1, &changes);
367 	if (changed) {
368 		klgrpset_or(*changed, changes);
369 		klgrpset_or(*changed, lgrp1->lgrp_id);
370 		count++;
371 	}
372 
373 	/*
374 	 * Reparent children lgroups of lgrp1 to lgrp2
375 	 */
376 	for (i = 0; i <= lgrp_alloc_max; i++) {
377 		if (i == lgrp2->lgrp_id ||
378 		    !klgrpset_ismember(lgrp1->lgrp_children, i))
379 			continue;
380 		child = lgrp_table[i];
381 		if (!LGRP_EXISTS(child))
382 			continue;
383 #ifdef	DEBUG
384 		if (lgrp_topo_debug > 0)
385 			prom_printf("lgrp_consolidate: reparent "
386 			    "lgrp %d to lgrp %d\n",
387 			    child->lgrp_id, lgrp2->lgrp_id);
388 #endif	/* DEBUG */
389 		klgrpset_or(lgrp2->lgrp_leaves, child->lgrp_leaves);
390 		klgrpset_add(lgrp2->lgrp_children, child->lgrp_id);
391 		lgrp2->lgrp_childcnt++;
392 		child->lgrp_parent = lgrp2;
393 		if (changed) {
394 			klgrpset_add(*changed, child->lgrp_id);
395 			klgrpset_add(*changed, lgrp2->lgrp_id);
396 		}
397 		count += 2;
398 	}
399 
400 	/*
401 	 * Proprogate leaves from lgrp2 to root
402 	 */
403 	child = lgrp2;
404 	parent = child->lgrp_parent;
405 	while (parent != NULL) {
406 		klgrpset_or(parent->lgrp_leaves, child->lgrp_leaves);
407 		if (changed)
408 			klgrpset_add(*changed, parent->lgrp_id);
409 		count++;
410 		child = parent;
411 		parent = parent->lgrp_parent;
412 	}
413 
414 #ifdef	DEBUG
415 	if (lgrp_topo_debug > 0)
416 		prom_printf("lgrp_consolidate: destroy lgrp %d at 0x%p\n",
417 		    lgrp1->lgrp_id, (void *)lgrp1);
418 	if (lgrp_topo_debug > 1 && changed)
419 		prom_printf("lgrp_consolidate: changed %d lgrps: 0x%llx\n",
420 		    count, (u_longlong_t)*changed);
421 #endif	/* DEBUG */
422 
423 	lgrp_destroy(lgrp1);
424 
425 	return (count);
426 }
427 
428 /*
429  * Collapse duplicates of target lgroups given
430  */
431 int
lgrp_collapse_dups(klgrpset_t target_set,int equidist_only,klgrpset_t * changed)432 lgrp_collapse_dups(klgrpset_t target_set, int equidist_only,
433     klgrpset_t *changed)
434 {
435 	klgrpset_t	changes;
436 	int		count;
437 	int		i;
438 
439 	count = 0;
440 	if (changed)
441 		klgrpset_clear(*changed);
442 
443 	if (lgrp_collapse_off)
444 		return (0);
445 
446 #ifdef	DEBUG
447 	if (lgrp_topo_debug > 0)
448 		prom_printf("lgrp_collapse_dups(0x%llx)\n",
449 		    (u_longlong_t)target_set);
450 #endif	/* DEBUG */
451 
452 	/*
453 	 * Look for duplicates of each target lgroup
454 	 */
455 	for (i = 0; i <= lgrp_alloc_max; i++) {
456 		int	j;
457 		lgrp_t	*keep;
458 		lgrp_t	*target;
459 
460 		target = lgrp_table[i];
461 
462 		/*
463 		 * Skip to next lgroup if there isn't one here, this is root
464 		 * or leaf lgroup, or this isn't a target lgroup
465 		 */
466 		if (!LGRP_EXISTS(target) ||
467 		    target == lgrp_root || target->lgrp_childcnt == 0 ||
468 		    !klgrpset_ismember(target_set, target->lgrp_id))
469 			continue;
470 
471 		/*
472 		 * Find all lgroups with same resources and latency
473 		 */
474 #ifdef	DEBUG
475 		if (lgrp_topo_debug > 1)
476 			prom_printf("lgrp_collapse_dups: find "
477 			    "dups of lgrp %d at 0x%p\n",
478 			    target->lgrp_id, (void *)target);
479 #endif	/* DEBUG */
480 		keep = NULL;
481 		for (j = 0; j <= lgrp_alloc_max; j++) {
482 			lgrp_t	*lgrp;
483 
484 			lgrp = lgrp_table[j];
485 
486 			/*
487 			 * Skip lgroup if there isn't one here, this is root
488 			 * lgroup or leaf (which shouldn't have dups), or this
489 			 * lgroup doesn't have same resources
490 			 */
491 			if (!LGRP_EXISTS(lgrp) ||
492 			    lgrp->lgrp_childcnt == 0 ||
493 			    !lgrp_rsets_equal(lgrp->lgrp_set,
494 			    target->lgrp_set) ||
495 			    (lgrp->lgrp_latency != target->lgrp_latency &&
496 			    equidist_only))
497 				continue;
498 
499 			/*
500 			 * Keep first matching lgroup (but always keep root)
501 			 * and consolidate other duplicates into it
502 			 */
503 			if (keep == NULL) {
504 				keep = lgrp;
505 #ifdef	DEBUG
506 				if (lgrp_topo_debug > 1)
507 					prom_printf("lgrp_collapse_dups: "
508 					    "keep lgrp %d at 0x%p\n",
509 					    keep->lgrp_id, (void *)keep);
510 #endif	/* DEBUG */
511 			} else {
512 				if (lgrp == lgrp_root) {
513 					lgrp = keep;
514 					keep = lgrp_root;
515 				}
516 #ifdef	DEBUG
517 				if (lgrp_topo_debug > 0)
518 					prom_printf("lgrp_collapse_dups:"
519 					    " consolidate lgrp %d at 0x%p"
520 					    " into lgrp %d at 0x%p\n",
521 					    lgrp->lgrp_id, (void *)lgrp,
522 					    keep->lgrp_id, (void *)keep);
523 #endif	/* DEBUG */
524 				count += lgrp_consolidate(lgrp, keep,
525 				    &changes);
526 				if (changed)
527 					klgrpset_or(*changed, changes);
528 			}
529 		}
530 	}
531 
532 #ifdef	DEBUG
533 	if (lgrp_topo_debug > 1 && changed)
534 		prom_printf("lgrp_collapse_dups: changed %d lgrps: 0x%llx\n",
535 		    count, (u_longlong_t)*changed);
536 #endif	/* DEBUG */
537 
538 	return (count);
539 }
540 
541 
542 /*
543  * Create new parent lgroup with given latency and resources for
544  * specified child lgroup, and insert it into hierarchy
545  */
546 int
lgrp_new_parent(lgrp_t * child,int latency,klgrpset_t * rset,klgrpset_t * changed)547 lgrp_new_parent(lgrp_t *child, int latency, klgrpset_t *rset,
548     klgrpset_t *changed)
549 {
550 	int	count;
551 	lgrp_t	*new;
552 	lgrp_t	*old;
553 
554 	count = 0;
555 	if (changed)
556 		klgrpset_clear(*changed);
557 
558 	/*
559 	 * Create lgroup and set its latency and resources
560 	 */
561 	new = lgrp_create();
562 	new->lgrp_latency = latency;
563 	lgrp_rsets_add(rset, new->lgrp_set);
564 
565 	/*
566 	 * Insert new lgroup into hierarchy
567 	 */
568 	old = child->lgrp_parent;
569 	new->lgrp_parent = old;
570 	klgrpset_add(new->lgrp_children, child->lgrp_id);
571 	new->lgrp_childcnt++;
572 	klgrpset_add(new->lgrp_children, child->lgrp_id);
573 	klgrpset_copy(new->lgrp_leaves, child->lgrp_leaves);
574 
575 	child->lgrp_parent = new;
576 	if (old) {
577 		klgrpset_del(old->lgrp_children, child->lgrp_id);
578 		klgrpset_add(old->lgrp_children, new->lgrp_id);
579 		if (changed)
580 			klgrpset_add(*changed, old->lgrp_id);
581 		count++;
582 	}
583 
584 	if (changed) {
585 		klgrpset_add(*changed, child->lgrp_id);
586 		klgrpset_add(*changed, new->lgrp_id);
587 	}
588 	count += 2;
589 
590 #ifdef	DEBUG
591 	if (lgrp_topo_debug > 1 && changed)
592 		prom_printf("lgrp_new_parent: changed %d lgrps: 0x%llx\n",
593 		    count, (u_longlong_t)*changed);
594 #endif	/* DEBUG */
595 
596 	return (count);
597 }
598 
599 
600 /*
601  * Proprogate resources of new leaf into parent lgroup of given child
602  */
603 int
lgrp_proprogate(lgrp_t * newleaf,lgrp_t * child,int latency,klgrpset_t * changed)604 lgrp_proprogate(lgrp_t *newleaf, lgrp_t *child, int latency,
605     klgrpset_t *changed)
606 {
607 	int	count;
608 	lgrp_t	*parent;
609 
610 	count = 0;
611 	if (changed)
612 		klgrpset_clear(*changed);
613 
614 	if (child == NULL || child->lgrp_parent == NULL)
615 		return (0);
616 
617 	parent = child->lgrp_parent;
618 	klgrpset_or(parent->lgrp_leaves, child->lgrp_leaves);
619 	if (changed)
620 		klgrpset_add(*changed, parent->lgrp_id);
621 	count++;
622 
623 	/*
624 	 * Don't proprogate new leaf resources to parent if it already
625 	 * contains these resources
626 	 */
627 	if (lgrp_rsets_member_all(parent->lgrp_set, newleaf->lgrp_id)) {
628 #ifdef	DEBUG
629 		if (lgrp_topo_debug > 1 && changed)
630 			prom_printf("lgrp_proprogate: changed %d lgrps:"
631 			    " 0x%llx\n",
632 			    count, (u_longlong_t)*changed);
633 #endif	/* DEBUG */
634 		return (count);
635 	}
636 
637 	/*
638 	 * Add leaf resources to parent lgroup
639 	 */
640 	lgrp_rsets_add(newleaf->lgrp_set, parent->lgrp_set);
641 
642 #ifdef	DEBUG
643 	if (lgrp_topo_debug > 1) {
644 		prom_printf("lgrp_proprogate: newleaf %d(0x%p), "
645 		    "latency %d, child %d(0x%p), parent %d(0x%p)\n",
646 		    newleaf->lgrp_id, (void *)newleaf, latency, child->lgrp_id,
647 		    (void *)child, parent->lgrp_id, (void *)parent);
648 		prom_printf("lgrp_proprogate: parent's leaves becomes 0x%llx\n",
649 		    (u_longlong_t)parent->lgrp_leaves);
650 	}
651 	if (lgrp_topo_debug > 0) {
652 		prom_printf("lgrp_proprogate: adding to parent %d (0x%p)\n",
653 		    parent->lgrp_id, (void *)parent);
654 		lgrp_rsets_print("parent resources become:", parent->lgrp_set);
655 	}
656 
657 	if (lgrp_topo_debug > 2 && changed)
658 		prom_printf("lgrp_proprogate: changed %d lgrps: 0x%llx\n",
659 		    count, (u_longlong_t)*changed);
660 
661 #endif	/* DEBUG */
662 
663 	return (count);
664 }
665 
666 
667 /*
668  * Split parent lgroup of given child if child's leaf decendant (oldleaf) has
669  * different latency to new leaf lgroup (newleaf) than leaf lgroups of given
670  * child's siblings
671  */
672 int
lgrp_split(lgrp_t * oldleaf,lgrp_t * newleaf,lgrp_t * child,klgrpset_t * changed)673 lgrp_split(lgrp_t *oldleaf, lgrp_t *newleaf, lgrp_t *child,
674     klgrpset_t *changed)
675 {
676 	klgrpset_t	changes;
677 	int		count;
678 	int		i;
679 	int		latency;
680 	lgrp_t		*parent;
681 
682 	count = 0;
683 	if (changed)
684 		klgrpset_clear(*changed);
685 
686 	if (lgrp_split_off || newleaf == NULL || child == NULL)
687 		return (0);
688 
689 	/*
690 	 * Parent must have more than one child to have a child split from it
691 	 * and root lgroup contains all resources and never needs to be split
692 	 */
693 	parent = child->lgrp_parent;
694 	if (parent == NULL || parent->lgrp_childcnt < 2 || parent == lgrp_root)
695 		return (0);
696 
697 #ifdef	DEBUG
698 	if (lgrp_topo_debug > 1)
699 		prom_printf("lgrp_split(0x%p[%d],0x%p[%d],0x%p[%d],0x%p)\n",
700 		    (void *)oldleaf, oldleaf->lgrp_id,
701 		    (void *)newleaf, newleaf->lgrp_id,
702 		    (void *)child, child->lgrp_id, (void *)changed);
703 #endif	/* DEBUG */
704 
705 	/*
706 	 * Get latency between new leaf and old leaf whose lineage it is
707 	 * being added
708 	 */
709 	latency = lgrp_plat_latency(oldleaf->lgrp_plathand,
710 	    newleaf->lgrp_plathand);
711 
712 	/*
713 	 * Check whether all sibling leaves of given child lgroup have same
714 	 * latency to new leaf
715 	 */
716 	for (i = 0; i <= lgrp_alloc_max; i++) {
717 		lgrp_t		*grandparent;
718 		lgrp_t		*lgrp;
719 		int		sibling_latency;
720 
721 		lgrp = lgrp_table[i];
722 
723 		/*
724 		 * Skip non-existent lgroups, old leaf, and any lgroups that
725 		 * don't have parent as common ancestor
726 		 */
727 		if (!LGRP_EXISTS(lgrp) || lgrp == oldleaf ||
728 		    !klgrpset_ismember(parent->lgrp_leaves, lgrp->lgrp_id))
729 			continue;
730 
731 		/*
732 		 * Same latency, so skip
733 		 */
734 		sibling_latency = lgrp_plat_latency(lgrp->lgrp_plathand,
735 		    newleaf->lgrp_plathand);
736 #ifdef	DEBUG
737 		if (lgrp_topo_debug > 1)
738 			prom_printf("lgrp_split: latency(%d,%d) %d,"
739 			    " latency(%d,%d) %d\n",
740 			    oldleaf->lgrp_id, newleaf->lgrp_id, latency,
741 			    lgrp->lgrp_id, newleaf->lgrp_id, sibling_latency);
742 #endif	/* DEBUG */
743 		if (sibling_latency == latency)
744 			continue;
745 
746 		/*
747 		 * Different latencies, so  remove child from its parent and
748 		 * make new parent for old leaf with same latency and same
749 		 * resources
750 		 */
751 		parent->lgrp_childcnt--;
752 		klgrpset_del(parent->lgrp_children, child->lgrp_id);
753 		klgrpset_del(parent->lgrp_leaves, oldleaf->lgrp_id);
754 		grandparent = parent->lgrp_parent;
755 		if (grandparent) {
756 			grandparent->lgrp_childcnt++;
757 			klgrpset_add(grandparent->lgrp_children,
758 			    child->lgrp_id);
759 			count++;
760 			if (changed)
761 				klgrpset_add(*changed, grandparent->lgrp_id);
762 		}
763 		child->lgrp_parent = grandparent;
764 
765 		count += lgrp_new_parent(child, parent->lgrp_latency,
766 		    parent->lgrp_set, &changes);
767 		if (changed) {
768 			klgrpset_or(*changed, changes);
769 
770 			klgrpset_add(*changed, parent->lgrp_id);
771 			klgrpset_add(*changed, child->lgrp_id);
772 			count += 2;
773 		}
774 
775 		parent = child->lgrp_parent;
776 #ifdef	DEBUG
777 		if (lgrp_topo_debug > 0) {
778 			prom_printf("lgrp_split: new parent %d (0x%p) for"
779 			    " lgrp %d (0x%p)\n",
780 			    parent->lgrp_id, (void *)parent,
781 			    child->lgrp_id, (void *)child);
782 			lgrp_rsets_print("new parent resources:",
783 			    parent->lgrp_set);
784 		}
785 
786 		if (lgrp_topo_debug > 1 && changed)
787 			prom_printf("lgrp_split: changed %d lgrps: 0x%llx\n",
788 			    count, (u_longlong_t)*changed);
789 #endif	/* DEBUG */
790 
791 		return (count);
792 	}
793 
794 #ifdef	DEBUG
795 	if (lgrp_topo_debug > 1)
796 		prom_printf("lgrp_split: no changes\n");
797 #endif	/* DEBUG */
798 
799 	return (count);
800 }
801 
802 
803 /*
804  * Return height of lgroup topology from given lgroup to root
805  */
806 int
lgrp_topo_height(lgrp_t * lgrp)807 lgrp_topo_height(lgrp_t *lgrp)
808 {
809 	int	nlevels;
810 
811 	if (!LGRP_EXISTS(lgrp))
812 		return (0);
813 
814 	nlevels = 0;
815 	while (lgrp != NULL) {
816 		lgrp = lgrp->lgrp_parent;
817 		nlevels++;
818 	}
819 	return (nlevels);
820 }
821 
822 
823 /*
824  * Add resources of new leaf to old leaf's lineage
825  *
826  * Assumes the following:
827  * - Lgroup hierarchy consists of at least a root lgroup and its leaves
828  *   including old and new ones given below
829  * - New leaf lgroup has been created and does not need to have its resources
830  *   added to it
831  * - Latencies have been set for root and leaf lgroups
832  */
833 int
lgrp_lineage_add(lgrp_t * newleaf,lgrp_t * oldleaf,klgrpset_t * changed)834 lgrp_lineage_add(lgrp_t *newleaf, lgrp_t *oldleaf, klgrpset_t *changed)
835 {
836 	klgrpset_t	changes;
837 	lgrp_t		*child;
838 	klgrpset_t	collapse;
839 	int		count;
840 	int		latency;
841 	int		nlevels;
842 	lgrp_t		*parent;
843 	int		proprogate;
844 	int		total;
845 
846 
847 	count = total = 0;
848 	if (changed)
849 		klgrpset_clear(*changed);
850 
851 	if (newleaf == NULL || oldleaf == NULL || newleaf == oldleaf)
852 		return (0);
853 
854 #ifdef	DEBUG
855 	if (lgrp_topo_debug > 0)
856 		prom_printf("\nlgrp_lineage_add(0x%p[%d],0x%p[%d],0x%p)\n",
857 		    (void *)newleaf, newleaf->lgrp_id,
858 		    (void *)oldleaf, oldleaf->lgrp_id,
859 		    (void *)changed);
860 #endif	/* DEBUG */
861 
862 	/*
863 	 * Get latency between old and new leaves, so we can determine
864 	 * where the new leaf fits in the old leaf's lineage
865 	 */
866 	latency = lgrp_plat_latency(oldleaf->lgrp_plathand,
867 	    newleaf->lgrp_plathand);
868 
869 	/*
870 	 * Determine height of lgroup topology from old leaf to root lgroup,
871 	 * so height of topology may be limited if necessary
872 	 */
873 	nlevels = lgrp_topo_height(oldleaf);
874 
875 #ifdef	DEBUG
876 	if (lgrp_topo_debug > 1)
877 		prom_printf("lgrp_lineage_add: latency(%d,%d) 0x%x, ht %d\n",
878 		    oldleaf->lgrp_id, newleaf->lgrp_id, latency, nlevels);
879 #endif	/* DEBUG */
880 
881 	/*
882 	 * Can't add new leaf to old leaf's lineage if we haven't
883 	 * determined latency between them yet
884 	 */
885 	if (latency == 0)
886 		return (0);
887 
888 	child = oldleaf;
889 	parent = child->lgrp_parent;
890 	proprogate = 0;
891 	klgrpset_clear(collapse);
892 
893 	/*
894 	 * Lineage of old leaf is basically a sorted list of the other leaves
895 	 * from closest to farthest, so find where to add new leaf to the
896 	 * lineage and proprogate its resources from that point up to the root
897 	 * lgroup since parent lgroups contain all the resources of their
898 	 * children
899 	 */
900 	do {
901 		klgrpset_t	rset[LGRP_RSRC_COUNT];
902 
903 #ifdef	DEBUG
904 		if (lgrp_topo_debug > 1)
905 			prom_printf("lgrp_lineage_add: child %d (0x%p), parent"
906 			    " %d (0x%p)\n",
907 			    child->lgrp_id, (void *)child,
908 			    parent->lgrp_id, (void *)parent);
909 #endif	/* DEBUG */
910 
911 		/*
912 		 * See whether parent lgroup needs to be split
913 		 *
914 		 * May need to split parent lgroup when it is ancestor to more
915 		 * than one leaf, but all its leaves don't have latency to new
916 		 * leaf within the parent lgroup's latency
917 		 * NOTE: Don't want to collapse this lgroup since we just split
918 		 * it from parent
919 		 */
920 		count = lgrp_split(oldleaf, newleaf, child, &changes);
921 		if (count) {
922 #ifdef	DEBUG
923 			if (lgrp_topo_debug > 0)
924 				prom_printf("lgrp_lineage_add: setting parent"
925 				    " for child %d from %d to %d\n",
926 				    child->lgrp_id, parent->lgrp_id,
927 				    child->lgrp_parent->lgrp_id);
928 #endif	/* DEBUG */
929 			parent = child->lgrp_parent;
930 			total += count;
931 			if (changed)
932 				klgrpset_or(*changed, changes);
933 		}
934 
935 		/*
936 		 * Already found where resources of new leaf belong in old
937 		 * leaf's lineage, so proprogate resources of new leaf up
938 		 * through rest of ancestors
939 		 */
940 		if (proprogate) {
941 			total += lgrp_proprogate(newleaf, child, latency,
942 			    &changes);
943 			if (changed)
944 				klgrpset_or(*changed, changes);
945 
946 			parent = child->lgrp_parent;
947 			klgrpset_add(collapse, parent->lgrp_id);
948 			child = parent;
949 			parent = parent->lgrp_parent;
950 			continue;
951 		}
952 
953 #ifdef	DEBUG
954 		if (lgrp_topo_debug > 1)
955 			prom_printf("lgrp_lineage_add: latency 0x%x,"
956 			    " parent latency 0x%x\n",
957 			    latency, parent->lgrp_latency);
958 #endif	/* DEBUG */
959 		/*
960 		 * As we work our way from the old leaf to the root lgroup,
961 		 * new leaf resources should go in between two lgroups or into
962 		 * one of the parent lgroups somewhere along the line
963 		 */
964 		if (latency < parent->lgrp_latency) {
965 			lgrp_t	*intermed;
966 
967 			/*
968 			 * New leaf resources should go in between current
969 			 * child and parent
970 			 */
971 #ifdef	DEBUG
972 			if (lgrp_topo_debug > 0)
973 				prom_printf("lgrp_lineage_add: "
974 				    "latency < parent latency\n");
975 #endif	/* DEBUG */
976 
977 			/*
978 			 * Create lgroup with desired resources and insert it
979 			 * between child and parent
980 			 */
981 			lgrp_rsets_copy(child->lgrp_set, rset);
982 			lgrp_rsets_add(newleaf->lgrp_set, rset);
983 			if (nlevels >= lgrp_topo_levels) {
984 
985 #ifdef	DEBUG
986 				if (lgrp_topo_debug > 0) {
987 					prom_printf("lgrp_lineage_add: nlevels "
988 					    "%d > lgrp_topo_levels %d\n",
989 					    nlevels, lgrp_topo_levels);
990 					lgrp_rsets_print("rset ", rset);
991 				}
992 #endif	/* DEBUG */
993 
994 				if (parent == lgrp_root) {
995 					/*
996 					 * Don't proprogate new leaf resources
997 					 * to parent, if it already contains
998 					 * these resources
999 					 */
1000 					if (lgrp_rsets_member_all(
1001 					    parent->lgrp_set, newleaf->lgrp_id))
1002 						break;
1003 
1004 					total += lgrp_proprogate(newleaf, child,
1005 					    latency, &changes);
1006 					break;
1007 				}
1008 
1009 #ifdef	DEBUG
1010 				if (lgrp_topo_debug > 0) {
1011 					prom_printf("lgrp_lineage_add: "
1012 					    "replaced parent lgrp %d at 0x%p"
1013 					    " for lgrp %d\n",
1014 					    parent->lgrp_id, (void *)parent,
1015 					    child->lgrp_id);
1016 					lgrp_rsets_print("old parent"
1017 					    " resources:", parent->lgrp_set);
1018 					lgrp_rsets_print("new parent "
1019 					    "resources:", rset);
1020 				}
1021 #endif	/* DEBUG */
1022 				/*
1023 				 * Replace contents of parent with new
1024 				 * leaf + child resources since new leaf is
1025 				 * closer and shift its parent's resources to
1026 				 * its parent, etc. until root lgroup reached
1027 				 */
1028 				lgrp_rsets_replace(rset, latency, parent, 1);
1029 				if (*changed)
1030 					klgrpset_or(*changed, parent->lgrp_id);
1031 				total++;
1032 				proprogate++;
1033 			} else {
1034 
1035 #ifdef	DEBUG
1036 				if (lgrp_topo_debug > 0) {
1037 					prom_printf("lgrp_lineage_add: "
1038 					    "lgrp_new_parent(0x%p,%d)\n",
1039 					    (void *)child, latency);
1040 					lgrp_rsets_print("rset ", rset);
1041 				}
1042 #endif	/* DEBUG */
1043 
1044 				total += lgrp_new_parent(child, latency, rset,
1045 				    &changes);
1046 				intermed = child->lgrp_parent;
1047 				klgrpset_add(collapse, intermed->lgrp_id);
1048 				if (changed)
1049 					klgrpset_or(*changed, changes);
1050 				child = intermed;
1051 				proprogate++;
1052 #ifdef	DEBUG
1053 				if (lgrp_topo_debug > 0) {
1054 					prom_printf("lgrp_lineage_add: new "
1055 					    "parent lgrp %d at 0x%p for "
1056 					    "lgrp %d\n", intermed->lgrp_id,
1057 					    (void *)intermed, child->lgrp_id);
1058 					lgrp_rsets_print("new parent "
1059 					    "resources:", rset);
1060 				}
1061 #endif	/* DEBUG */
1062 				continue;
1063 			}
1064 
1065 		} else if (latency == parent->lgrp_latency) {
1066 			/*
1067 			 * New leaf resources should go into parent
1068 			 */
1069 #ifdef	DEBUG
1070 			if (lgrp_topo_debug > 0)
1071 				prom_printf("lgrp_lineage_add: latency == "
1072 				    "parent latency\n");
1073 #endif	/* DEBUG */
1074 
1075 			/*
1076 			 * It's already there, so don't need to do anything.
1077 			 */
1078 			if (lgrp_rsets_member_all(parent->lgrp_set,
1079 			    newleaf->lgrp_id))
1080 				break;
1081 
1082 			total += lgrp_proprogate(newleaf, child, latency,
1083 			    &changes);
1084 			parent = child->lgrp_parent;
1085 			klgrpset_add(collapse, parent->lgrp_id);
1086 			if (changed)
1087 				klgrpset_or(*changed, changes);
1088 
1089 			proprogate++;
1090 		}
1091 
1092 		child = parent;
1093 		parent = parent->lgrp_parent;
1094 	} while (parent != NULL);
1095 
1096 	/*
1097 	 * Consolidate any duplicate lgroups of ones just changed
1098 	 * Assume that there were no duplicates before last round of changes
1099 	 */
1100 #ifdef	DEBUG
1101 	if (lgrp_topo_debug > 1)
1102 		prom_printf("lgrp_lineage_add: collapsing dups....\n");
1103 #endif	/* DEBUG */
1104 
1105 	total += lgrp_collapse_dups(collapse, lgrp_collapse_equidist,
1106 	    &changes);
1107 	if (changed)
1108 		klgrpset_or(*changed, changes);
1109 
1110 #ifdef	DEBUG
1111 	if (lgrp_topo_debug > 1 && changed)
1112 		prom_printf("lgrp_lineage_add: changed %d lgrps: 0x%llx\n",
1113 		    total, (u_longlong_t)*changed);
1114 #endif	/* DEBUG */
1115 
1116 	return (total);
1117 }
1118 
1119 
1120 /*
1121  * Add leaf lgroup to lgroup topology
1122  */
1123 int
lgrp_leaf_add(lgrp_t * leaf,lgrp_t ** lgrps,int lgrp_count,klgrpset_t * changed)1124 lgrp_leaf_add(lgrp_t *leaf, lgrp_t **lgrps, int lgrp_count,
1125     klgrpset_t *changed)
1126 {
1127 	klgrpset_t	changes;
1128 	int		count;
1129 	int		i;
1130 	int		latency;
1131 
1132 	ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
1133 	    !lgrp_initialized);
1134 
1135 #ifdef	DEBUG
1136 	if (lgrp_topo_debug > 1)
1137 		prom_printf("\nlgrp_leaf_add(0x%p[%d],0x%p,%d,0x%p)\n",
1138 		    (void *)leaf, leaf->lgrp_id, (void *)lgrps, lgrp_count,
1139 		    (void *)changed);
1140 #endif	/* DEBUG */
1141 
1142 	count = 0;
1143 	if (changed)
1144 		klgrpset_clear(*changed);
1145 
1146 	/*
1147 	 * Initialize parent of leaf lgroup to root
1148 	 */
1149 	if (leaf->lgrp_parent == NULL) {
1150 		leaf->lgrp_parent = lgrp_root;
1151 		lgrp_root->lgrp_childcnt++;
1152 		klgrpset_add(lgrp_root->lgrp_children, leaf->lgrp_id);
1153 
1154 		klgrpset_or(lgrp_root->lgrp_leaves, leaf->lgrp_leaves);
1155 		lgrp_rsets_add(leaf->lgrp_set, lgrp_root->lgrp_set);
1156 
1157 #ifdef	DEBUG
1158 		if (lgrp_topo_debug > 1)
1159 			lgrp_rsets_print("lgrp_leaf_add: root lgrp resources",
1160 			    lgrp_root->lgrp_set);
1161 #endif	/* DEBUG */
1162 
1163 		if (changed) {
1164 			klgrpset_add(*changed, lgrp_root->lgrp_id);
1165 			klgrpset_add(*changed, leaf->lgrp_id);
1166 		}
1167 		count += 2;
1168 	}
1169 
1170 	/*
1171 	 * Can't add leaf lgroup to rest of topology (and vice versa) unless
1172 	 * latency for it is available
1173 	 */
1174 	latency = lgrp_plat_latency(leaf->lgrp_plathand, leaf->lgrp_plathand);
1175 	if (latency == 0) {
1176 #ifdef	DEBUG
1177 		if (lgrp_topo_debug > 1 && changed)
1178 			prom_printf("lgrp_leaf_add: changed %d lgrps: 0x%llx\n",
1179 			    count, (u_longlong_t)*changed);
1180 #endif	/* DEBUG */
1181 		return (count);
1182 	}
1183 
1184 	/*
1185 	 * Make sure that root and leaf lgroup latencies are set
1186 	 */
1187 	lgrp_root->lgrp_latency = lgrp_plat_latency(lgrp_root->lgrp_plathand,
1188 	    lgrp_root->lgrp_plathand);
1189 	leaf->lgrp_latency = latency;
1190 
1191 	/*
1192 	 * Add leaf to lineage of other leaves and vice versa
1193 	 * since leaves come into existence at different times
1194 	 */
1195 	for (i = 0; i < lgrp_count; i++) {
1196 		lgrp_t		*lgrp;
1197 
1198 		lgrp = lgrps[i];
1199 
1200 		/*
1201 		 * Skip non-existent lgroups, new leaf lgroup, and
1202 		 * non-leaf lgroups
1203 		 */
1204 		if (!LGRP_EXISTS(lgrp) || lgrp == leaf ||
1205 		    lgrp->lgrp_childcnt != 0) {
1206 #ifdef	DEBUG
1207 			if (lgrp_topo_debug > 1)
1208 				prom_printf("lgrp_leaf_add: skip "
1209 				    "lgrp %d at 0x%p\n",
1210 				    lgrp->lgrp_id, (void *)lgrp);
1211 #endif	/* DEBUG */
1212 			continue;
1213 		}
1214 
1215 #ifdef	DEBUG
1216 		if (lgrp_topo_debug > 0)
1217 			prom_printf("lgrp_leaf_add: lgrp %d (0x%p) =>"
1218 			    " lgrp %d (0x%p)\n",
1219 			    leaf->lgrp_id, (void *)leaf, lgrp->lgrp_id,
1220 			    (void *)lgrp);
1221 #endif	/* DEBUG */
1222 
1223 		count += lgrp_lineage_add(leaf, lgrp, &changes);
1224 		if (changed)
1225 			klgrpset_or(*changed, changes);
1226 
1227 		count += lgrp_lineage_add(lgrp, leaf, &changes);
1228 		if (changed)
1229 			klgrpset_or(*changed, changes);
1230 	}
1231 
1232 #ifdef	DEBUG
1233 	if (lgrp_topo_debug > 1 && changed)
1234 		prom_printf("lgrp_leaf_add: changed %d lgrps: 0x%llx\n",
1235 		    count, (u_longlong_t)*changed);
1236 #endif	/* DEBUG */
1237 
1238 	return (count);
1239 }
1240 
1241 
1242 /*
1243  * Remove resources of leaf from lgroup hierarchy
1244  */
1245 int
lgrp_leaf_delete(lgrp_t * leaf,lgrp_t ** lgrps,int lgrp_count,klgrpset_t * changed)1246 lgrp_leaf_delete(lgrp_t *leaf, lgrp_t **lgrps, int lgrp_count,
1247     klgrpset_t *changed)
1248 {
1249 	klgrpset_t	changes;
1250 	klgrpset_t	collapse;
1251 	int		count;
1252 	int		i;
1253 	lgrp_t		*lgrp;
1254 
1255 	ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
1256 	    !lgrp_initialized);
1257 
1258 	count = 0;
1259 	klgrpset_clear(collapse);
1260 	if (changed)
1261 		klgrpset_clear(*changed);
1262 
1263 	/*
1264 	 * Nothing to do if no leaf given
1265 	 */
1266 	if (leaf == NULL)
1267 		return (0);
1268 
1269 #ifdef	DEBUG
1270 	if (lgrp_topo_debug > 0)
1271 		prom_printf("lgrp_leaf_delete(0x%p[%d],0x%p,%d,0x%p)\n",
1272 		    (void *)leaf, leaf->lgrp_id, (void *)lgrps, lgrp_count,
1273 		    (void *)changed);
1274 #endif	/* DEBUG */
1275 
1276 	/*
1277 	 * Remove leaf from any lgroups containing its resources
1278 	 */
1279 	for (i = 0; i < lgrp_count; i++) {
1280 		lgrp = lgrps[i];
1281 		if (lgrp == NULL || lgrp->lgrp_id == LGRP_NONE ||
1282 		    !lgrp_rsets_member(lgrp->lgrp_set, leaf->lgrp_id))
1283 			continue;
1284 
1285 #ifdef	DEBUG
1286 		if (lgrp_topo_debug > 0)
1287 			prom_printf("lgrp_leaf_delete: remove leaf from"
1288 			    " lgrp %d at %p\n", lgrp->lgrp_id, (void *)lgrp);
1289 #endif	/* DEBUG */
1290 
1291 		lgrp_rsets_delete(lgrp, leaf->lgrp_id, 0);
1292 		klgrpset_del(lgrp->lgrp_leaves, leaf->lgrp_id);
1293 
1294 		klgrpset_add(collapse, lgrp->lgrp_id);
1295 		count++;
1296 	}
1297 
1298 	/*
1299 	 * Remove leaf and its ancestors that don't have any other children
1300 	 */
1301 #ifdef	DEBUG
1302 	if (lgrp_topo_debug > 1)
1303 		prom_printf("lgrp_leaf_delete: remove leaf and ancestors\n");
1304 #endif	/* DEBUG */
1305 
1306 	count += lgrp_ancestor_delete(leaf, &changes);
1307 	klgrpset_or(collapse, changes);
1308 	klgrpset_add(collapse, leaf->lgrp_id);
1309 	count++;
1310 	lgrp_destroy(leaf);
1311 
1312 	/*
1313 	 * Consolidate any duplicate lgroups of ones just changed
1314 	 * Assume that there were no duplicates before last round of changes
1315 	 */
1316 #ifdef	DEBUG
1317 	if (lgrp_topo_debug > 1)
1318 		prom_printf("lgrp_leaf_delete: collapsing dups\n");
1319 #endif	/* DEBUG */
1320 	count += lgrp_collapse_dups(collapse, lgrp_collapse_equidist,
1321 	    &changes);
1322 	klgrpset_or(collapse, changes);
1323 	if (changed)
1324 		klgrpset_copy(*changed, collapse);
1325 
1326 #ifdef	DEBUG
1327 	if (lgrp_topo_debug > 1 && changed)
1328 		prom_printf("lgrp_leaf_delete: changed %d lgrps: 0x%llx\n",
1329 		    count, (u_longlong_t)*changed);
1330 #endif	/* DEBUG */
1331 
1332 	return (count);
1333 }
1334 
1335 
1336 /*
1337  * Flatten lgroup topology down to height specified
1338  */
1339 int
lgrp_topo_flatten(int levels,lgrp_t ** lgrps,int lgrp_count,klgrpset_t * changed)1340 lgrp_topo_flatten(int levels, lgrp_t **lgrps, int lgrp_count,
1341     klgrpset_t *changed)
1342 {
1343 	int	count;
1344 	int	i;
1345 	lgrp_t	*lgrp;
1346 	lgrp_handle_t hdl;
1347 
1348 	/*
1349 	 * Only flatten down to 2 level for now
1350 	 */
1351 	if (levels != 2)
1352 		return (0);
1353 
1354 	/*
1355 	 * Look for non-leaf lgroups to remove and leaf lgroups to reparent
1356 	 */
1357 	count = 0;
1358 	for (i = 0; i <= lgrp_count; i++) {
1359 		/*
1360 		 * Skip non-existent lgroups and root
1361 		 */
1362 		lgrp = lgrps[i];
1363 		if (!LGRP_EXISTS(lgrp))
1364 			continue;
1365 
1366 		hdl = lgrp->lgrp_plathand;
1367 
1368 		if (lgrp == lgrp_root) {
1369 			lgrp->lgrp_latency = lgrp_plat_latency(hdl, hdl);
1370 			continue;
1371 		}
1372 
1373 		if (lgrp->lgrp_childcnt > 0) {
1374 			lgrp_t	*parent;
1375 
1376 			/*
1377 			 * Remove non-leaf lgroup from lgroup topology
1378 			 */
1379 			parent = lgrp->lgrp_parent;
1380 			if (changed) {
1381 				klgrpset_add(*changed, lgrp->lgrp_id);
1382 				klgrpset_add(*changed, parent->lgrp_id);
1383 				count += 2;
1384 			}
1385 			if (parent) {
1386 				klgrpset_del(parent->lgrp_children,
1387 				    lgrp->lgrp_id);
1388 				parent->lgrp_childcnt--;
1389 			}
1390 			lgrp_destroy(lgrp);
1391 		} else if (lgrp->lgrp_parent != lgrp_root) {
1392 			/*
1393 			 * Reparent leaf lgroup to root
1394 			 */
1395 			if (changed) {
1396 				klgrpset_add(*changed, lgrp_root->lgrp_id);
1397 				klgrpset_add(*changed, lgrp->lgrp_id);
1398 				count += 2;
1399 			}
1400 			lgrp->lgrp_parent = lgrp_root;
1401 			klgrpset_add(lgrp_root->lgrp_children, lgrp->lgrp_id);
1402 			lgrp_root->lgrp_childcnt++;
1403 			klgrpset_add(lgrp_root->lgrp_leaves, lgrp->lgrp_id);
1404 
1405 			lgrp->lgrp_latency = lgrp_plat_latency(hdl, hdl);
1406 		}
1407 	}
1408 
1409 	return (count);
1410 }
1411 
1412 
1413 /*
1414  * Return current height limit for lgroup topology
1415  */
1416 int
lgrp_topo_ht_limit(void)1417 lgrp_topo_ht_limit(void)
1418 {
1419 	return (lgrp_topo_levels);
1420 }
1421 
1422 
1423 /*
1424  * Return default height limit for lgroup topology
1425  */
1426 int
lgrp_topo_ht_limit_default(void)1427 lgrp_topo_ht_limit_default(void)
1428 {
1429 	return (LGRP_TOPO_LEVELS);
1430 }
1431 
1432 
1433 /*
1434  * Set height limit for lgroup topology
1435  */
1436 int
lgrp_topo_ht_limit_set(int ht)1437 lgrp_topo_ht_limit_set(int ht)
1438 {
1439 	if (ht > LGRP_TOPO_LEVELS_MAX)
1440 		lgrp_topo_levels = LGRP_TOPO_LEVELS_MAX;
1441 	else
1442 		lgrp_topo_levels = ht;
1443 
1444 	return (ht);
1445 }
1446 
1447 
1448 /*
1449  * Update lgroup topology for any leaves that don't have their latency set
1450  *
1451  * This may happen on some machines when the lgroup platform support doesn't
1452  * know the latencies between nodes soon enough to provide it when the
1453  * resources are being added.  If the lgroup platform code needs to probe
1454  * memory to determine the latencies between nodes, it must wait until the
1455  * CPUs become active so at least one CPU in each node can probe memory in
1456  * each node.
1457  */
1458 int
lgrp_topo_update(lgrp_t ** lgrps,int lgrp_count,klgrpset_t * changed)1459 lgrp_topo_update(lgrp_t **lgrps, int lgrp_count, klgrpset_t *changed)
1460 {
1461 	klgrpset_t	changes;
1462 	int		count;
1463 	int		i;
1464 	lgrp_t		*lgrp;
1465 
1466 	count = 0;
1467 	if (changed)
1468 		klgrpset_clear(*changed);
1469 
1470 	/*
1471 	 * For UMA machines, make sure that root lgroup contains all
1472 	 * resources.  The root lgrp should also name itself as its own leaf
1473 	 */
1474 	if (nlgrps == 1) {
1475 		for (i = 0; i < LGRP_RSRC_COUNT; i++)
1476 			klgrpset_add(lgrp_root->lgrp_set[i],
1477 			    lgrp_root->lgrp_id);
1478 		klgrpset_add(lgrp_root->lgrp_leaves, lgrp_root->lgrp_id);
1479 		return (0);
1480 	}
1481 
1482 	mutex_enter(&cpu_lock);
1483 	pause_cpus(NULL, NULL);
1484 
1485 	/*
1486 	 * Look for any leaf lgroup without its latency set, finish adding it
1487 	 * to the lgroup topology assuming that it exists and has the root
1488 	 * lgroup as its parent, and update the memory nodes of all lgroups
1489 	 * that have it as a memory resource.
1490 	 */
1491 	for (i = 0; i < lgrp_count; i++) {
1492 		lgrp = lgrps[i];
1493 
1494 		/*
1495 		 * Skip non-existent and non-leaf lgroups and any lgroup
1496 		 * with its latency set already
1497 		 */
1498 		if (lgrp == NULL || lgrp->lgrp_id == LGRP_NONE ||
1499 		    lgrp->lgrp_childcnt != 0 || lgrp->lgrp_latency != 0)
1500 			continue;
1501 
1502 #ifdef	DEBUG
1503 		if (lgrp_topo_debug > 1) {
1504 			prom_printf("\nlgrp_topo_update: updating lineage "
1505 			    "of lgrp %d at 0x%p\n", lgrp->lgrp_id,
1506 			    (void *)lgrp);
1507 		}
1508 #endif	/* DEBUG */
1509 
1510 		count += lgrp_leaf_add(lgrp, lgrps, lgrp_count, &changes);
1511 		if (changed)
1512 			klgrpset_or(*changed, changes);
1513 
1514 		if (!klgrpset_isempty(changes))
1515 			(void) lgrp_mnode_update(changes, NULL);
1516 
1517 #ifdef	DEBUG
1518 		if (lgrp_topo_debug > 1 && changed)
1519 			prom_printf("lgrp_topo_update: changed %d lgrps: "
1520 			    "0x%llx\n",
1521 			    count, (u_longlong_t)*changed);
1522 #endif	/* DEBUG */
1523 	}
1524 
1525 	if (lgrp_topo_levels < LGRP_TOPO_LEVELS && lgrp_topo_levels == 2) {
1526 		count += lgrp_topo_flatten(2, lgrps, lgrp_count, changed);
1527 		(void) lpl_topo_flatten(2);
1528 	}
1529 
1530 	start_cpus();
1531 	mutex_exit(&cpu_lock);
1532 
1533 	return (count);
1534 }
1535 
1536 #ifdef	DEBUG
1537 void
lgrp_print(lgrp_t * lgrp)1538 lgrp_print(lgrp_t *lgrp)
1539 {
1540 	lgrp_t	*parent;
1541 
1542 	prom_printf("LGRP %d", lgrp->lgrp_id);
1543 	if (lgrp->lgrp_childcnt == 0)
1544 		prom_printf(" (plathand %p)\n",
1545 		    (void *)lgrp->lgrp_plathand);
1546 	else
1547 		prom_printf("\n");
1548 
1549 	prom_printf("\tlatency %d\n", lgrp->lgrp_latency);
1550 
1551 	lgrp_rsets_print("\tresources", lgrp->lgrp_set);
1552 
1553 	parent = lgrp->lgrp_parent;
1554 	prom_printf("\tparent 0x%p", (void *)parent);
1555 	if (parent)
1556 		prom_printf("[%d]\n", parent->lgrp_id);
1557 	else
1558 		prom_printf("\n");
1559 
1560 	prom_printf("\tchild count %d, children ", lgrp->lgrp_childcnt);
1561 	klgrpset_print(lgrp->lgrp_children);
1562 
1563 	prom_printf("\tleaves ");
1564 	klgrpset_print(lgrp->lgrp_leaves);
1565 }
1566 
1567 
1568 void
lgrp_topo_print(lgrp_t ** lgrps,int lgrp_max)1569 lgrp_topo_print(lgrp_t **lgrps, int lgrp_max)
1570 {
1571 	klgrpset_t	siblings;
1572 
1573 	lgrp_print(lgrp_root);
1574 	siblings = lgrp_root->lgrp_children;
1575 	while (!klgrpset_isempty(siblings)) {
1576 		klgrpset_t	children;
1577 		int		i;
1578 
1579 		klgrpset_clear(children);
1580 		for (i = 0; i <= lgrp_max; i++) {
1581 			lgrp_t	*lgrp;
1582 
1583 			lgrp = lgrps[i];
1584 			if (lgrp == NULL || !klgrpset_ismember(siblings, i))
1585 				continue;
1586 			lgrp_print(lgrp);
1587 			klgrpset_or(children, lgrp->lgrp_children);
1588 		}
1589 		klgrpset_copy(siblings, children);
1590 	}
1591 }
1592 #endif	/* DEBUG */
1593