1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
65 	/* Only 36 bits needed for full 64-bit address space. */
74  * For a full 64-bit address space, there are 36 bits in play in an
79  * +----+----+----+----+----+----+----+----+----+--------
99 	(esid2base(ua->ua_base, ua->ua_level) == ua->ua_base)
108 	mask = ~((1ULL << shift) - 1);  in esid2base()
123 	idx = esid2idx(esid, parent->ua_level);  in make_new_leaf()
124 	KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!"));  in make_new_leaf()
130 	child->ua_level = UAD_LEAF_LEVEL;  in make_new_leaf()
131 	child->ua_base = esid2base(esid, child->ua_level);  in make_new_leaf()
132 	idx = esid2idx(esid, child->ua_level);  in make_new_leaf()
133 	child->u.slb_entries[idx].slbv = slbv;  in make_new_leaf()
134 	child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;  in make_new_leaf()
135 	setbit(&child->ua_alloc, idx);  in make_new_leaf()
137 	retval = &child->u.slb_entries[idx];  in make_new_leaf()
146 	idx = esid2idx(esid, parent->ua_level);  in make_new_leaf()
147 	parent->u.ua_child[idx] = child;  in make_new_leaf()
148 	setbit(&parent->ua_alloc, idx);  in make_new_leaf()
160 	struct slbtnode *child, *inter;  in make_intermediate()  local
163 	idx = esid2idx(esid, parent->ua_level);  in make_intermediate()
164 	child = parent->u.ua_child[idx];  in make_intermediate()
165 	KASSERT(esid2base(esid, child->ua_level) != child->ua_base,  in make_intermediate()
170 	 * meet.  It must be lower than parent->ua_level or we would  in make_intermediate()
173 	level = child->ua_level + 1;  in make_intermediate()
175 	    esid2base(child->ua_base, level))  in make_intermediate()
177 	KASSERT(level < parent->ua_level,  in make_intermediate()
180 	    level, esid, child->ua_base, parent));  in make_intermediate()
183 	inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO);  in make_intermediate()
184 	KASSERT(inter != NULL, ("unhandled NULL case"));  in make_intermediate()
187 	inter->ua_level = level;  in make_intermediate()
188 	inter->ua_base = esid2base(esid, inter->ua_level);  in make_intermediate()
189 	idx = esid2idx(child->ua_base, inter->ua_level);  in make_intermediate()
190 	inter->u.ua_child[idx] = child;  in make_intermediate()
191 	setbit(&inter->ua_alloc, idx);  in make_intermediate()
195 	idx = esid2idx(inter->ua_base, parent->ua_level);  in make_intermediate()
196 	parent->u.ua_child[idx] = inter;  in make_intermediate()
197 	setbit(&parent->ua_alloc, idx);  in make_intermediate()
199 	return (inter);  in make_intermediate()
211 	 * Figure out if this is a large-page mapping.  in kernel_va_to_slbv()
236 	ua = pm->pm_slb_tree_root;  in user_va_to_slb_entry()
240 		    ua->ua_base, ua->ua_level));  in user_va_to_slb_entry()
241 		idx = esid2idx(esid, ua->ua_level);  in user_va_to_slb_entry()
247 		if (ua->ua_level == UAD_LEAF_LEVEL)  in user_va_to_slb_entry()
248 			return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ?  in user_va_to_slb_entry()
249 			    &ua->u.slb_entries[idx] : NULL);  in user_va_to_slb_entry()
257 		ua = ua->u.ua_child[idx];  in user_va_to_slb_entry()
259 		    esid2base(esid, ua->ua_level) != ua->ua_base)  in user_va_to_slb_entry()
286 	return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);  in va_to_vsid()
293 	struct slbtnode *ua, *next, *inter;  in allocate_user_vsid()  local
306 	ua = pm->pm_slb_tree_root;  in allocate_user_vsid()
311 		   ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));  in allocate_user_vsid()
312 		idx = esid2idx(esid, ua->ua_level);  in allocate_user_vsid()
314 		if (ua->ua_level == UAD_LEAF_LEVEL) {  in allocate_user_vsid()
315 			ua->u.slb_entries[idx].slbv = slbv;  in allocate_user_vsid()
317 			ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT)  in allocate_user_vsid()
319 			setbit(&ua->ua_alloc, idx);  in allocate_user_vsid()
320 			slb = &ua->u.slb_entries[idx];  in allocate_user_vsid()
324 		next = ua->u.ua_child[idx];  in allocate_user_vsid()
334 		if (esid2base(esid, next->ua_level) != next->ua_base) {  in allocate_user_vsid()
335 			inter = make_intermediate(esid, ua);  in allocate_user_vsid()
336 			slb = make_new_leaf(esid, slbv, inter);  in allocate_user_vsid()
345 	 * SLB mapping, so pre-spill this entry.  in allocate_user_vsid()
361 	ua = pm->pm_slb_tree_root;  in free_vsid()
365 		   ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level));  in free_vsid()
367 		idx = esid2idx(esid, ua->ua_level);  in free_vsid()
368 		if (ua->ua_level == UAD_LEAF_LEVEL) {  in free_vsid()
369 			ua->u.slb_entries[idx].slbv = 0;  in free_vsid()
371 			ua->u.slb_entries[idx].slbe = 0;  in free_vsid()
372 			clrbit(&ua->ua_alloc, idx);  in free_vsid()
376 		ua = ua->u.ua_child[idx];  in free_vsid()
378 		    esid2base(esid, ua->ua_level) != ua->ua_base) {  in free_vsid()
393 		if (ua->ua_level != UAD_LEAF_LEVEL) {  in free_slb_tree_node()
394 			if (ua->u.ua_child[idx] != NULL)  in free_slb_tree_node()
395 				free_slb_tree_node(ua->u.ua_child[idx]);  in free_slb_tree_node()
397 			if (ua->u.slb_entries[idx].slbv != 0)  in free_slb_tree_node()
398 				moea64_release_vsid(ua->u.slb_entries[idx].slbv  in free_slb_tree_node()
410 	free_slb_tree_node(pm->pm_slb_tree_root);  in slb_free_tree()
420 	root->ua_level = UAD_ROOT_LEVEL;  in slb_alloc_tree()
478 	if (pm->pm_slb_len < n_slbs) {  in slb_insert_user()
479 		i = pm->pm_slb_len;  in slb_insert_user()
480 		pm->pm_slb_len++;  in slb_insert_user()
486 	pm->pm_slb[i] = slb;  in slb_insert_user()
545 /* Handle kernel SLB faults -- runs in real mode, all seat belts off */
598 	if (pm->pm_slb == NULL)  in handle_user_slb_spill()
599 		return (-1);  in handle_user_slb_spill()
607 		/* allocate_vsid auto-spills it */  in handle_user_slb_spill()
612 		 * XXX: Per-thread SLB caches would be better.  in handle_user_slb_spill()
614 		for (i = 0; i < pm->pm_slb_len; i++)  in handle_user_slb_spill()
615 			if (pm->pm_slb[i] == user_entry)  in handle_user_slb_spill()
618 		if (i == pm->pm_slb_len)  in handle_user_slb_spill()