xref: /titanic_52/usr/src/uts/sun4u/opl/io/drmach.c (revision 28406508b3a9fa1994317b410fa4afd93b3f8cc4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 
29 #include <sys/debug.h>
30 #include <sys/types.h>
31 #include <sys/varargs.h>
32 #include <sys/errno.h>
33 #include <sys/cred.h>
34 #include <sys/dditypes.h>
35 #include <sys/devops.h>
36 #include <sys/modctl.h>
37 #include <sys/poll.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/sunndi.h>
42 #include <sys/ndi_impldefs.h>
43 #include <sys/stat.h>
44 #include <sys/kmem.h>
45 #include <sys/vmem.h>
46 #include <sys/opl_olympus_regs.h>
47 #include <sys/cpuvar.h>
48 #include <sys/cpupart.h>
49 #include <sys/mem_config.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/systm.h>
52 #include <sys/machsystm.h>
53 #include <sys/autoconf.h>
54 #include <sys/cmn_err.h>
55 #include <sys/sysmacros.h>
56 #include <sys/x_call.h>
57 #include <sys/promif.h>
58 #include <sys/prom_plat.h>
59 #include <sys/membar.h>
60 #include <vm/seg_kmem.h>
61 #include <sys/mem_cage.h>
62 #include <sys/stack.h>
63 #include <sys/archsystm.h>
64 #include <vm/hat_sfmmu.h>
65 #include <sys/pte.h>
66 #include <sys/mmu.h>
67 #include <sys/cpu_module.h>
68 #include <sys/obpdefs.h>
69 #include <sys/note.h>
70 #include <sys/ontrap.h>
71 #include <sys/cpu_sgnblk_defs.h>
72 #include <sys/opl.h>
73 
74 
75 #include <sys/promimpl.h>
76 #include <sys/prom_plat.h>
77 #include <sys/kobj.h>
78 
79 #include <sys/sysevent.h>
80 #include <sys/sysevent/dr.h>
81 #include <sys/sysevent/eventdefs.h>
82 
83 #include <sys/drmach.h>
84 #include <sys/dr_util.h>
85 
86 #include <sys/fcode.h>
87 #include <sys/opl_cfg.h>
88 
89 extern void		bcopy32_il(uint64_t, uint64_t);
90 extern void		flush_cache_il(void);
91 extern void		drmach_sleep_il(void);
92 
93 typedef struct {
94 	struct drmach_node	*node;
95 	void			*data;
96 } drmach_node_walk_args_t;
97 
98 typedef struct drmach_node {
99 	void		*here;
100 
101 	pnode_t		(*get_dnode)(struct drmach_node *node);
102 	int		(*walk)(struct drmach_node *node, void *data,
103 				int (*cb)(drmach_node_walk_args_t *args));
104 	dev_info_t	*(*n_getdip)(struct drmach_node *node);
105 	int		(*n_getproplen)(struct drmach_node *node, char *name,
106 				int *len);
107 	int		(*n_getprop)(struct drmach_node *node, char *name,
108 				void *buf, int len);
109 	int		(*get_parent)(struct drmach_node *node,
110 				struct drmach_node *pnode);
111 } drmach_node_t;
112 
113 typedef struct {
114 	int		 min_index;
115 	int		 max_index;
116 	int		 arr_sz;
117 	drmachid_t	*arr;
118 } drmach_array_t;
119 
120 typedef struct {
121 	void		*isa;
122 
123 	void		(*dispose)(drmachid_t);
124 	sbd_error_t	*(*release)(drmachid_t);
125 	sbd_error_t	*(*status)(drmachid_t, drmach_status_t *);
126 
127 	char		 name[MAXNAMELEN];
128 } drmach_common_t;
129 
130 typedef	struct {
131 	uint32_t	core_present;
132 	uint32_t	core_hotadded;
133 	uint32_t	core_started;
134 } drmach_cmp_t;
135 
136 typedef struct {
137 	drmach_common_t	 cm;
138 	int		 bnum;
139 	int		 assigned;
140 	int		 powered;
141 	int		 connected;
142 	int		 cond;
143 	drmach_node_t	*tree;
144 	drmach_array_t	*devices;
145 	int		boot_board;	/* if board exists on bootup */
146 	drmach_cmp_t	cores[OPL_MAX_COREID_PER_BOARD];
147 } drmach_board_t;
148 
149 typedef struct {
150 	drmach_common_t	 cm;
151 	drmach_board_t	*bp;
152 	int		 unum;
153 	int		portid;
154 	int		 busy;
155 	int		 powered;
156 	const char	*type;
157 	drmach_node_t	*node;
158 } drmach_device_t;
159 
160 typedef struct drmach_cpu {
161 	drmach_device_t  dev;
162 	processorid_t    cpuid;
163 	int		sb;
164 	int		chipid;
165 	int		coreid;
166 	int		strandid;
167 	int		status;
168 #define	OPL_CPU_HOTADDED	1
169 } drmach_cpu_t;
170 
171 typedef struct drmach_mem {
172 	drmach_device_t  dev;
173 	uint64_t	slice_base;
174 	uint64_t	slice_size;
175 	uint64_t	base_pa;	/* lowest installed memory base */
176 	uint64_t	nbytes;		/* size of installed memory */
177 	struct memlist *memlist;
178 } drmach_mem_t;
179 
180 typedef struct drmach_io {
181 	drmach_device_t  dev;
182 	int	channel;
183 	int	leaf;
184 } drmach_io_t;
185 
186 typedef struct drmach_domain_info {
187 	uint32_t	floating;
188 	int		allow_dr;
189 } drmach_domain_info_t;
190 
191 drmach_domain_info_t drmach_domain;
192 
193 typedef struct {
194 	int		 flags;
195 	drmach_device_t	*dp;
196 	sbd_error_t	*err;
197 	dev_info_t	*dip;
198 } drmach_config_args_t;
199 
200 typedef struct {
201 	drmach_board_t	*obj;
202 	int		 ndevs;
203 	void		*a;
204 	sbd_error_t	*(*found)(void *a, const char *, int, drmachid_t);
205 	sbd_error_t	*err;
206 } drmach_board_cb_data_t;
207 
208 static drmach_array_t	*drmach_boards;
209 
210 static sbd_error_t	*drmach_device_new(drmach_node_t *,
211 				drmach_board_t *, int, drmachid_t *);
212 static sbd_error_t	*drmach_cpu_new(drmach_device_t *, drmachid_t *);
213 static sbd_error_t	*drmach_mem_new(drmach_device_t *, drmachid_t *);
214 static sbd_error_t	*drmach_io_new(drmach_device_t *, drmachid_t *);
215 
216 static dev_info_t	*drmach_node_ddi_get_dip(drmach_node_t *np);
217 static int		 drmach_node_ddi_get_prop(drmach_node_t *np,
218 				char *name, void *buf, int len);
219 static int		 drmach_node_ddi_get_proplen(drmach_node_t *np,
220 				char *name, int *len);
221 
222 static int 		drmach_get_portid(drmach_node_t *);
223 static	sbd_error_t	*drmach_i_status(drmachid_t, drmach_status_t *);
224 static int		opl_check_dr_status();
225 static void		drmach_io_dispose(drmachid_t);
226 static sbd_error_t	*drmach_io_release(drmachid_t);
227 static sbd_error_t	*drmach_io_status(drmachid_t, drmach_status_t *);
228 static int 		drmach_init(void);
229 static void 		drmach_fini(void);
230 static void		drmach_swap_pa(drmach_mem_t *, drmach_mem_t *);
231 static drmach_board_t	*drmach_get_board_by_bnum(int);
232 
233 /* options for the second argument in drmach_add_remove_cpu() */
234 #define	HOTADD_CPU	1
235 #define	HOTREMOVE_CPU	2
236 
237 #define	ON_BOARD_CORE_NUM(x)	(((uint_t)(x) / OPL_MAX_STRANDID_PER_CORE) & \
238 	(OPL_MAX_COREID_PER_BOARD - 1))
239 
240 extern struct cpu	*SIGBCPU;
241 
242 static int		drmach_name2type_idx(char *);
243 static drmach_board_t	*drmach_board_new(int, int);
244 
245 #ifdef DEBUG
246 
247 #define	DRMACH_PR		if (drmach_debug) printf
248 int drmach_debug = 1;		 /* set to non-zero to enable debug messages */
249 #else
250 
251 #define	DRMACH_PR		_NOTE(CONSTANTCONDITION) if (0) printf
252 #endif /* DEBUG */
253 
254 
255 #define	DRMACH_OBJ(id)		((drmach_common_t *)id)
256 
257 #define	DRMACH_NULL_ID(id)	((id) == 0)
258 
259 #define	DRMACH_IS_BOARD_ID(id)	\
260 	((id != 0) &&		\
261 	(DRMACH_OBJ(id)->isa == (void *)drmach_board_new))
262 
263 #define	DRMACH_IS_CPU_ID(id)	\
264 	((id != 0) &&		\
265 	(DRMACH_OBJ(id)->isa == (void *)drmach_cpu_new))
266 
267 #define	DRMACH_IS_MEM_ID(id)	\
268 	((id != 0) &&		\
269 	(DRMACH_OBJ(id)->isa == (void *)drmach_mem_new))
270 
271 #define	DRMACH_IS_IO_ID(id)	\
272 	((id != 0) &&		\
273 	(DRMACH_OBJ(id)->isa == (void *)drmach_io_new))
274 
275 #define	DRMACH_IS_DEVICE_ID(id)					\
276 	((id != 0) &&						\
277 	(DRMACH_OBJ(id)->isa == (void *)drmach_cpu_new ||	\
278 	    DRMACH_OBJ(id)->isa == (void *)drmach_mem_new ||	\
279 	    DRMACH_OBJ(id)->isa == (void *)drmach_io_new))
280 
281 #define	DRMACH_IS_ID(id)					\
282 	((id != 0) &&						\
283 	(DRMACH_OBJ(id)->isa == (void *)drmach_board_new ||	\
284 	    DRMACH_OBJ(id)->isa == (void *)drmach_cpu_new ||	\
285 	    DRMACH_OBJ(id)->isa == (void *)drmach_mem_new ||	\
286 	    DRMACH_OBJ(id)->isa == (void *)drmach_io_new))
287 
288 #define	DRMACH_INTERNAL_ERROR() \
289 	drerr_new(1, EOPL_INTERNAL, drmach_ie_fmt, __LINE__)
290 
291 static char		*drmach_ie_fmt = "drmach.c %d";
292 
293 static struct {
294 	const char	*name;
295 	const char	*type;
296 	sbd_error_t	*(*new)(drmach_device_t *, drmachid_t *);
297 } drmach_name2type[] = {
298 	{ "cpu",	DRMACH_DEVTYPE_CPU,		drmach_cpu_new },
299 	{ "pseudo-mc",	DRMACH_DEVTYPE_MEM,		drmach_mem_new },
300 	{ "pci",	DRMACH_DEVTYPE_PCI,		drmach_io_new  },
301 };
302 
303 /* utility */
304 #define	MBYTE	(1048576ull)
305 
306 /*
307  * drmach autoconfiguration data structures and interfaces
308  */
309 
310 extern struct mod_ops mod_miscops;
311 
312 static struct modlmisc modlmisc = {
313 	&mod_miscops,
314 	"OPL DR 1.1"
315 };
316 
317 static struct modlinkage modlinkage = {
318 	MODREV_1,
319 	(void *)&modlmisc,
320 	NULL
321 };
322 
323 static krwlock_t drmach_boards_rwlock;
324 
325 typedef const char	*fn_t;
326 
327 int
328 _init(void)
329 {
330 	int err;
331 
332 	if ((err = drmach_init()) != 0) {
333 		return (err);
334 	}
335 
336 	if ((err = mod_install(&modlinkage)) != 0) {
337 		drmach_fini();
338 	}
339 
340 	return (err);
341 }
342 
343 int
344 _fini(void)
345 {
346 	int	err;
347 
348 	if ((err = mod_remove(&modlinkage)) == 0)
349 		drmach_fini();
350 
351 	return (err);
352 }
353 
354 int
355 _info(struct modinfo *modinfop)
356 {
357 	return (mod_info(&modlinkage, modinfop));
358 }
359 
360 struct drmach_mc_lookup {
361 	int	bnum;
362 	drmach_board_t	*bp;
363 	dev_info_t *dip;	/* rv - set if found */
364 };
365 
366 #define	_ptob64(p) ((uint64_t)(p) << PAGESHIFT)
367 #define	_b64top(b) ((pgcnt_t)((b) >> PAGESHIFT))
368 
369 static int
370 drmach_setup_mc_info(dev_info_t *dip, drmach_mem_t *mp)
371 {
372 	uint64_t	memory_ranges[128];
373 	int len;
374 	struct memlist	*ml;
375 	int rv;
376 	hwd_sb_t *hwd;
377 	hwd_memory_t *pm;
378 
379 	len = sizeof (memory_ranges);
380 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
381 		DDI_PROP_DONTPASS, "sb-mem-ranges",
382 	    (caddr_t)&memory_ranges[0], &len) != DDI_PROP_SUCCESS) {
383 		mp->slice_base = 0;
384 		mp->slice_size = 0;
385 		return (-1);
386 	}
387 	mp->slice_base = memory_ranges[0];
388 	mp->slice_size = memory_ranges[1];
389 
390 	if (!mp->dev.bp->boot_board) {
391 		int i;
392 
393 		rv = opl_read_hwd(mp->dev.bp->bnum, NULL,  NULL, NULL, &hwd);
394 
395 		if (rv != 0) {
396 			return (-1);
397 		}
398 
399 		ml = NULL;
400 		pm = &hwd->sb_cmu.cmu_memory;
401 		for (i = 0; i < HWD_MAX_MEM_CHUNKS; i++) {
402 			if (pm->mem_chunks[i].chnk_size > 0) {
403 				ml = memlist_add_span(ml,
404 					pm->mem_chunks[i].chnk_start_address,
405 					pm->mem_chunks[i].chnk_size);
406 			}
407 		}
408 	} else {
409 		/*
410 		 * we intersect phys_install to get base_pa.
411 		 * This only works at bootup time.
412 		 */
413 
414 		memlist_read_lock();
415 		ml = memlist_dup(phys_install);
416 		memlist_read_unlock();
417 
418 		ml = memlist_del_span(ml, 0ull, mp->slice_base);
419 		if (ml) {
420 			uint64_t basepa, endpa;
421 			endpa = _ptob64(physmax + 1);
422 
423 			basepa = mp->slice_base + mp->slice_size;
424 
425 			ml = memlist_del_span(ml, basepa, endpa - basepa);
426 		}
427 	}
428 
429 	if (ml) {
430 		uint64_t nbytes = 0;
431 		struct memlist *p;
432 		for (p = ml; p; p = p->next) {
433 			nbytes += p->size;
434 		}
435 		if ((mp->nbytes = nbytes) > 0)
436 			mp->base_pa = ml->address;
437 		else
438 			mp->base_pa = 0;
439 		mp->memlist = ml;
440 	} else {
441 		mp->base_pa = 0;
442 		mp->nbytes = 0;
443 	}
444 	return (0);
445 }
446 
447 
448 struct drmach_hotcpu {
449 	drmach_board_t *bp;
450 	int	bnum;
451 	int	core_id;
452 	int 	rv;
453 	int	option;
454 };
455 
456 static int
457 drmach_cpu_cb(dev_info_t *dip, void *arg)
458 {
459 	struct drmach_hotcpu *p = (struct drmach_hotcpu *)arg;
460 	char name[OBP_MAXDRVNAME];
461 	int len = OBP_MAXDRVNAME;
462 	int bnum, core_id, strand_id;
463 	drmach_board_t *bp;
464 
465 	if (dip == ddi_root_node()) {
466 		return (DDI_WALK_CONTINUE);
467 	}
468 
469 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
470 	    DDI_PROP_DONTPASS, "name",
471 	    (caddr_t)name, &len) != DDI_PROP_SUCCESS) {
472 		return (DDI_WALK_PRUNECHILD);
473 	}
474 
475 	/* only cmp has board number */
476 	bnum = -1;
477 	len = sizeof (bnum);
478 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
479 	    DDI_PROP_DONTPASS, OBP_BOARDNUM,
480 	    (caddr_t)&bnum, &len) != DDI_PROP_SUCCESS) {
481 		bnum = -1;
482 	}
483 
484 	if (strcmp(name, "cmp") == 0) {
485 		if (bnum != p->bnum)
486 			return (DDI_WALK_PRUNECHILD);
487 		return (DDI_WALK_CONTINUE);
488 	}
489 	/* we have already pruned all unwanted cores and cpu's above */
490 	if (strcmp(name, "core") == 0) {
491 		return (DDI_WALK_CONTINUE);
492 	}
493 	if (strcmp(name, "cpu") == 0) {
494 		processorid_t cpuid;
495 		len = sizeof (cpuid);
496 		if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
497 		    DDI_PROP_DONTPASS, "cpuid",
498 		    (caddr_t)&cpuid, &len) != DDI_PROP_SUCCESS) {
499 			p->rv = -1;
500 			return (DDI_WALK_TERMINATE);
501 		}
502 
503 		core_id = p->core_id;
504 
505 		bnum = LSB_ID(cpuid);
506 
507 		if (ON_BOARD_CORE_NUM(cpuid) != core_id)
508 			return (DDI_WALK_CONTINUE);
509 
510 		bp = p->bp;
511 		ASSERT(bnum == bp->bnum);
512 
513 		if (p->option == HOTADD_CPU) {
514 			if (prom_hotaddcpu(cpuid) != 0) {
515 				p->rv = -1;
516 				return (DDI_WALK_TERMINATE);
517 			}
518 			strand_id = STRAND_ID(cpuid);
519 			bp->cores[core_id].core_hotadded |= (1 << strand_id);
520 		} else if (p->option == HOTREMOVE_CPU) {
521 			if (prom_hotremovecpu(cpuid) != 0) {
522 				p->rv = -1;
523 				return (DDI_WALK_TERMINATE);
524 			}
525 			strand_id = STRAND_ID(cpuid);
526 			bp->cores[core_id].core_hotadded &= ~(1 << strand_id);
527 		}
528 		return (DDI_WALK_CONTINUE);
529 	}
530 
531 	return (DDI_WALK_PRUNECHILD);
532 }
533 
534 
535 static int
536 drmach_add_remove_cpu(int bnum, int core_id, int option)
537 {
538 	struct drmach_hotcpu arg;
539 	drmach_board_t *bp;
540 
541 	bp = drmach_get_board_by_bnum(bnum);
542 	ASSERT(bp);
543 
544 	arg.bp = bp;
545 	arg.bnum = bnum;
546 	arg.core_id = core_id;
547 	arg.rv = 0;
548 	arg.option = option;
549 	ddi_walk_devs(ddi_root_node(), drmach_cpu_cb, (void *)&arg);
550 	return (arg.rv);
551 }
552 
553 struct drmach_setup_core_arg {
554 	drmach_board_t *bp;
555 };
556 
557 static int
558 drmach_setup_core_cb(dev_info_t *dip, void *arg)
559 {
560 	struct drmach_setup_core_arg *p = (struct drmach_setup_core_arg *)arg;
561 	char name[OBP_MAXDRVNAME];
562 	int len = OBP_MAXDRVNAME;
563 	int bnum;
564 	int core_id, strand_id;
565 
566 	if (dip == ddi_root_node()) {
567 		return (DDI_WALK_CONTINUE);
568 	}
569 
570 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
571 	    DDI_PROP_DONTPASS, "name",
572 	    (caddr_t)name, &len) != DDI_PROP_SUCCESS) {
573 		return (DDI_WALK_PRUNECHILD);
574 	}
575 
576 	/* only cmp has board number */
577 	bnum = -1;
578 	len = sizeof (bnum);
579 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
580 	    DDI_PROP_DONTPASS, OBP_BOARDNUM,
581 	    (caddr_t)&bnum, &len) != DDI_PROP_SUCCESS) {
582 		bnum = -1;
583 	}
584 
585 	if (strcmp(name, "cmp") == 0) {
586 		if (bnum != p->bp->bnum)
587 			return (DDI_WALK_PRUNECHILD);
588 		return (DDI_WALK_CONTINUE);
589 	}
590 	/* we have already pruned all unwanted cores and cpu's above */
591 	if (strcmp(name, "core") == 0) {
592 		return (DDI_WALK_CONTINUE);
593 	}
594 	if (strcmp(name, "cpu") == 0) {
595 		processorid_t cpuid;
596 		len = sizeof (cpuid);
597 		if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
598 		    DDI_PROP_DONTPASS, "cpuid",
599 		    (caddr_t)&cpuid, &len) != DDI_PROP_SUCCESS) {
600 			return (DDI_WALK_TERMINATE);
601 		}
602 		bnum = LSB_ID(cpuid);
603 		ASSERT(bnum == p->bp->bnum);
604 		core_id = ON_BOARD_CORE_NUM(cpuid);
605 		strand_id = STRAND_ID(cpuid);
606 		p->bp->cores[core_id].core_present |= (1 << strand_id);
607 		return (DDI_WALK_CONTINUE);
608 	}
609 
610 	return (DDI_WALK_PRUNECHILD);
611 }
612 
613 
614 static void
615 drmach_setup_core_info(drmach_board_t *obj)
616 {
617 	struct drmach_setup_core_arg arg;
618 	int i;
619 
620 	for (i = 0; i < OPL_MAX_COREID_PER_BOARD; i++) {
621 		obj->cores[i].core_present = 0;
622 		obj->cores[i].core_hotadded = 0;
623 		obj->cores[i].core_started = 0;
624 	}
625 	arg.bp = obj;
626 	ddi_walk_devs(ddi_root_node(), drmach_setup_core_cb, (void *)&arg);
627 
628 	for (i = 0; i < OPL_MAX_COREID_PER_BOARD; i++) {
629 		if (obj->boot_board) {
630 			obj->cores[i].core_hotadded =
631 				obj->cores[i].core_started =
632 				obj->cores[i].core_present;
633 		}
634 	}
635 }
636 
637 /*
638  * drmach_node_* routines serve the purpose of separating the
639  * rest of the code from the device tree and OBP.  This is necessary
640  * because of In-Kernel-Probing.  Devices probed after stod, are probed
641  * by the in-kernel-prober, not OBP.  These devices, therefore, do not
642  * have dnode ids.
643  */
644 
645 typedef struct {
646 	drmach_node_walk_args_t	*nwargs;
647 	int 			(*cb)(drmach_node_walk_args_t *args);
648 	int			err;
649 } drmach_node_ddi_walk_args_t;
650 
651 static int
652 drmach_node_ddi_walk_cb(dev_info_t *dip, void *arg)
653 {
654 	drmach_node_ddi_walk_args_t	*nargs;
655 
656 	nargs = (drmach_node_ddi_walk_args_t *)arg;
657 
658 	/*
659 	 * dip doesn't have to be held here as we are called
660 	 * from ddi_walk_devs() which holds the dip.
661 	 */
662 	nargs->nwargs->node->here = (void *)dip;
663 
664 	nargs->err = nargs->cb(nargs->nwargs);
665 
666 
667 	/*
668 	 * Set "here" to NULL so that unheld dip is not accessible
669 	 * outside ddi_walk_devs()
670 	 */
671 	nargs->nwargs->node->here = NULL;
672 
673 	if (nargs->err)
674 		return (DDI_WALK_TERMINATE);
675 	else
676 		return (DDI_WALK_CONTINUE);
677 }
678 
679 static int
680 drmach_node_ddi_walk(drmach_node_t *np, void *data,
681 		int (*cb)(drmach_node_walk_args_t *args))
682 {
683 	drmach_node_walk_args_t		args;
684 	drmach_node_ddi_walk_args_t	nargs;
685 
686 
687 	/* initialized args structure for callback */
688 	args.node = np;
689 	args.data = data;
690 
691 	nargs.nwargs = &args;
692 	nargs.cb = cb;
693 	nargs.err = 0;
694 
695 	/*
696 	 * Root node doesn't have to be held in any way.
697 	 */
698 	ddi_walk_devs(ddi_root_node(), drmach_node_ddi_walk_cb,
699 		(void *)&nargs);
700 
701 	return (nargs.err);
702 }
703 
704 static int
705 drmach_node_ddi_get_parent(drmach_node_t *np, drmach_node_t *pp)
706 {
707 	dev_info_t	*ndip;
708 	static char	*fn = "drmach_node_ddi_get_parent";
709 
710 	ndip = np->n_getdip(np);
711 	if (ndip == NULL) {
712 		cmn_err(CE_WARN, "%s: NULL dip", fn);
713 		return (-1);
714 	}
715 
716 	bcopy(np, pp, sizeof (drmach_node_t));
717 
718 	pp->here = (void *)ddi_get_parent(ndip);
719 	if (pp->here == NULL) {
720 		cmn_err(CE_WARN, "%s: NULL parent dip", fn);
721 		return (-1);
722 	}
723 
724 	return (0);
725 }
726 
727 /*ARGSUSED*/
728 static pnode_t
729 drmach_node_ddi_get_dnode(drmach_node_t *np)
730 {
731 	return ((pnode_t)NULL);
732 }
733 
734 static drmach_node_t *
735 drmach_node_new(void)
736 {
737 	drmach_node_t *np;
738 
739 	np = kmem_zalloc(sizeof (drmach_node_t), KM_SLEEP);
740 
741 	np->get_dnode = drmach_node_ddi_get_dnode;
742 	np->walk = drmach_node_ddi_walk;
743 	np->n_getdip = drmach_node_ddi_get_dip;
744 	np->n_getproplen = drmach_node_ddi_get_proplen;
745 	np->n_getprop = drmach_node_ddi_get_prop;
746 	np->get_parent = drmach_node_ddi_get_parent;
747 
748 	return (np);
749 }
750 
751 static void
752 drmach_node_dispose(drmach_node_t *np)
753 {
754 	kmem_free(np, sizeof (*np));
755 }
756 
757 static dev_info_t *
758 drmach_node_ddi_get_dip(drmach_node_t *np)
759 {
760 	return ((dev_info_t *)np->here);
761 }
762 
763 static int
764 drmach_node_walk(drmach_node_t *np, void *param,
765 		int (*cb)(drmach_node_walk_args_t *args))
766 {
767 	return (np->walk(np, param, cb));
768 }
769 
770 static int
771 drmach_node_ddi_get_prop(drmach_node_t *np, char *name, void *buf, int len)
772 {
773 	int		rv = 0;
774 	dev_info_t	*ndip;
775 	static char	*fn = "drmach_node_ddi_get_prop";
776 
777 
778 	ndip = np->n_getdip(np);
779 	if (ndip == NULL) {
780 		cmn_err(CE_WARN, "%s: NULL dip", fn);
781 		rv = -1;
782 	} else if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ndip,
783 	    DDI_PROP_DONTPASS, name,
784 	    (caddr_t)buf, &len) != DDI_PROP_SUCCESS) {
785 		rv = -1;
786 	}
787 
788 	return (rv);
789 }
790 
791 static int
792 drmach_node_ddi_get_proplen(drmach_node_t *np, char *name, int *len)
793 {
794 	int		rv = 0;
795 	dev_info_t	*ndip;
796 
797 	ndip = np->n_getdip(np);
798 	if (ndip == NULL) {
799 		rv = -1;
800 	} else if (ddi_getproplen(DDI_DEV_T_ANY, ndip, DDI_PROP_DONTPASS,
801 		name, len) != DDI_PROP_SUCCESS) {
802 		rv = -1;
803 	}
804 
805 	return (rv);
806 }
807 
808 static drmachid_t
809 drmach_node_dup(drmach_node_t *np)
810 {
811 	drmach_node_t *dup;
812 
813 	dup = drmach_node_new();
814 	dup->here = np->here;
815 	dup->get_dnode = np->get_dnode;
816 	dup->walk = np->walk;
817 	dup->n_getdip = np->n_getdip;
818 	dup->n_getproplen = np->n_getproplen;
819 	dup->n_getprop = np->n_getprop;
820 	dup->get_parent = np->get_parent;
821 
822 	return (dup);
823 }
824 
825 /*
826  * drmach_array provides convenient array construction, access,
827  * bounds checking and array destruction logic.
828  */
829 
830 static drmach_array_t *
831 drmach_array_new(int min_index, int max_index)
832 {
833 	drmach_array_t *arr;
834 
835 	arr = kmem_zalloc(sizeof (drmach_array_t), KM_SLEEP);
836 
837 	arr->arr_sz = (max_index - min_index + 1) * sizeof (void *);
838 	if (arr->arr_sz > 0) {
839 		arr->min_index = min_index;
840 		arr->max_index = max_index;
841 
842 		arr->arr = kmem_zalloc(arr->arr_sz, KM_SLEEP);
843 		return (arr);
844 	} else {
845 		kmem_free(arr, sizeof (*arr));
846 		return (0);
847 	}
848 }
849 
850 static int
851 drmach_array_set(drmach_array_t *arr, int idx, drmachid_t val)
852 {
853 	if (idx < arr->min_index || idx > arr->max_index)
854 		return (-1);
855 	else {
856 		arr->arr[idx - arr->min_index] = val;
857 		return (0);
858 	}
859 	/*NOTREACHED*/
860 }
861 
862 static int
863 drmach_array_get(drmach_array_t *arr, int idx, drmachid_t *val)
864 {
865 	if (idx < arr->min_index || idx > arr->max_index)
866 		return (-1);
867 	else {
868 		*val = arr->arr[idx - arr->min_index];
869 		return (0);
870 	}
871 	/*NOTREACHED*/
872 }
873 
874 static int
875 drmach_array_first(drmach_array_t *arr, int *idx, drmachid_t *val)
876 {
877 	int rv;
878 
879 	*idx = arr->min_index;
880 	while ((rv = drmach_array_get(arr, *idx, val)) == 0 && *val == NULL)
881 		*idx += 1;
882 
883 	return (rv);
884 }
885 
886 static int
887 drmach_array_next(drmach_array_t *arr, int *idx, drmachid_t *val)
888 {
889 	int rv;
890 
891 	*idx += 1;
892 	while ((rv = drmach_array_get(arr, *idx, val)) == 0 && *val == NULL)
893 		*idx += 1;
894 
895 	return (rv);
896 }
897 
898 static void
899 drmach_array_dispose(drmach_array_t *arr, void (*disposer)(drmachid_t))
900 {
901 	drmachid_t	val;
902 	int		idx;
903 	int		rv;
904 
905 	rv = drmach_array_first(arr, &idx, &val);
906 	while (rv == 0) {
907 		(*disposer)(val);
908 		rv = drmach_array_next(arr, &idx, &val);
909 	}
910 
911 	kmem_free(arr->arr, arr->arr_sz);
912 	kmem_free(arr, sizeof (*arr));
913 }
914 
915 static drmach_board_t *
916 drmach_get_board_by_bnum(int bnum)
917 {
918 	drmachid_t id;
919 
920 	if (drmach_array_get(drmach_boards, bnum, &id) == 0)
921 		return ((drmach_board_t *)id);
922 	else
923 		return (NULL);
924 }
925 
926 static pnode_t
927 drmach_node_get_dnode(drmach_node_t *np)
928 {
929 	return (np->get_dnode(np));
930 }
931 
932 /*ARGSUSED*/
933 sbd_error_t *
934 drmach_configure(drmachid_t id, int flags)
935 {
936 	drmach_device_t		*dp;
937 	sbd_error_t		*err = NULL;
938 	dev_info_t		*rdip;
939 	dev_info_t		*fdip = NULL;
940 
941 	if (DRMACH_IS_CPU_ID(id)) {
942 		return (NULL);
943 	}
944 	if (!DRMACH_IS_DEVICE_ID(id))
945 		return (drerr_new(0, EOPL_INAPPROP, NULL));
946 	dp = id;
947 	rdip = dp->node->n_getdip(dp->node);
948 
949 	ASSERT(rdip);
950 
951 	ASSERT(e_ddi_branch_held(rdip));
952 
953 	if (e_ddi_branch_configure(rdip, &fdip, 0) != 0) {
954 		char *path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
955 		dev_info_t *dip = (fdip != NULL) ? fdip : rdip;
956 
957 		(void) ddi_pathname(dip, path);
958 		err = drerr_new(1,  EOPL_DRVFAIL, path);
959 
960 		kmem_free(path, MAXPATHLEN);
961 
962 		/* If non-NULL, fdip is returned held and must be released */
963 		if (fdip != NULL)
964 			ddi_release_devi(fdip);
965 	}
966 
967 	return (err);
968 }
969 
970 
971 static sbd_error_t *
972 drmach_device_new(drmach_node_t *node,
973 	drmach_board_t *bp, int portid, drmachid_t *idp)
974 {
975 	int		 i;
976 	int		 rv;
977 	drmach_device_t	proto;
978 	sbd_error_t	*err;
979 	char		 name[OBP_MAXDRVNAME];
980 
981 	rv = node->n_getprop(node, "name", name, OBP_MAXDRVNAME);
982 	if (rv) {
983 		/* every node is expected to have a name */
984 		err = drerr_new(1, EOPL_GETPROP,
985 			"device node %s: property %s",
986 			ddi_node_name(node->n_getdip(node)), "name");
987 		return (err);
988 	}
989 
990 	/*
991 	 * The node currently being examined is not listed in the name2type[]
992 	 * array.  In this case, the node is no interest to drmach.  Both
993 	 * dp and err are initialized here to yield nothing (no device or
994 	 * error structure) for this case.
995 	 */
996 	i = drmach_name2type_idx(name);
997 
998 
999 	if (i < 0) {
1000 		*idp = (drmachid_t)0;
1001 		return (NULL);
1002 	}
1003 
1004 	/* device specific new function will set unum */
1005 
1006 	bzero(&proto, sizeof (proto));
1007 	proto.type = drmach_name2type[i].type;
1008 	proto.bp = bp;
1009 	proto.node = node;
1010 	proto.portid = portid;
1011 
1012 	return (drmach_name2type[i].new(&proto, idp));
1013 }
1014 
1015 static void
1016 drmach_device_dispose(drmachid_t id)
1017 {
1018 	drmach_device_t *self = id;
1019 
1020 	self->cm.dispose(id);
1021 }
1022 
1023 
1024 static drmach_board_t *
1025 drmach_board_new(int bnum, int boot_board)
1026 {
1027 	static sbd_error_t *drmach_board_release(drmachid_t);
1028 	static sbd_error_t *drmach_board_status(drmachid_t, drmach_status_t *);
1029 
1030 	drmach_board_t	*bp;
1031 
1032 	bp = kmem_zalloc(sizeof (drmach_board_t), KM_SLEEP);
1033 
1034 	bp->cm.isa = (void *)drmach_board_new;
1035 	bp->cm.release = drmach_board_release;
1036 	bp->cm.status = drmach_board_status;
1037 
1038 	(void) drmach_board_name(bnum, bp->cm.name, sizeof (bp->cm.name));
1039 
1040 	bp->bnum = bnum;
1041 	bp->devices = NULL;
1042 	bp->connected = boot_board;
1043 	bp->tree = drmach_node_new();
1044 	bp->assigned = boot_board;
1045 	bp->powered = boot_board;
1046 	bp->boot_board = boot_board;
1047 
1048 	/*
1049 	 * If this is not bootup initialization, we have to wait till
1050 	 * IKP sets up the device nodes in drmach_board_connect().
1051 	 */
1052 	if (boot_board)
1053 		drmach_setup_core_info(bp);
1054 
1055 	drmach_array_set(drmach_boards, bnum, bp);
1056 	return (bp);
1057 }
1058 
1059 static void
1060 drmach_board_dispose(drmachid_t id)
1061 {
1062 	drmach_board_t *bp;
1063 
1064 	ASSERT(DRMACH_IS_BOARD_ID(id));
1065 	bp = id;
1066 
1067 	if (bp->tree)
1068 		drmach_node_dispose(bp->tree);
1069 
1070 	if (bp->devices)
1071 		drmach_array_dispose(bp->devices, drmach_device_dispose);
1072 
1073 	kmem_free(bp, sizeof (*bp));
1074 }
1075 
1076 static sbd_error_t *
1077 drmach_board_status(drmachid_t id, drmach_status_t *stat)
1078 {
1079 	sbd_error_t	*err = NULL;
1080 	drmach_board_t	*bp;
1081 
1082 	if (!DRMACH_IS_BOARD_ID(id))
1083 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1084 	bp = id;
1085 
1086 	stat->assigned = bp->assigned;
1087 	stat->powered = bp->powered;
1088 	stat->busy = 0;			/* assume not busy */
1089 	stat->configured = 0;		/* assume not configured */
1090 	stat->empty = 0;
1091 	stat->cond = bp->cond = SBD_COND_OK;
1092 	strncpy(stat->type, "System Brd", sizeof (stat->type));
1093 	stat->info[0] = '\0';
1094 
1095 	if (bp->devices) {
1096 		int		 rv;
1097 		int		 d_idx;
1098 		drmachid_t	 d_id;
1099 
1100 		rv = drmach_array_first(bp->devices, &d_idx, &d_id);
1101 		while (rv == 0) {
1102 			drmach_status_t	d_stat;
1103 
1104 			err = drmach_i_status(d_id, &d_stat);
1105 			if (err)
1106 				break;
1107 
1108 			stat->busy |= d_stat.busy;
1109 			stat->configured |= d_stat.configured;
1110 
1111 			rv = drmach_array_next(bp->devices, &d_idx, &d_id);
1112 		}
1113 	}
1114 
1115 	return (err);
1116 }
1117 
1118 int
1119 drmach_board_is_floating(drmachid_t id)
1120 {
1121 	drmach_board_t *bp;
1122 
1123 	if (!DRMACH_IS_BOARD_ID(id))
1124 		return (0);
1125 
1126 	bp = (drmach_board_t *)id;
1127 
1128 	return ((drmach_domain.floating & (1 << bp->bnum)) ? 1 : 0);
1129 }
1130 
1131 static int
1132 drmach_init(void)
1133 {
1134 	dev_info_t	*rdip;
1135 	int		i, rv, len;
1136 	int		*floating;
1137 
1138 	rw_init(&drmach_boards_rwlock, NULL, RW_DEFAULT, NULL);
1139 
1140 	drmach_boards = drmach_array_new(0, MAX_BOARDS - 1);
1141 
1142 	rdip = ddi_root_node();
1143 
1144 	if (ddi_getproplen(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS,
1145 		"floating-boards", &len) != DDI_PROP_SUCCESS) {
1146 		cmn_err(CE_WARN, "Cannot get floating-boards proplen\n");
1147 	} else {
1148 		floating = (int *)kmem_alloc(len, KM_SLEEP);
1149 		rv = ddi_prop_op(DDI_DEV_T_ANY, rdip,
1150 			PROP_LEN_AND_VAL_BUF, DDI_PROP_DONTPASS,
1151 			"floating-boards", (caddr_t)floating, &len);
1152 		if (rv != DDI_PROP_SUCCESS) {
1153 			cmn_err(CE_WARN, "Cannot get floating-boards prop\n");
1154 		} else {
1155 			drmach_domain.floating = 0;
1156 			for (i = 0; i < len / sizeof (int); i++) {
1157 				drmach_domain.floating |= (1 << floating[i]);
1158 			}
1159 		}
1160 		kmem_free(floating, len);
1161 	}
1162 	drmach_domain.allow_dr = opl_check_dr_status();
1163 
1164 	rdip = ddi_get_child(ddi_root_node());
1165 	do {
1166 		int		 bnum;
1167 		drmachid_t	 id;
1168 
1169 		bnum = -1;
1170 		bnum = ddi_getprop(DDI_DEV_T_ANY, rdip,
1171 			DDI_PROP_DONTPASS, OBP_BOARDNUM, -1);
1172 		if (bnum == -1)
1173 			continue;
1174 
1175 		if (drmach_array_get(drmach_boards, bnum, &id) == -1) {
1176 			cmn_err(CE_WARN, "Device node 0x%p has"
1177 				" invalid property value, %s=%d",
1178 					rdip, OBP_BOARDNUM, bnum);
1179 			goto error;
1180 		} else if (id == NULL) {
1181 			(void) drmach_board_new(bnum, 1);
1182 		}
1183 	} while ((rdip = ddi_get_next_sibling(rdip)) != NULL);
1184 
1185 	opl_hold_devtree();
1186 
1187 	/*
1188 	 * Initialize the IKP feature.
1189 	 *
1190 	 * This can be done only after DR has acquired a hold on all the
1191 	 * device nodes that are interesting to IKP.
1192 	 */
1193 	if (opl_init_cfg() != 0) {
1194 		cmn_err(CE_WARN, "DR - IKP initialization failed");
1195 
1196 		opl_release_devtree();
1197 
1198 		goto error;
1199 	}
1200 
1201 	return (0);
1202 error:
1203 	drmach_array_dispose(drmach_boards, drmach_board_dispose);
1204 	rw_destroy(&drmach_boards_rwlock);
1205 	return (ENXIO);
1206 }
1207 
1208 static void
1209 drmach_fini(void)
1210 {
1211 	rw_enter(&drmach_boards_rwlock, RW_WRITER);
1212 	drmach_array_dispose(drmach_boards, drmach_board_dispose);
1213 	drmach_boards = NULL;
1214 	rw_exit(&drmach_boards_rwlock);
1215 
1216 	/*
1217 	 * Walk immediate children of the root devinfo node
1218 	 * releasing holds acquired on branches in drmach_init()
1219 	 */
1220 
1221 	opl_release_devtree();
1222 
1223 	rw_destroy(&drmach_boards_rwlock);
1224 }
1225 
1226 /*
1227  *	Each system board contains 2 Oberon PCI bridge and
1228  *	1 CMUCH.
1229  *	Each oberon has 2 channels.
1230  *	Each channel has 2 pci-ex leaf.
1231  *	Each CMUCH has 1 pci bus.
1232  *
1233  *
1234  *	Device Path:
1235  *	/pci@<portid>,reg
1236  *
1237  *	where
1238  *	portid[10] = 0
1239  *	portid[9:0] = LLEAF_ID[9:0] of the Oberon Channel
1240  *
1241  *	LLEAF_ID[9:8] = 0
1242  *	LLEAF_ID[8:4] = LSB_ID[4:0]
1243  *	LLEAF_ID[3:1] = IO Channel#[2:0] (0,1,2,3 for Oberon)
1244  *			channel 4 is pcicmu
1245  *	LLEAF_ID[0] = PCI Leaf Number (0 for leaf-A, 1 for leaf-B)
1246  *
1247  *	Properties:
1248  *	name = pci
1249  *	device_type = "pciex"
1250  *	board# = LSBID
1251  *	reg = int32 * 2, Oberon CSR space of the leaf and the UBC space
1252  *	portid = Jupiter Bus Device ID ((LSB_ID << 3)|pciport#)
1253  */
1254 
1255 static sbd_error_t *
1256 drmach_io_new(drmach_device_t *proto, drmachid_t *idp)
1257 {
1258 	drmach_io_t	*ip;
1259 
1260 	int		 portid;
1261 
1262 	portid = proto->portid;
1263 	ASSERT(portid != -1);
1264 	proto->unum = portid & (MAX_IO_UNITS_PER_BOARD - 1);
1265 
1266 	ip = kmem_zalloc(sizeof (drmach_io_t), KM_SLEEP);
1267 	bcopy(proto, &ip->dev, sizeof (ip->dev));
1268 	ip->dev.node = drmach_node_dup(proto->node);
1269 	ip->dev.cm.isa = (void *)drmach_io_new;
1270 	ip->dev.cm.dispose = drmach_io_dispose;
1271 	ip->dev.cm.release = drmach_io_release;
1272 	ip->dev.cm.status = drmach_io_status;
1273 	ip->channel = (portid >> 1) & 0x7;
1274 	ip->leaf = (portid & 0x1);
1275 
1276 	snprintf(ip->dev.cm.name, sizeof (ip->dev.cm.name), "%s%d",
1277 		ip->dev.type, ip->dev.unum);
1278 
1279 	*idp = (drmachid_t)ip;
1280 	return (NULL);
1281 }
1282 
1283 
1284 static void
1285 drmach_io_dispose(drmachid_t id)
1286 {
1287 	drmach_io_t *self;
1288 
1289 	ASSERT(DRMACH_IS_IO_ID(id));
1290 
1291 	self = id;
1292 	if (self->dev.node)
1293 		drmach_node_dispose(self->dev.node);
1294 
1295 	kmem_free(self, sizeof (*self));
1296 }
1297 
1298 /*ARGSUSED*/
1299 sbd_error_t *
1300 drmach_pre_op(int cmd, drmachid_t id, drmach_opts_t *opts)
1301 {
1302 	drmach_board_t	*bp = (drmach_board_t *)id;
1303 	sbd_error_t	*err = NULL;
1304 
1305 	/* allow status and ncm operations to always succeed */
1306 	if ((cmd == SBD_CMD_STATUS) || (cmd == SBD_CMD_GETNCM)) {
1307 		return (NULL);
1308 	}
1309 
1310 	/* check all other commands for the required option string */
1311 
1312 	if ((opts->size > 0) && (opts->copts != NULL)) {
1313 
1314 		DRMACH_PR("platform options: %s\n", opts->copts);
1315 
1316 		if (strstr(opts->copts, "opldr") == NULL) {
1317 			err = drerr_new(1, EOPL_SUPPORT, NULL);
1318 		}
1319 	} else {
1320 		err = drerr_new(1, EOPL_SUPPORT, NULL);
1321 	}
1322 
1323 	if (!err && id && DRMACH_IS_BOARD_ID(id)) {
1324 		switch (cmd) {
1325 			case SBD_CMD_TEST:
1326 			case SBD_CMD_STATUS:
1327 			case SBD_CMD_GETNCM:
1328 				break;
1329 			case SBD_CMD_CONNECT:
1330 				if (bp->connected)
1331 					err = drerr_new(0, ESBD_STATE, NULL);
1332 				else if (!drmach_domain.allow_dr)
1333 					err = drerr_new(1, EOPL_SUPPORT,
1334 						NULL);
1335 				break;
1336 			case SBD_CMD_DISCONNECT:
1337 				if (!bp->connected)
1338 					err = drerr_new(0, ESBD_STATE, NULL);
1339 				else if (!drmach_domain.allow_dr)
1340 					err = drerr_new(1, EOPL_SUPPORT,
1341 						NULL);
1342 				break;
1343 			default:
1344 				if (!drmach_domain.allow_dr)
1345 					err = drerr_new(1, EOPL_SUPPORT,
1346 						NULL);
1347 				break;
1348 
1349 		}
1350 	}
1351 
1352 	return (err);
1353 }
1354 
1355 /*ARGSUSED*/
1356 sbd_error_t *
1357 drmach_post_op(int cmd, drmachid_t id, drmach_opts_t *opts)
1358 {
1359 	return (NULL);
1360 }
1361 
1362 sbd_error_t *
1363 drmach_board_assign(int bnum, drmachid_t *id)
1364 {
1365 	sbd_error_t	*err = NULL;
1366 
1367 	rw_enter(&drmach_boards_rwlock, RW_WRITER);
1368 
1369 	if (drmach_array_get(drmach_boards, bnum, id) == -1) {
1370 		err = drerr_new(1, EOPL_BNUM, "%d", bnum);
1371 	} else {
1372 		drmach_board_t	*bp;
1373 
1374 		if (*id)
1375 			rw_downgrade(&drmach_boards_rwlock);
1376 
1377 		bp = *id;
1378 		if (!(*id))
1379 			bp = *id  =
1380 				(drmachid_t)drmach_board_new(bnum, 0);
1381 		bp->assigned = 1;
1382 	}
1383 
1384 	rw_exit(&drmach_boards_rwlock);
1385 
1386 	return (err);
1387 }
1388 
1389 /*ARGSUSED*/
1390 sbd_error_t *
1391 drmach_board_connect(drmachid_t id, drmach_opts_t *opts)
1392 {
1393 	drmach_board_t	*obj = (drmach_board_t *)id;
1394 
1395 	if (!DRMACH_IS_BOARD_ID(id))
1396 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1397 
1398 	if (opl_probe_sb(obj->bnum) != 0)
1399 		return (drerr_new(1, EOPL_PROBE, NULL));
1400 
1401 	(void) prom_attach_notice(obj->bnum);
1402 
1403 	drmach_setup_core_info(obj);
1404 
1405 	obj->connected = 1;
1406 
1407 	return (NULL);
1408 }
1409 
1410 static int drmach_cache_flush_flag[NCPU];
1411 
1412 /*ARGSUSED*/
1413 static void
1414 drmach_flush_cache(uint64_t id, uint64_t dummy)
1415 {
1416 	extern void cpu_flush_ecache(void);
1417 
1418 	cpu_flush_ecache();
1419 	drmach_cache_flush_flag[id] = 0;
1420 }
1421 
1422 static void
1423 drmach_flush_all()
1424 {
1425 	cpuset_t	xc_cpuset;
1426 	int		i;
1427 
1428 	xc_cpuset = cpu_ready_set;
1429 	for (i = 0; i < NCPU; i++) {
1430 		if (CPU_IN_SET(xc_cpuset, i)) {
1431 			drmach_cache_flush_flag[i] = 1;
1432 			xc_one(i, drmach_flush_cache, i, 0);
1433 			while (drmach_cache_flush_flag[i]) {
1434 				DELAY(1000);
1435 			}
1436 		}
1437 	}
1438 }
1439 
1440 static int
1441 drmach_disconnect_cpus(drmach_board_t *bp)
1442 {
1443 	int i, bnum;
1444 
1445 	bnum = bp->bnum;
1446 
1447 	for (i = 0; i < OPL_MAX_COREID_PER_BOARD; i++) {
1448 	    if (bp->cores[i].core_present) {
1449 		if (bp->cores[i].core_started)
1450 		    return (-1);
1451 		if (bp->cores[i].core_hotadded) {
1452 		    if (drmach_add_remove_cpu(bnum, i, HOTREMOVE_CPU)) {
1453 			cmn_err(CE_WARN,
1454 			    "Failed to remove CMP %d on board %d\n",
1455 			    i, bnum);
1456 			return (-1);
1457 		    }
1458 		}
1459 	    }
1460 	}
1461 	return (0);
1462 }
1463 
1464 /*ARGSUSED*/
1465 sbd_error_t *
1466 drmach_board_disconnect(drmachid_t id, drmach_opts_t *opts)
1467 {
1468 	drmach_board_t *obj;
1469 	int rv = 0;
1470 	sbd_error_t		*err = NULL;
1471 
1472 	if (DRMACH_NULL_ID(id))
1473 		return (NULL);
1474 
1475 	if (!DRMACH_IS_BOARD_ID(id))
1476 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1477 
1478 	obj = (drmach_board_t *)id;
1479 
1480 	if (drmach_disconnect_cpus(obj)) {
1481 		err = drerr_new(1, EOPL_DEPROBE, obj->cm.name);
1482 		return (err);
1483 	}
1484 
1485 	rv = opl_unprobe_sb(obj->bnum);
1486 
1487 	if (rv == 0) {
1488 		prom_detach_notice(obj->bnum);
1489 		obj->connected = 0;
1490 
1491 	} else
1492 		err = drerr_new(1, EOPL_DEPROBE, obj->cm.name);
1493 
1494 	return (err);
1495 }
1496 
1497 static int
1498 drmach_get_portid(drmach_node_t *np)
1499 {
1500 	int		portid;
1501 	char		type[OBP_MAXPROPNAME];
1502 
1503 	if (np->n_getprop(np, "portid", &portid, sizeof (portid)) == 0)
1504 		return (portid);
1505 
1506 	/*
1507 	 * Get the device_type property to see if we should
1508 	 * continue processing this node.
1509 	 */
1510 	if (np->n_getprop(np, "device_type", &type, sizeof (type)) != 0)
1511 		return (-1);
1512 
1513 	if (strcmp(type, OPL_CPU_NODE) == 0) {
1514 		/*
1515 		 * We return cpuid because it has no portid
1516 		 */
1517 		if (np->n_getprop(np, "cpuid", &portid, sizeof (portid)) == 0)
1518 			return (portid);
1519 	}
1520 
1521 	return (-1);
1522 }
1523 
1524 /*
1525  * This is a helper function to determine if a given
1526  * node should be considered for a dr operation according
1527  * to predefined dr type nodes and the node's name.
1528  * Formal Parameter : The name of a device node.
1529  * Return Value: -1, name does not map to a valid dr type.
1530  *		 A value greater or equal to 0, name is a valid dr type.
1531  */
1532 static int
1533 drmach_name2type_idx(char *name)
1534 {
1535 	int 	index, ntypes;
1536 
1537 	if (name == NULL)
1538 		return (-1);
1539 
1540 	/*
1541 	 * Determine how many possible types are currently supported
1542 	 * for dr.
1543 	 */
1544 	ntypes = sizeof (drmach_name2type) / sizeof (drmach_name2type[0]);
1545 
1546 	/* Determine if the node's name correspond to a predefined type. */
1547 	for (index = 0; index < ntypes; index++) {
1548 		if (strcmp(drmach_name2type[index].name, name) == 0)
1549 			/* The node is an allowed type for dr. */
1550 			return (index);
1551 	}
1552 
1553 	/*
1554 	 * If the name of the node does not map to any of the
1555 	 * types in the array drmach_name2type then the node is not of
1556 	 * interest to dr.
1557 	 */
1558 	return (-1);
1559 }
1560 
1561 /*
1562  * there is some complication on OPL:
1563  * - pseudo-mc nodes do not have portid property
1564  * - portid[9:5] of cmp node is LSB #, portid[7:3] of pci is LSB#
1565  * - cmp has board#
1566  * - core and cpu nodes do not have portid and board# properties
1567  * starcat uses portid to derive the board# but that does not work
1568  * for us.  starfire reads board# property to filter the devices.
1569  * That does not work either.  So for these specific device,
1570  * we use specific hard coded methods to get the board# -
1571  * cpu: LSB# = CPUID[9:5]
1572  */
1573 
1574 static int
1575 drmach_board_find_devices_cb(drmach_node_walk_args_t *args)
1576 {
1577 	drmach_node_t			*node = args->node;
1578 	drmach_board_cb_data_t		*data = args->data;
1579 	drmach_board_t			*obj = data->obj;
1580 
1581 	int		rv, portid;
1582 	int		bnum;
1583 	drmachid_t	id;
1584 	drmach_device_t	*device;
1585 	char name[OBP_MAXDRVNAME];
1586 
1587 	portid = drmach_get_portid(node);
1588 	/*
1589 	 * core, cpu and pseudo-mc do not have portid
1590 	 * we use cpuid as the portid of the cpu node
1591 	 * for pseudo-mc, we do not use portid info.
1592 	 */
1593 
1594 	rv = node->n_getprop(node, "name", name, OBP_MAXDRVNAME);
1595 	if (rv)
1596 		return (0);
1597 
1598 
1599 	rv = node->n_getprop(node, OBP_BOARDNUM, &bnum, sizeof (bnum));
1600 
1601 	if (rv) {
1602 		/*
1603 		 * cpu does not have board# property.  We use
1604 		 * CPUID[9:5]
1605 		 */
1606 		if (strcmp("cpu", name) == 0) {
1607 			bnum = (portid >> 5) & 0x1f;
1608 		} else
1609 			return (0);
1610 	}
1611 
1612 
1613 	if (bnum != obj->bnum)
1614 		return (0);
1615 
1616 	if (drmach_name2type_idx(name) < 0) {
1617 		return (0);
1618 	}
1619 
1620 	/*
1621 	 * Create a device data structure from this node data.
1622 	 * The call may yield nothing if the node is not of interest
1623 	 * to drmach.
1624 	 */
1625 	data->err = drmach_device_new(node, obj, portid, &id);
1626 	if (data->err)
1627 		return (-1);
1628 	else if (!id) {
1629 		/*
1630 		 * drmach_device_new examined the node we passed in
1631 		 * and determined that it was one not of interest to
1632 		 * drmach.  So, it is skipped.
1633 		 */
1634 		return (0);
1635 	}
1636 
1637 	rv = drmach_array_set(obj->devices, data->ndevs++, id);
1638 	if (rv) {
1639 		data->err = DRMACH_INTERNAL_ERROR();
1640 		return (-1);
1641 	}
1642 	device = id;
1643 
1644 	data->err = (*data->found)(data->a, device->type, device->unum, id);
1645 	return (data->err == NULL ? 0 : -1);
1646 }
1647 
1648 sbd_error_t *
1649 drmach_board_find_devices(drmachid_t id, void *a,
1650 	sbd_error_t *(*found)(void *a, const char *, int, drmachid_t))
1651 {
1652 	drmach_board_t		*bp = (drmach_board_t *)id;
1653 	sbd_error_t		*err;
1654 	int			 max_devices;
1655 	int			 rv;
1656 	drmach_board_cb_data_t	data;
1657 
1658 
1659 	if (!DRMACH_IS_BOARD_ID(id))
1660 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1661 
1662 	max_devices  = MAX_CPU_UNITS_PER_BOARD;
1663 	max_devices += MAX_MEM_UNITS_PER_BOARD;
1664 	max_devices += MAX_IO_UNITS_PER_BOARD;
1665 
1666 	bp->devices = drmach_array_new(0, max_devices);
1667 
1668 	if (bp->tree == NULL)
1669 		bp->tree = drmach_node_new();
1670 
1671 	data.obj = bp;
1672 	data.ndevs = 0;
1673 	data.found = found;
1674 	data.a = a;
1675 	data.err = NULL;
1676 
1677 	rv = drmach_node_walk(bp->tree, &data, drmach_board_find_devices_cb);
1678 	if (rv == 0)
1679 		err = NULL;
1680 	else {
1681 		drmach_array_dispose(bp->devices, drmach_device_dispose);
1682 		bp->devices = NULL;
1683 
1684 		if (data.err)
1685 			err = data.err;
1686 		else
1687 			err = DRMACH_INTERNAL_ERROR();
1688 	}
1689 
1690 	return (err);
1691 }
1692 
1693 int
1694 drmach_board_lookup(int bnum, drmachid_t *id)
1695 {
1696 	int	rv = 0;
1697 
1698 	rw_enter(&drmach_boards_rwlock, RW_READER);
1699 	if (drmach_array_get(drmach_boards, bnum, id)) {
1700 		*id = 0;
1701 		rv = -1;
1702 	}
1703 	rw_exit(&drmach_boards_rwlock);
1704 	return (rv);
1705 }
1706 
1707 sbd_error_t *
1708 drmach_board_name(int bnum, char *buf, int buflen)
1709 {
1710 	snprintf(buf, buflen, "SB%d", bnum);
1711 	return (NULL);
1712 }
1713 
1714 sbd_error_t *
1715 drmach_board_poweroff(drmachid_t id)
1716 {
1717 	drmach_board_t	*bp;
1718 	sbd_error_t	*err;
1719 	drmach_status_t	 stat;
1720 
1721 	if (DRMACH_NULL_ID(id))
1722 		return (NULL);
1723 
1724 	if (!DRMACH_IS_BOARD_ID(id))
1725 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1726 	bp = id;
1727 
1728 	err = drmach_board_status(id, &stat);
1729 
1730 	if (!err) {
1731 		if (stat.configured || stat.busy)
1732 			err = drerr_new(0, EOPL_CONFIGBUSY, bp->cm.name);
1733 		else {
1734 			bp->powered = 0;
1735 		}
1736 	}
1737 	return (err);
1738 }
1739 
1740 sbd_error_t *
1741 drmach_board_poweron(drmachid_t id)
1742 {
1743 	drmach_board_t	*bp;
1744 
1745 	if (!DRMACH_IS_BOARD_ID(id))
1746 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1747 	bp = id;
1748 
1749 	bp->powered = 1;
1750 
1751 	return (NULL);
1752 }
1753 
1754 static sbd_error_t *
1755 drmach_board_release(drmachid_t id)
1756 {
1757 	if (!DRMACH_IS_BOARD_ID(id))
1758 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1759 	return (NULL);
1760 }
1761 
1762 /*ARGSUSED*/
1763 sbd_error_t *
1764 drmach_board_test(drmachid_t id, drmach_opts_t *opts, int force)
1765 {
1766 	return (NULL);
1767 }
1768 
1769 sbd_error_t *
1770 drmach_board_unassign(drmachid_t id)
1771 {
1772 	drmach_board_t	*bp;
1773 	sbd_error_t	*err;
1774 	drmach_status_t	 stat;
1775 
1776 	if (DRMACH_NULL_ID(id))
1777 		return (NULL);
1778 
1779 	if (!DRMACH_IS_BOARD_ID(id)) {
1780 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1781 	}
1782 	bp = id;
1783 
1784 	rw_enter(&drmach_boards_rwlock, RW_WRITER);
1785 
1786 	err = drmach_board_status(id, &stat);
1787 	if (err) {
1788 		rw_exit(&drmach_boards_rwlock);
1789 		return (err);
1790 	}
1791 	if (stat.configured || stat.busy) {
1792 		err = drerr_new(0, EOPL_CONFIGBUSY, bp->cm.name);
1793 	} else {
1794 		if (drmach_array_set(drmach_boards, bp->bnum, 0) != 0)
1795 			err = DRMACH_INTERNAL_ERROR();
1796 		else
1797 			drmach_board_dispose(bp);
1798 	}
1799 	rw_exit(&drmach_boards_rwlock);
1800 	return (err);
1801 }
1802 
1803 /*
1804  * We have to do more on OPL - e.g. set up sram tte, read cpuid, strand id,
1805  * implementation #, etc
1806  */
1807 
1808 static sbd_error_t *
1809 drmach_cpu_new(drmach_device_t *proto, drmachid_t *idp)
1810 {
1811 	static void drmach_cpu_dispose(drmachid_t);
1812 	static sbd_error_t *drmach_cpu_release(drmachid_t);
1813 	static sbd_error_t *drmach_cpu_status(drmachid_t, drmach_status_t *);
1814 
1815 	int		 portid;
1816 	drmach_cpu_t	*cp = NULL;
1817 
1818 	/* portid is CPUID of the node */
1819 	portid = proto->portid;
1820 	ASSERT(portid != -1);
1821 
1822 	/* unum = (CMP/CHIP ID) + (ON_BOARD_CORE_NUM * MAX_CMPID_PER_BOARD) */
1823 	proto->unum = ((portid/OPL_MAX_CPUID_PER_CMP) &
1824 		(OPL_MAX_CMPID_PER_BOARD - 1)) +
1825 		((portid & (OPL_MAX_CPUID_PER_CMP - 1)) *
1826 		(OPL_MAX_CMPID_PER_BOARD));
1827 
1828 	cp = kmem_zalloc(sizeof (drmach_cpu_t), KM_SLEEP);
1829 	bcopy(proto, &cp->dev, sizeof (cp->dev));
1830 	cp->dev.node = drmach_node_dup(proto->node);
1831 	cp->dev.cm.isa = (void *)drmach_cpu_new;
1832 	cp->dev.cm.dispose = drmach_cpu_dispose;
1833 	cp->dev.cm.release = drmach_cpu_release;
1834 	cp->dev.cm.status = drmach_cpu_status;
1835 
1836 	snprintf(cp->dev.cm.name, sizeof (cp->dev.cm.name), "%s%d",
1837 		cp->dev.type, cp->dev.unum);
1838 
1839 /*
1840  *	CPU ID representation
1841  *	CPUID[9:5] = SB#
1842  *	CPUID[4:3] = Chip#
1843  *	CPUID[2:1] = Core# (Only 2 core for OPL)
1844  *	CPUID[0:0] = Strand#
1845  */
1846 
1847 /*
1848  *	reg property of the strand contains strand ID
1849  *	reg property of the parent node contains core ID
1850  *	We should use them.
1851  */
1852 	cp->cpuid = portid;
1853 	cp->sb = (portid >> 5) & 0x1f;
1854 	cp->chipid = (portid >> 3) & 0x3;
1855 	cp->coreid = (portid >> 1) & 0x3;
1856 	cp->strandid = portid & 0x1;
1857 
1858 	*idp = (drmachid_t)cp;
1859 	return (NULL);
1860 }
1861 
1862 
1863 static void
1864 drmach_cpu_dispose(drmachid_t id)
1865 {
1866 	drmach_cpu_t	*self;
1867 
1868 	ASSERT(DRMACH_IS_CPU_ID(id));
1869 
1870 	self = id;
1871 	if (self->dev.node)
1872 		drmach_node_dispose(self->dev.node);
1873 
1874 	kmem_free(self, sizeof (*self));
1875 }
1876 
1877 static int
1878 drmach_cpu_start(struct cpu *cp)
1879 {
1880 	int		cpuid = cp->cpu_id;
1881 	extern int	restart_other_cpu(int);
1882 
1883 	ASSERT(MUTEX_HELD(&cpu_lock));
1884 	ASSERT(cpunodes[cpuid].nodeid != (pnode_t)0);
1885 
1886 	cp->cpu_flags &= ~CPU_POWEROFF;
1887 
1888 	/*
1889 	 * NOTE: restart_other_cpu pauses cpus during the
1890 	 *	 slave cpu start.  This helps to quiesce the
1891 	 *	 bus traffic a bit which makes the tick sync
1892 	 *	 routine in the prom more robust.
1893 	 */
1894 	DRMACH_PR("COLD START for cpu (%d)\n", cpuid);
1895 
1896 	restart_other_cpu(cpuid);
1897 
1898 	return (0);
1899 }
1900 
1901 static sbd_error_t *
1902 drmach_cpu_release(drmachid_t id)
1903 {
1904 	if (!DRMACH_IS_CPU_ID(id))
1905 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1906 
1907 	return (NULL);
1908 }
1909 
1910 static sbd_error_t *
1911 drmach_cpu_status(drmachid_t id, drmach_status_t *stat)
1912 {
1913 	drmach_cpu_t *cp;
1914 	drmach_device_t *dp;
1915 
1916 	ASSERT(DRMACH_IS_CPU_ID(id));
1917 	cp = (drmach_cpu_t *)id;
1918 	dp = &cp->dev;
1919 
1920 	stat->assigned = dp->bp->assigned;
1921 	stat->powered = dp->bp->powered;
1922 	mutex_enter(&cpu_lock);
1923 	stat->configured = (cpu_get(cp->cpuid) != NULL);
1924 	mutex_exit(&cpu_lock);
1925 	stat->busy = dp->busy;
1926 	strncpy(stat->type, dp->type, sizeof (stat->type));
1927 	stat->info[0] = '\0';
1928 
1929 	return (NULL);
1930 }
1931 
1932 sbd_error_t *
1933 drmach_cpu_disconnect(drmachid_t id)
1934 {
1935 
1936 	if (!DRMACH_IS_CPU_ID(id))
1937 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1938 
1939 	return (NULL);
1940 }
1941 
1942 sbd_error_t *
1943 drmach_cpu_get_id(drmachid_t id, processorid_t *cpuid)
1944 {
1945 	drmach_cpu_t *cpu;
1946 
1947 	if (!DRMACH_IS_CPU_ID(id))
1948 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1949 	cpu = (drmach_cpu_t *)id;
1950 
1951 	/* get from cpu directly on OPL */
1952 	*cpuid = cpu->cpuid;
1953 	return (NULL);
1954 }
1955 
1956 sbd_error_t *
1957 drmach_cpu_get_impl(drmachid_t id, int *ip)
1958 {
1959 	drmach_device_t *cpu;
1960 	drmach_node_t	*np;
1961 	drmach_node_t	pp;
1962 	int		impl;
1963 	char		type[OBP_MAXPROPNAME];
1964 
1965 	if (!DRMACH_IS_CPU_ID(id))
1966 		return (drerr_new(0, EOPL_INAPPROP, NULL));
1967 
1968 	cpu = id;
1969 	np = cpu->node;
1970 
1971 	if (np->get_parent(np, &pp) != 0) {
1972 		return (DRMACH_INTERNAL_ERROR());
1973 	}
1974 
1975 	/* the parent should be core */
1976 
1977 	if (pp.n_getprop(&pp, "device_type", &type, sizeof (type)) != 0) {
1978 		return (drerr_new(0, EOPL_GETPROP, NULL));
1979 	}
1980 
1981 	if (strcmp(type, OPL_CORE_NODE) == 0) {
1982 		if (pp.n_getprop(&pp, "implementation#",
1983 			&impl, sizeof (impl)) != 0) {
1984 			return (drerr_new(0, EOPL_GETPROP, NULL));
1985 		}
1986 	} else {
1987 		return (DRMACH_INTERNAL_ERROR());
1988 	}
1989 
1990 	*ip = impl;
1991 
1992 	return (NULL);
1993 }
1994 
1995 sbd_error_t *
1996 drmach_get_dip(drmachid_t id, dev_info_t **dip)
1997 {
1998 	drmach_device_t	*dp;
1999 
2000 	if (!DRMACH_IS_DEVICE_ID(id))
2001 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2002 	dp = id;
2003 
2004 	*dip = dp->node->n_getdip(dp->node);
2005 	return (NULL);
2006 }
2007 
2008 sbd_error_t *
2009 drmach_io_is_attached(drmachid_t id, int *yes)
2010 {
2011 	drmach_device_t *dp;
2012 	dev_info_t	*dip;
2013 	int		state;
2014 
2015 	if (!DRMACH_IS_IO_ID(id))
2016 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2017 	dp = id;
2018 
2019 	dip = dp->node->n_getdip(dp->node);
2020 	if (dip == NULL) {
2021 		*yes = 0;
2022 		return (NULL);
2023 	}
2024 
2025 	state = ddi_get_devstate(dip);
2026 	*yes = ((i_ddi_node_state(dip) >= DS_ATTACHED) ||
2027 	    (state == DDI_DEVSTATE_UP));
2028 
2029 	return (NULL);
2030 }
2031 
2032 struct drmach_io_cb {
2033 	char	*name;	/* name of the node */
2034 	int	(*func)(dev_info_t *);
2035 	int	rv;
2036 	dev_info_t *dip;
2037 };
2038 
2039 #define	DRMACH_IO_POST_ATTACH	0
2040 #define	DRMACH_IO_PRE_RELEASE	1
2041 
2042 static int
2043 drmach_io_cb_check(dev_info_t *dip, void *arg)
2044 {
2045 	struct drmach_io_cb *p = (struct drmach_io_cb *)arg;
2046 	char name[OBP_MAXDRVNAME];
2047 	int len = OBP_MAXDRVNAME;
2048 
2049 	if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
2050 		DDI_PROP_DONTPASS, "name",
2051 	    (caddr_t)name, &len) != DDI_PROP_SUCCESS) {
2052 		return (DDI_WALK_PRUNECHILD);
2053 	}
2054 
2055 	if (strcmp(name, p->name) == 0) {
2056 		ndi_hold_devi(dip);
2057 		p->dip = dip;
2058 		return (DDI_WALK_TERMINATE);
2059 	}
2060 
2061 	return (DDI_WALK_CONTINUE);
2062 }
2063 
2064 
2065 static int
2066 drmach_console_ops(drmachid_t *id, int state)
2067 {
2068 	drmach_io_t *obj = (drmach_io_t *)id;
2069 	struct drmach_io_cb arg;
2070 	int (*msudetp)(dev_info_t *);
2071 	int (*msuattp)(dev_info_t *);
2072 	dev_info_t *dip, *pdip;
2073 	int circ;
2074 
2075 	/* 4 is pcicmu channel */
2076 	if (obj->channel != 4)
2077 		return (0);
2078 
2079 	arg.name = "serial";
2080 	arg.func = NULL;
2081 	if (state == DRMACH_IO_PRE_RELEASE) {
2082 		msudetp = (int (*)(dev_info_t *))
2083 		    modgetsymvalue("oplmsu_dr_detach", 0);
2084 		if (msudetp != NULL)
2085 			arg.func = msudetp;
2086 	} else if (state == DRMACH_IO_POST_ATTACH) {
2087 		msuattp = (int (*)(dev_info_t *))
2088 		    modgetsymvalue("oplmsu_dr_attach", 0);
2089 		if (msuattp != NULL)
2090 			arg.func = msuattp;
2091 	} else {
2092 		return (0);
2093 	}
2094 
2095 	if (arg.func == NULL) {
2096 		return (0);
2097 	}
2098 
2099 	arg.rv = 0;
2100 	arg.dip = NULL;
2101 
2102 	dip = obj->dev.node->n_getdip(obj->dev.node);
2103 	if (pdip = ddi_get_parent(dip)) {
2104 		ndi_hold_devi(pdip);
2105 		ndi_devi_enter(pdip, &circ);
2106 	} else {
2107 		/* this cannot happen unless something bad happens */
2108 		return (-1);
2109 	}
2110 
2111 	ddi_walk_devs(dip, drmach_io_cb_check, (void *)&arg);
2112 
2113 	ndi_devi_exit(pdip, circ);
2114 	ndi_rele_devi(pdip);
2115 
2116 	if (arg.dip) {
2117 		arg.rv = (*arg.func)(arg.dip);
2118 		ndi_rele_devi(arg.dip);
2119 	} else {
2120 		arg.rv = -1;
2121 	}
2122 
2123 	return (arg.rv);
2124 }
2125 
2126 sbd_error_t *
2127 drmach_io_pre_release(drmachid_t id)
2128 {
2129 	int rv;
2130 
2131 	if (!DRMACH_IS_IO_ID(id))
2132 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2133 
2134 	rv = drmach_console_ops(id, DRMACH_IO_PRE_RELEASE);
2135 
2136 	if (rv != 0)
2137 		cmn_err(CE_WARN, "IO callback failed in pre-release\n");
2138 
2139 	return (NULL);
2140 }
2141 
2142 static sbd_error_t *
2143 drmach_io_release(drmachid_t id)
2144 {
2145 	if (!DRMACH_IS_IO_ID(id))
2146 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2147 	return (NULL);
2148 }
2149 
2150 sbd_error_t *
2151 drmach_io_unrelease(drmachid_t id)
2152 {
2153 	if (!DRMACH_IS_IO_ID(id))
2154 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2155 	return (NULL);
2156 }
2157 
2158 /*ARGSUSED*/
2159 sbd_error_t *
2160 drmach_io_post_release(drmachid_t id)
2161 {
2162 	return (NULL);
2163 }
2164 
2165 /*ARGSUSED*/
2166 sbd_error_t *
2167 drmach_io_post_attach(drmachid_t id)
2168 {
2169 	int rv;
2170 
2171 	if (!DRMACH_IS_IO_ID(id))
2172 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2173 
2174 	rv = drmach_console_ops(id, DRMACH_IO_POST_ATTACH);
2175 
2176 	if (rv != 0)
2177 		cmn_err(CE_WARN, "IO callback failed in post-attach\n");
2178 
2179 	return (0);
2180 }
2181 
2182 static sbd_error_t *
2183 drmach_io_status(drmachid_t id, drmach_status_t *stat)
2184 {
2185 	drmach_device_t *dp;
2186 	sbd_error_t	*err;
2187 	int		 configured;
2188 
2189 	ASSERT(DRMACH_IS_IO_ID(id));
2190 	dp = id;
2191 
2192 	err = drmach_io_is_attached(id, &configured);
2193 	if (err)
2194 		return (err);
2195 
2196 	stat->assigned = dp->bp->assigned;
2197 	stat->powered = dp->bp->powered;
2198 	stat->configured = (configured != 0);
2199 	stat->busy = dp->busy;
2200 	strncpy(stat->type, dp->type, sizeof (stat->type));
2201 	stat->info[0] = '\0';
2202 
2203 	return (NULL);
2204 }
2205 
2206 static sbd_error_t *
2207 drmach_mem_new(drmach_device_t *proto, drmachid_t *idp)
2208 {
2209 	static void drmach_mem_dispose(drmachid_t);
2210 	static sbd_error_t *drmach_mem_release(drmachid_t);
2211 	static sbd_error_t *drmach_mem_status(drmachid_t, drmach_status_t *);
2212 	dev_info_t *dip;
2213 	int rv;
2214 
2215 	drmach_mem_t	*mp;
2216 
2217 	rv = 0;
2218 
2219 	if ((proto->node->n_getproplen(proto->node, "mc-addr", &rv) < 0) ||
2220 		(rv <= 0)) {
2221 		*idp = (drmachid_t)0;
2222 		return (NULL);
2223 	}
2224 
2225 	mp = kmem_zalloc(sizeof (drmach_mem_t), KM_SLEEP);
2226 	proto->unum = 0;
2227 
2228 	bcopy(proto, &mp->dev, sizeof (mp->dev));
2229 	mp->dev.node = drmach_node_dup(proto->node);
2230 	mp->dev.cm.isa = (void *)drmach_mem_new;
2231 	mp->dev.cm.dispose = drmach_mem_dispose;
2232 	mp->dev.cm.release = drmach_mem_release;
2233 	mp->dev.cm.status = drmach_mem_status;
2234 
2235 	snprintf(mp->dev.cm.name,
2236 		sizeof (mp->dev.cm.name), "%s", mp->dev.type);
2237 
2238 	dip = mp->dev.node->n_getdip(mp->dev.node);
2239 	if (drmach_setup_mc_info(dip, mp) != 0) {
2240 		return (drerr_new(1, EOPL_MC_SETUP, NULL));
2241 	}
2242 
2243 	/* make sure we do not create memoryless nodes */
2244 	if (mp->nbytes == 0) {
2245 		*idp = (drmachid_t)NULL;
2246 		kmem_free(mp, sizeof (drmach_mem_t));
2247 	} else
2248 		*idp = (drmachid_t)mp;
2249 
2250 	return (NULL);
2251 }
2252 
2253 static void
2254 drmach_mem_dispose(drmachid_t id)
2255 {
2256 	drmach_mem_t *mp;
2257 
2258 	ASSERT(DRMACH_IS_MEM_ID(id));
2259 
2260 
2261 	mp = id;
2262 
2263 	if (mp->dev.node)
2264 		drmach_node_dispose(mp->dev.node);
2265 
2266 	if (mp->memlist) {
2267 		memlist_delete(mp->memlist);
2268 		mp->memlist = NULL;
2269 	}
2270 
2271 	kmem_free(mp, sizeof (*mp));
2272 }
2273 
2274 sbd_error_t *
2275 drmach_mem_add_span(drmachid_t id, uint64_t basepa, uint64_t size)
2276 {
2277 	pfn_t		basepfn = (pfn_t)(basepa >> PAGESHIFT);
2278 	pgcnt_t		npages = (pgcnt_t)(size >> PAGESHIFT);
2279 	int		rv;
2280 
2281 	ASSERT(size != 0);
2282 
2283 	if (!DRMACH_IS_MEM_ID(id))
2284 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2285 
2286 	kcage_range_lock();
2287 	rv = kcage_range_add(basepfn, npages, 1);
2288 	kcage_range_unlock();
2289 	if (rv == ENOMEM) {
2290 		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
2291 			(size == 0 ? 0 : size / MBYTE));
2292 	} else if (rv != 0) {
2293 		/* catch this in debug kernels */
2294 		ASSERT(0);
2295 
2296 		cmn_err(CE_WARN, "unexpected kcage_range_add"
2297 			" return value %d", rv);
2298 	}
2299 
2300 	if (rv) {
2301 		return (DRMACH_INTERNAL_ERROR());
2302 	}
2303 	else
2304 		return (NULL);
2305 }
2306 
2307 sbd_error_t *
2308 drmach_mem_del_span(drmachid_t id, uint64_t basepa, uint64_t size)
2309 {
2310 	pfn_t		basepfn = (pfn_t)(basepa >> PAGESHIFT);
2311 	pgcnt_t		npages = (pgcnt_t)(size >> PAGESHIFT);
2312 	int		rv;
2313 
2314 	if (!DRMACH_IS_MEM_ID(id))
2315 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2316 
2317 	if (size > 0) {
2318 		kcage_range_lock();
2319 		rv = kcage_range_delete_post_mem_del(basepfn, npages);
2320 		kcage_range_unlock();
2321 		if (rv != 0) {
2322 			cmn_err(CE_WARN,
2323 			    "unexpected kcage_range_delete_post_mem_del"
2324 			    " return value %d", rv);
2325 			return (DRMACH_INTERNAL_ERROR());
2326 		}
2327 	}
2328 
2329 	return (NULL);
2330 }
2331 
2332 sbd_error_t *
2333 drmach_mem_disable(drmachid_t id)
2334 {
2335 	if (!DRMACH_IS_MEM_ID(id))
2336 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2337 	else {
2338 		drmach_flush_all();
2339 		return (NULL);
2340 	}
2341 }
2342 
2343 sbd_error_t *
2344 drmach_mem_enable(drmachid_t id)
2345 {
2346 	if (!DRMACH_IS_MEM_ID(id))
2347 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2348 	else
2349 		return (NULL);
2350 }
2351 
2352 sbd_error_t *
2353 drmach_mem_get_info(drmachid_t id, drmach_mem_info_t *mem)
2354 {
2355 	drmach_mem_t *mp;
2356 
2357 	if (!DRMACH_IS_MEM_ID(id))
2358 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2359 
2360 	mp = (drmach_mem_t *)id;
2361 
2362 	/*
2363 	 * This is only used by dr to round up/down the memory
2364 	 * for copying. Our unit of memory isolation is 64 MB.
2365 	 */
2366 
2367 	mem->mi_alignment_mask = (64 * 1024 * 1024 - 1);
2368 	mem->mi_basepa = mp->base_pa;
2369 	mem->mi_size = mp->nbytes;
2370 	mem->mi_slice_size = mp->slice_size;
2371 
2372 	return (NULL);
2373 }
2374 
2375 sbd_error_t *
2376 drmach_mem_get_base_physaddr(drmachid_t id, uint64_t *pa)
2377 {
2378 	drmach_mem_t *mp;
2379 
2380 	if (!DRMACH_IS_MEM_ID(id))
2381 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2382 
2383 	mp = (drmach_mem_t *)id;
2384 
2385 	*pa = mp->base_pa;
2386 	return (NULL);
2387 }
2388 
2389 sbd_error_t *
2390 drmach_mem_get_memlist(drmachid_t id, struct memlist **ml)
2391 {
2392 	drmach_mem_t	*mem;
2393 	int		rv;
2394 	struct memlist	*mlist;
2395 
2396 	if (!DRMACH_IS_MEM_ID(id))
2397 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2398 
2399 	mem = (drmach_mem_t *)id;
2400 	mlist = memlist_dup(mem->memlist);
2401 
2402 #ifdef DEBUG
2403 	/*
2404 	 * Make sure the incoming memlist doesn't already
2405 	 * intersect with what's present in the system (phys_install).
2406 	 */
2407 	memlist_read_lock();
2408 	rv = memlist_intersect(phys_install, mlist);
2409 	memlist_read_unlock();
2410 	if (rv) {
2411 		DRMACH_PR("Derived memlist intersects"
2412 			" with phys_install\n");
2413 		memlist_dump(mlist);
2414 
2415 		DRMACH_PR("phys_install memlist:\n");
2416 		memlist_dump(phys_install);
2417 
2418 		memlist_delete(mlist);
2419 		return (DRMACH_INTERNAL_ERROR());
2420 	}
2421 
2422 	DRMACH_PR("Derived memlist:");
2423 	memlist_dump(mlist);
2424 #endif
2425 	*ml = mlist;
2426 
2427 	return (NULL);
2428 }
2429 
2430 sbd_error_t *
2431 drmach_mem_get_slice_size(drmachid_t id, uint64_t *bytes)
2432 {
2433 	drmach_mem_t	*mem;
2434 
2435 	if (!DRMACH_IS_MEM_ID(id))
2436 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2437 
2438 	mem = (drmach_mem_t *)id;
2439 
2440 	*bytes = mem->slice_size;
2441 
2442 	return (NULL);
2443 }
2444 
2445 
2446 /* ARGSUSED */
2447 processorid_t
2448 drmach_mem_cpu_affinity(drmachid_t id)
2449 {
2450 	return (CPU_CURRENT);
2451 }
2452 
2453 static sbd_error_t *
2454 drmach_mem_release(drmachid_t id)
2455 {
2456 	if (!DRMACH_IS_MEM_ID(id))
2457 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2458 	return (NULL);
2459 }
2460 
2461 static sbd_error_t *
2462 drmach_mem_status(drmachid_t id, drmach_status_t *stat)
2463 {
2464 	drmach_mem_t *dp;
2465 	uint64_t	 pa, slice_size;
2466 	struct memlist	*ml;
2467 
2468 	ASSERT(DRMACH_IS_MEM_ID(id));
2469 	dp = id;
2470 
2471 	/* get starting physical address of target memory */
2472 	pa = dp->base_pa;
2473 
2474 	/* round down to slice boundary */
2475 	slice_size = dp->slice_size;
2476 	pa &= ~(slice_size - 1);
2477 
2478 	/* stop at first span that is in slice */
2479 	memlist_read_lock();
2480 	for (ml = phys_install; ml; ml = ml->next)
2481 		if (ml->address >= pa && ml->address < pa + slice_size)
2482 			break;
2483 	memlist_read_unlock();
2484 
2485 	stat->assigned = dp->dev.bp->assigned;
2486 	stat->powered = dp->dev.bp->powered;
2487 	stat->configured = (ml != NULL);
2488 	stat->busy = dp->dev.busy;
2489 	strncpy(stat->type, dp->dev.type, sizeof (stat->type));
2490 	stat->info[0] = '\0';
2491 
2492 	return (NULL);
2493 }
2494 
2495 
2496 sbd_error_t *
2497 drmach_board_deprobe(drmachid_t id)
2498 {
2499 	drmach_board_t	*bp;
2500 
2501 	if (!DRMACH_IS_BOARD_ID(id))
2502 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2503 
2504 	bp = id;
2505 
2506 	cmn_err(CE_CONT, "DR: detach board %d\n", bp->bnum);
2507 
2508 	if (bp->tree) {
2509 		drmach_node_dispose(bp->tree);
2510 		bp->tree = NULL;
2511 	}
2512 	if (bp->devices) {
2513 		drmach_array_dispose(bp->devices, drmach_device_dispose);
2514 		bp->devices = NULL;
2515 	}
2516 
2517 	bp->boot_board = 0;
2518 
2519 	return (NULL);
2520 }
2521 
2522 /*ARGSUSED*/
2523 static sbd_error_t *
2524 drmach_pt_ikprobe(drmachid_t id, drmach_opts_t *opts)
2525 {
2526 	drmach_board_t		*bp = (drmach_board_t *)id;
2527 	sbd_error_t		*err = NULL;
2528 	int	rv;
2529 
2530 	if (!DRMACH_IS_BOARD_ID(id))
2531 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2532 
2533 	DRMACH_PR("calling opl_probe_board for bnum=%d\n", bp->bnum);
2534 	rv = opl_probe_sb(bp->bnum);
2535 	if (rv != 0) {
2536 		err = drerr_new(1, EOPL_PROBE, bp->cm.name);
2537 		return (err);
2538 	}
2539 	return (err);
2540 }
2541 
2542 /*ARGSUSED*/
2543 static sbd_error_t *
2544 drmach_pt_ikdeprobe(drmachid_t id, drmach_opts_t *opts)
2545 {
2546 	drmach_board_t	*bp;
2547 	sbd_error_t	*err = NULL;
2548 	int	rv;
2549 
2550 	if (!DRMACH_IS_BOARD_ID(id))
2551 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2552 	bp = (drmach_board_t *)id;
2553 
2554 	cmn_err(CE_CONT, "DR: in-kernel unprobe board %d\n", bp->bnum);
2555 
2556 	rv = opl_unprobe_sb(bp->bnum);
2557 	if (rv != 0) {
2558 		err = drerr_new(1, EOPL_DEPROBE, bp->cm.name);
2559 	}
2560 
2561 	return (err);
2562 }
2563 
2564 
2565 /*ARGSUSED*/
2566 sbd_error_t *
2567 drmach_pt_readmem(drmachid_t id, drmach_opts_t *opts)
2568 {
2569 	struct memlist	*ml;
2570 	uint64_t	src_pa;
2571 	uint64_t	dst_pa;
2572 	uint64_t	dst;
2573 
2574 	dst_pa = va_to_pa(&dst);
2575 
2576 	memlist_read_lock();
2577 	for (ml = phys_install; ml; ml = ml->next) {
2578 		uint64_t	nbytes;
2579 
2580 		src_pa = ml->address;
2581 		nbytes = ml->size;
2582 
2583 		while (nbytes != 0ull) {
2584 
2585 			/* copy 32 bytes at arc_pa to dst_pa */
2586 			bcopy32_il(src_pa, dst_pa);
2587 
2588 			/* increment by 32 bytes */
2589 			src_pa += (4 * sizeof (uint64_t));
2590 
2591 			/* decrement by 32 bytes */
2592 			nbytes -= (4 * sizeof (uint64_t));
2593 		}
2594 	}
2595 	memlist_read_unlock();
2596 
2597 	return (NULL);
2598 }
2599 
2600 static struct {
2601 	const char	*name;
2602 	sbd_error_t	*(*handler)(drmachid_t id, drmach_opts_t *opts);
2603 } drmach_pt_arr[] = {
2604 	{ "readmem",		drmach_pt_readmem		},
2605 	{ "ikprobe",	drmach_pt_ikprobe	},
2606 	{ "ikdeprobe",	drmach_pt_ikdeprobe	},
2607 
2608 	/* the following line must always be last */
2609 	{ NULL,			NULL				}
2610 };
2611 
2612 /*ARGSUSED*/
2613 sbd_error_t *
2614 drmach_passthru(drmachid_t id, drmach_opts_t *opts)
2615 {
2616 	int		i;
2617 	sbd_error_t	*err;
2618 
2619 	i = 0;
2620 	while (drmach_pt_arr[i].name != NULL) {
2621 		int len = strlen(drmach_pt_arr[i].name);
2622 
2623 		if (strncmp(drmach_pt_arr[i].name, opts->copts, len) == 0)
2624 			break;
2625 
2626 		i += 1;
2627 	}
2628 
2629 	if (drmach_pt_arr[i].name == NULL)
2630 		err = drerr_new(0, EOPL_UNKPTCMD, opts->copts);
2631 	else
2632 		err = (*drmach_pt_arr[i].handler)(id, opts);
2633 
2634 	return (err);
2635 }
2636 
2637 sbd_error_t *
2638 drmach_release(drmachid_t id)
2639 {
2640 	drmach_common_t *cp;
2641 
2642 	if (!DRMACH_IS_DEVICE_ID(id))
2643 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2644 	cp = id;
2645 
2646 	return (cp->release(id));
2647 }
2648 
2649 sbd_error_t *
2650 drmach_status(drmachid_t id, drmach_status_t *stat)
2651 {
2652 	drmach_common_t *cp;
2653 	sbd_error_t	*err;
2654 
2655 	rw_enter(&drmach_boards_rwlock, RW_READER);
2656 
2657 	if (!DRMACH_IS_ID(id)) {
2658 		rw_exit(&drmach_boards_rwlock);
2659 		return (drerr_new(0, EOPL_NOTID, NULL));
2660 	}
2661 	cp = (drmach_common_t *)id;
2662 	err = cp->status(id, stat);
2663 
2664 	rw_exit(&drmach_boards_rwlock);
2665 
2666 	return (err);
2667 }
2668 
2669 static sbd_error_t *
2670 drmach_i_status(drmachid_t id, drmach_status_t *stat)
2671 {
2672 	drmach_common_t *cp;
2673 
2674 	if (!DRMACH_IS_ID(id))
2675 		return (drerr_new(0, EOPL_NOTID, NULL));
2676 	cp = id;
2677 
2678 	return (cp->status(id, stat));
2679 }
2680 
2681 /*ARGSUSED*/
2682 sbd_error_t *
2683 drmach_unconfigure(drmachid_t id, int flags)
2684 {
2685 	drmach_device_t *dp;
2686 	dev_info_t	*rdip, *fdip = NULL;
2687 	char name[OBP_MAXDRVNAME];
2688 	int rv;
2689 
2690 	if (DRMACH_IS_CPU_ID(id))
2691 		return (NULL);
2692 
2693 	if (!DRMACH_IS_DEVICE_ID(id))
2694 		return (drerr_new(0, EOPL_INAPPROP, NULL));
2695 
2696 	dp = id;
2697 
2698 	rdip = dp->node->n_getdip(dp->node);
2699 
2700 	ASSERT(rdip);
2701 
2702 	rv = dp->node->n_getprop(dp->node, "name", name, OBP_MAXDRVNAME);
2703 
2704 	if (rv)
2705 		return (NULL);
2706 
2707 	/*
2708 	 * Note: FORCE flag is no longer necessary under devfs
2709 	 */
2710 
2711 	ASSERT(e_ddi_branch_held(rdip));
2712 	if (e_ddi_branch_unconfigure(rdip, &fdip, 0)) {
2713 		sbd_error_t	*err;
2714 		char		*path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
2715 
2716 		/*
2717 		 * If non-NULL, fdip is returned held and must be released.
2718 		 */
2719 		if (fdip != NULL) {
2720 			(void) ddi_pathname(fdip, path);
2721 			ndi_rele_devi(fdip);
2722 		} else {
2723 			(void) ddi_pathname(rdip, path);
2724 		}
2725 
2726 		err = drerr_new(1, EOPL_DRVFAIL, path);
2727 
2728 		kmem_free(path, MAXPATHLEN);
2729 
2730 		return (err);
2731 	}
2732 
2733 	return (NULL);
2734 }
2735 
2736 
2737 int
2738 drmach_cpu_poweron(struct cpu *cp)
2739 {
2740 	int bnum, cpuid, onb_core_num, strand_id;
2741 	drmach_board_t *bp;
2742 
2743 	DRMACH_PR("drmach_cpu_poweron: starting cpuid %d\n", cp->cpu_id);
2744 
2745 	cpuid = cp->cpu_id;
2746 	bnum = LSB_ID(cpuid);
2747 	onb_core_num = ON_BOARD_CORE_NUM(cpuid);
2748 	strand_id = STRAND_ID(cpuid);
2749 	bp = drmach_get_board_by_bnum(bnum);
2750 
2751 	ASSERT(bp);
2752 	if (bp->cores[onb_core_num].core_hotadded == 0) {
2753 		if (drmach_add_remove_cpu(bnum, onb_core_num,
2754 			HOTADD_CPU) != 0) {
2755 			cmn_err(CE_WARN, "Failed to add CMP %d on board %d\n",
2756 				onb_core_num, bnum);
2757 			return (EIO);
2758 		}
2759 	}
2760 
2761 	ASSERT(MUTEX_HELD(&cpu_lock));
2762 
2763 	if (drmach_cpu_start(cp) != 0) {
2764 		if (bp->cores[onb_core_num].core_started == 0) {
2765 			/*
2766 			 * we must undo the hotadd or no one will do that
2767 			 * If this fails, we will do this again in
2768 			 * drmach_board_disconnect.
2769 			 */
2770 			if (drmach_add_remove_cpu(bnum, onb_core_num,
2771 				HOTREMOVE_CPU) != 0) {
2772 				cmn_err(CE_WARN, "Failed to remove CMP %d "
2773 					"on board %d\n",
2774 					onb_core_num, bnum);
2775 			}
2776 		}
2777 		return (EBUSY);
2778 	} else {
2779 		bp->cores[onb_core_num].core_started |= (1 << strand_id);
2780 		return (0);
2781 	}
2782 }
2783 
2784 int
2785 drmach_cpu_poweroff(struct cpu *cp)
2786 {
2787 	int 		rv = 0;
2788 	processorid_t	cpuid = cp->cpu_id;
2789 
2790 	DRMACH_PR("drmach_cpu_poweroff: stopping cpuid %d\n", cp->cpu_id);
2791 
2792 	ASSERT(MUTEX_HELD(&cpu_lock));
2793 
2794 	/*
2795 	 * Capture all CPUs (except for detaching proc) to prevent
2796 	 * crosscalls to the detaching proc until it has cleared its
2797 	 * bit in cpu_ready_set.
2798 	 *
2799 	 * The CPU's remain paused and the prom_mutex is known to be free.
2800 	 * This prevents the x-trap victim from blocking when doing prom
2801 	 * IEEE-1275 calls at a high PIL level.
2802 	 */
2803 
2804 	promsafe_pause_cpus();
2805 
2806 	/*
2807 	 * Quiesce interrupts on the target CPU. We do this by setting
2808 	 * the CPU 'not ready'- (i.e. removing the CPU from cpu_ready_set) to
2809 	 * prevent it from receiving cross calls and cross traps.
2810 	 * This prevents the processor from receiving any new soft interrupts.
2811 	 */
2812 	mp_cpu_quiesce(cp);
2813 
2814 	rv = prom_stopcpu_bycpuid(cpuid);
2815 	if (rv == 0)
2816 		cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
2817 
2818 	start_cpus();
2819 
2820 	if (rv == 0) {
2821 		int bnum, onb_core_num, strand_id;
2822 		drmach_board_t *bp;
2823 
2824 		CPU_SIGNATURE(OS_SIG, SIGST_DETACHED, SIGSUBST_NULL, cpuid);
2825 
2826 		bnum = LSB_ID(cpuid);
2827 		onb_core_num = ON_BOARD_CORE_NUM(cpuid);
2828 		strand_id = STRAND_ID(cpuid);
2829 		bp = drmach_get_board_by_bnum(bnum);
2830 		ASSERT(bp);
2831 
2832 		bp->cores[onb_core_num].core_started &= ~(1 << strand_id);
2833 		if (bp->cores[onb_core_num].core_started == 0) {
2834 			if (drmach_add_remove_cpu(bnum, onb_core_num,
2835 				HOTREMOVE_CPU) != 0) {
2836 				cmn_err(CE_WARN,
2837 					"Failed to remove CMP %d LSB %d\n",
2838 					onb_core_num, bnum);
2839 				return (EIO);
2840 			}
2841 		}
2842 	}
2843 
2844 	return (rv);
2845 }
2846 
2847 /*ARGSUSED*/
2848 int
2849 drmach_verify_sr(dev_info_t *dip, int sflag)
2850 {
2851 	return (0);
2852 }
2853 
2854 void
2855 drmach_suspend_last(void)
2856 {
2857 }
2858 
2859 void
2860 drmach_resume_first(void)
2861 {
2862 }
2863 
2864 /*
2865  * Log a DR sysevent.
2866  * Return value: 0 success, non-zero failure.
2867  */
2868 int
2869 drmach_log_sysevent(int board, char *hint, int flag, int verbose)
2870 {
2871 	sysevent_t			*ev;
2872 	sysevent_id_t			eid;
2873 	int				rv, km_flag;
2874 	sysevent_value_t		evnt_val;
2875 	sysevent_attr_list_t		*evnt_attr_list = NULL;
2876 	char				attach_pnt[MAXNAMELEN];
2877 
2878 	km_flag = (flag == SE_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2879 	attach_pnt[0] = '\0';
2880 	if (drmach_board_name(board, attach_pnt, MAXNAMELEN)) {
2881 		rv = -1;
2882 		goto logexit;
2883 	}
2884 	if (verbose)
2885 		DRMACH_PR("drmach_log_sysevent: %s %s, flag: %d, verbose: %d\n",
2886 			attach_pnt, hint, flag, verbose);
2887 
2888 	if ((ev = sysevent_alloc(EC_DR, ESC_DR_AP_STATE_CHANGE,
2889 		SUNW_KERN_PUB"dr", km_flag)) == NULL) {
2890 		rv = -2;
2891 		goto logexit;
2892 	}
2893 	evnt_val.value_type = SE_DATA_TYPE_STRING;
2894 	evnt_val.value.sv_string = attach_pnt;
2895 	if ((rv = sysevent_add_attr(&evnt_attr_list, DR_AP_ID,
2896 		&evnt_val, km_flag)) != 0)
2897 		goto logexit;
2898 
2899 	evnt_val.value_type = SE_DATA_TYPE_STRING;
2900 	evnt_val.value.sv_string = hint;
2901 	if ((rv = sysevent_add_attr(&evnt_attr_list, DR_HINT,
2902 		&evnt_val, km_flag)) != 0) {
2903 		sysevent_free_attr(evnt_attr_list);
2904 		goto logexit;
2905 	}
2906 
2907 	(void) sysevent_attach_attributes(ev, evnt_attr_list);
2908 
2909 	/*
2910 	 * Log the event but do not sleep waiting for its
2911 	 * delivery. This provides insulation from syseventd.
2912 	 */
2913 	rv = log_sysevent(ev, SE_NOSLEEP, &eid);
2914 
2915 logexit:
2916 	if (ev)
2917 		sysevent_free(ev);
2918 	if ((rv != 0) && verbose)
2919 		cmn_err(CE_WARN,
2920 			"drmach_log_sysevent failed (rv %d) for %s  %s\n",
2921 			rv, attach_pnt, hint);
2922 
2923 	return (rv);
2924 }
2925 
2926 #define	OPL_DR_STATUS_PROP "dr-status"
2927 
2928 static int
2929 opl_check_dr_status()
2930 {
2931 	pnode_t	node;
2932 	int	rtn, len;
2933 	char	*str;
2934 
2935 	node = prom_rootnode();
2936 	if (node == OBP_BADNODE) {
2937 		return (1);
2938 	}
2939 
2940 	len = prom_getproplen(node, OPL_DR_STATUS_PROP);
2941 	if (len == -1) {
2942 		/*
2943 		 * dr-status doesn't exist when DR is activated and
2944 		 * any warning messages aren't needed.
2945 		 */
2946 		return (1);
2947 	}
2948 
2949 	str = (char *)kmem_zalloc(len+1, KM_SLEEP);
2950 	rtn = prom_getprop(node, OPL_DR_STATUS_PROP, str);
2951 	kmem_free(str, len + 1);
2952 	if (rtn == -1) {
2953 		return (1);
2954 	} else {
2955 		return (0);
2956 	}
2957 }
2958 
2959 /* we are allocating memlist from TLB locked pages to avoid tlbmisses */
2960 
2961 static struct memlist *
2962 drmach_memlist_add_span(drmach_copy_rename_program_t *p,
2963 	struct memlist *mlist, uint64_t base, uint64_t len)
2964 {
2965 	struct memlist	*ml, *tl, *nl;
2966 
2967 	if (len == 0ull)
2968 		return (NULL);
2969 
2970 	if (mlist == NULL) {
2971 		mlist = p->free_mlist;
2972 		if (mlist == NULL)
2973 			return (NULL);
2974 		p->free_mlist = mlist->next;
2975 		mlist->address = base;
2976 		mlist->size = len;
2977 		mlist->next = mlist->prev = NULL;
2978 
2979 		return (mlist);
2980 	}
2981 
2982 	for (tl = ml = mlist; ml; tl = ml, ml = ml->next) {
2983 		if (base < ml->address) {
2984 			if ((base + len) < ml->address) {
2985 				nl = p->free_mlist;
2986 				if (nl == NULL)
2987 					return (NULL);
2988 				p->free_mlist = nl->next;
2989 				nl->address = base;
2990 				nl->size = len;
2991 				nl->next = ml;
2992 				if ((nl->prev = ml->prev) != NULL)
2993 					nl->prev->next = nl;
2994 				ml->prev = nl;
2995 				if (mlist == ml)
2996 					mlist = nl;
2997 			} else {
2998 				ml->size = MAX((base + len),
2999 					(ml->address + ml->size)) -
3000 					base;
3001 				ml->address = base;
3002 			}
3003 			break;
3004 
3005 		} else if (base <= (ml->address + ml->size)) {
3006 			ml->size = MAX((base + len),
3007 				(ml->address + ml->size)) -
3008 				MIN(ml->address, base);
3009 			ml->address = MIN(ml->address, base);
3010 			break;
3011 		}
3012 	}
3013 	if (ml == NULL) {
3014 		nl = p->free_mlist;
3015 		if (nl == NULL)
3016 			return (NULL);
3017 		p->free_mlist = nl->next;
3018 		nl->address = base;
3019 		nl->size = len;
3020 		nl->next = NULL;
3021 		nl->prev = tl;
3022 		tl->next = nl;
3023 	}
3024 
3025 	return (mlist);
3026 }
3027 
3028 /*
3029  * The routine performs the necessary memory COPY and MC adr SWITCH.
3030  * Both operations MUST be at the same "level" so that the stack is
3031  * maintained correctly between the copy and switch.  The switch
3032  * portion implements a caching mechanism to guarantee the code text
3033  * is cached prior to execution.  This is to guard against possible
3034  * memory access while the MC adr's are being modified.
3035  *
3036  * IMPORTANT: The _drmach_copy_rename_end() function must immediately
3037  * follow drmach_copy_rename_prog__relocatable() so that the correct
3038  * "length" of the drmach_copy_rename_prog__relocatable can be
3039  * calculated.  This routine MUST be a LEAF function, i.e. it can
3040  * make NO function calls, primarily for two reasons:
3041  *
3042  *	1. We must keep the stack consistent across the "switch".
3043  *	2. Function calls are compiled to relative offsets, and
3044  *	   we execute this function we'll be executing it from
3045  *	   a copied version in a different area of memory, thus
3046  *	   the relative offsets will be bogus.
3047  *
3048  * Moreover, it must have the "__relocatable" suffix to inform DTrace
3049  * providers (and anything else, for that matter) that this
3050  * function's text is manually relocated elsewhere before it is
3051  * executed.  That is, it cannot be safely instrumented with any
3052  * methodology that is PC-relative.
3053  */
3054 
3055 /*
3056  * We multiply this to system_clock_frequency so we
3057  * are setting a delay of fmem_timeout second for
3058  * the rename command.  The spec says 15 second is
3059  * enough but the Fujitsu HW team suggested 17 sec.
3060  */
3061 static int	fmem_timeout = 17;
3062 static int	min_copy_size_per_sec = 20 * 1024 * 1024;
3063 int drmach_disable_mcopy = 0;
3064 
3065 /*
3066  * The following delay loop executes sleep instruction to yield the
3067  * CPU to other strands.  If this is not done, some strand will tie
3068  * up the CPU in busy loops while the other strand cannot do useful
3069  * work.  The copy procedure will take a much longer time without this.
3070  */
3071 #define	DR_DELAY_IL(ms, freq)					\
3072 	{							\
3073 		uint64_t start;					\
3074 		uint64_t nstick;				\
3075 		volatile uint64_t now;				\
3076 		nstick = ((uint64_t)ms * freq)/1000;		\
3077 		start = drmach_get_stick_il();			\
3078 		now = start;					\
3079 		while ((now - start) <= nstick) {		\
3080 			drmach_sleep_il();			\
3081 			now = drmach_get_stick_il();		\
3082 		}						\
3083 	}
3084 
3085 static int
3086 drmach_copy_rename_prog__relocatable(drmach_copy_rename_program_t *prog,
3087 	int cpuid)
3088 {
3089 	struct memlist		*ml;
3090 	register int		rtn;
3091 	int			i;
3092 	register uint64_t	curr, limit;
3093 	extern uint64_t		drmach_get_stick_il();
3094 	extern void		membar_sync_il();
3095 	extern void		flush_instr_mem_il(void*);
3096 	extern void		flush_windows_il(void);
3097 	uint64_t		copy_start;
3098 
3099 	/*
3100 	 * flush_windows is moved here to make sure all
3101 	 * registers used in the callers are flushed to
3102 	 * memory before the copy.
3103 	 *
3104 	 * If flush_windows() is called too early in the
3105 	 * calling function, the compiler might put some
3106 	 * data in the local registers after flush_windows().
3107 	 * After FMA, if there is any fill trap, the registers
3108 	 * will contain stale data.
3109 	 */
3110 
3111 	flush_windows_il();
3112 
3113 	prog->critical->stat[cpuid] = FMEM_LOOP_COPY_READY;
3114 	membar_sync_il();
3115 
3116 	if (prog->data->cpuid == cpuid) {
3117 		limit = drmach_get_stick_il();
3118 		limit += prog->critical->delay;
3119 		for (i = 0; i < NCPU; i++) {
3120 			if (CPU_IN_SET(prog->data->cpu_slave_set, i)) {
3121 			/* wait for all CPU's to be ready */
3122 			    for (;;) {
3123 				if (prog->critical->stat[i] ==
3124 					FMEM_LOOP_COPY_READY) {
3125 					break;
3126 				}
3127 				DR_DELAY_IL(1, prog->data->stick_freq);
3128 			    }
3129 			    curr = drmach_get_stick_il();
3130 			    if (curr > limit) {
3131 				prog->data->fmem_status.error =
3132 					FMEM_XC_TIMEOUT;
3133 				return (FMEM_XC_TIMEOUT);
3134 			    }
3135 			}
3136 		}
3137 		prog->data->fmem_status.stat = FMEM_LOOP_COPY_READY;
3138 		membar_sync_il();
3139 		copy_start = drmach_get_stick_il();
3140 	} else {
3141 		for (;;) {
3142 			if (prog->data->fmem_status.stat ==
3143 				FMEM_LOOP_COPY_READY) {
3144 				break;
3145 			}
3146 			if (prog->data->fmem_status.error) {
3147 				prog->data->error[cpuid] = FMEM_TERMINATE;
3148 				return (FMEM_TERMINATE);
3149 			}
3150 			DR_DELAY_IL(1, prog->data->stick_freq);
3151 		}
3152 	}
3153 
3154 	/*
3155 	 * DO COPY.
3156 	 */
3157 	if (CPU_IN_SET(prog->data->cpu_copy_set, cpuid)) {
3158 	    for (ml = prog->data->cpu_ml[cpuid]; ml; ml = ml->next) {
3159 		uint64_t	s_pa, t_pa;
3160 		uint64_t	nbytes;
3161 
3162 		s_pa = prog->data->s_copybasepa + ml->address;
3163 		t_pa = prog->data->t_copybasepa + ml->address;
3164 		nbytes = ml->size;
3165 
3166 		while (nbytes != 0ull) {
3167 			/* If the master has detected error, we just bail out */
3168 			if (prog->data->fmem_status.error) {
3169 				prog->data->error[cpuid] = FMEM_TERMINATE;
3170 				return (FMEM_TERMINATE);
3171 			}
3172 			/*
3173 			 * This copy does NOT use an ASI
3174 			 * that avoids the Ecache, therefore
3175 			 * the dst_pa addresses may remain
3176 			 * in our Ecache after the dst_pa
3177 			 * has been removed from the system.
3178 			 * A subsequent write-back to memory
3179 			 * will cause an ARB-stop because the
3180 			 * physical address no longer exists
3181 			 * in the system. Therefore we must
3182 			 * flush out local Ecache after we
3183 			 * finish the copy.
3184 			 */
3185 
3186 			/* copy 32 bytes at src_pa to dst_pa */
3187 			bcopy32_il(s_pa, t_pa);
3188 
3189 			/* increment the counter to signal that we are alive */
3190 			prog->stat->nbytes[cpuid] += 32;
3191 
3192 			/* increment by 32 bytes */
3193 			s_pa += (4 * sizeof (uint64_t));
3194 			t_pa += (4 * sizeof (uint64_t));
3195 
3196 			/* decrement by 32 bytes */
3197 			nbytes -= (4 * sizeof (uint64_t));
3198 		}
3199 	    }
3200 	    prog->critical->stat[cpuid] = FMEM_LOOP_COPY_DONE;
3201 	    membar_sync_il();
3202 	}
3203 
3204 	/*
3205 	 * Since bcopy32_il() does NOT use an ASI to bypass
3206 	 * the Ecache, we need to flush our Ecache after
3207 	 * the copy is complete.
3208 	 */
3209 	flush_cache_il();
3210 
3211 	/*
3212 	 * drmach_fmem_exec_script()
3213 	 */
3214 	if (prog->data->cpuid == cpuid) {
3215 		uint64_t	last, now;
3216 
3217 		limit = copy_start + prog->data->copy_delay;
3218 		for (i = 0; i < NCPU; i++) {
3219 			if (CPU_IN_SET(prog->data->cpu_slave_set, i)) {
3220 			    for (;;) {
3221 				/* we get FMEM_LOOP_FMEM_READY in normal case */
3222 				if (prog->critical->stat[i] ==
3223 					FMEM_LOOP_FMEM_READY) {
3224 					break;
3225 				}
3226 				/* got error traps */
3227 				if (prog->critical->stat[i] ==
3228 					FMEM_COPY_ERROR) {
3229 					prog->data->fmem_status.error =
3230 						FMEM_COPY_ERROR;
3231 					return (FMEM_COPY_ERROR);
3232 				}
3233 				/* if we have not reached limit, wait more */
3234 				curr = drmach_get_stick_il();
3235 				if (curr <= limit)
3236 					continue;
3237 
3238 				prog->data->slowest_cpuid = i;
3239 				prog->data->copy_wait_time =
3240 					curr - copy_start;
3241 
3242 				/* now check if slave is alive */
3243 				last = prog->stat->nbytes[i];
3244 
3245 				DR_DELAY_IL(1, prog->data->stick_freq);
3246 
3247 				now = prog->stat->nbytes[i];
3248 				if (now <= last) {
3249 					/* no progress, perhaps just finished */
3250 					DR_DELAY_IL(1, prog->data->stick_freq);
3251 					if (prog->critical->stat[i] ==
3252 						FMEM_LOOP_FMEM_READY)
3253 						break;
3254 					/* copy error */
3255 					if (prog->critical->stat[i] ==
3256 						FMEM_COPY_ERROR) {
3257 						prog->data->fmem_status.error =
3258 							FMEM_COPY_ERROR;
3259 						return (FMEM_COPY_ERROR);
3260 					}
3261 					prog->data->fmem_status.error =
3262 					    FMEM_COPY_TIMEOUT;
3263 					return (FMEM_COPY_TIMEOUT);
3264 				}
3265 			    }
3266 			}
3267 		}
3268 		prog->critical->stat[cpuid] = FMEM_LOOP_FMEM_READY;
3269 		prog->data->fmem_status.stat  = FMEM_LOOP_FMEM_READY;
3270 
3271 		membar_sync_il();
3272 		flush_instr_mem_il((void*) (prog->critical));
3273 		/*
3274 		 * drmach_fmem_exec_script()
3275 		 */
3276 		rtn = prog->critical->fmem((void *)prog->critical, PAGESIZE);
3277 		return (rtn);
3278 	} else {
3279 		flush_instr_mem_il((void*) (prog->critical));
3280 		/*
3281 		 * drmach_fmem_loop_script()
3282 		 */
3283 		rtn = prog->critical->loop((void *)(prog->critical),
3284 			PAGESIZE, (void *)&(prog->critical->stat[cpuid]));
3285 		prog->data->error[cpuid] = rtn;
3286 		/* slave thread does not care the rv */
3287 		return (0);
3288 	}
3289 }
3290 
3291 static void
3292 drmach_copy_rename_end(void)
3293 {
3294 	/*
3295 	 * IMPORTANT:	This function's location MUST be located immediately
3296 	 *		following drmach_copy_rename_prog__relocatable to
3297 	 *		accurately estimate its size.  Note that this assumes
3298 	 *		the compiler keeps these functions in the order in
3299 	 *		which they appear :-o
3300 	 */
3301 }
3302 
3303 
3304 static void
3305 drmach_setup_memlist(drmach_copy_rename_program_t *p)
3306 {
3307 	struct memlist *ml;
3308 	caddr_t buf;
3309 	int nbytes, s;
3310 
3311 	nbytes = PAGESIZE;
3312 	s = roundup(sizeof (struct memlist), sizeof (void *));
3313 	p->free_mlist = NULL;
3314 	buf = p->memlist_buffer;
3315 	while (nbytes >= sizeof (struct memlist)) {
3316 		ml = (struct memlist *)buf;
3317 		ml->next = p->free_mlist;
3318 		p->free_mlist = ml;
3319 		buf += s;
3320 		nbytes -= s;
3321 	}
3322 }
3323 
3324 static void
3325 drmach_lock_critical(caddr_t va, caddr_t new_va)
3326 {
3327 	tte_t tte;
3328 	int i;
3329 
3330 	kpreempt_disable();
3331 
3332 	for (i = 0; i < DRMACH_FMEM_LOCKED_PAGES; i++) {
3333 		vtag_flushpage(new_va, (uint64_t)ksfmmup);
3334 		sfmmu_memtte(&tte, va_to_pfn(va),
3335 			PROC_DATA|HAT_NOSYNC, TTE8K);
3336 		tte.tte_intlo |= TTE_LCK_INT;
3337 		sfmmu_dtlb_ld_kva(new_va, &tte);
3338 		sfmmu_itlb_ld_kva(new_va, &tte);
3339 		va += PAGESIZE;
3340 		new_va += PAGESIZE;
3341 	}
3342 }
3343 
3344 static void
3345 drmach_unlock_critical(caddr_t va)
3346 {
3347 	int i;
3348 
3349 	for (i = 0; i < DRMACH_FMEM_LOCKED_PAGES; i++) {
3350 		vtag_flushpage(va, (uint64_t)ksfmmup);
3351 		va += PAGESIZE;
3352 	}
3353 
3354 	kpreempt_enable();
3355 }
3356 
3357 sbd_error_t *
3358 drmach_copy_rename_init(drmachid_t t_id, drmachid_t s_id,
3359 	struct memlist *c_ml, drmachid_t *pgm_id)
3360 {
3361 	drmach_mem_t	*s_mem;
3362 	drmach_mem_t	*t_mem;
3363 	struct memlist	*x_ml;
3364 	uint64_t	s_copybasepa, t_copybasepa;
3365 	uint_t		len;
3366 	caddr_t		bp, wp;
3367 	int			s_bd, t_bd, cpuid, active_cpus, i;
3368 	uint64_t		c_addr;
3369 	size_t			c_size, copy_sz, sz;
3370 	extern void		drmach_fmem_loop_script();
3371 	extern void		drmach_fmem_loop_script_rtn();
3372 	extern int		drmach_fmem_exec_script();
3373 	extern void		drmach_fmem_exec_script_end();
3374 	sbd_error_t	*err;
3375 	drmach_copy_rename_program_t *prog = NULL;
3376 	drmach_copy_rename_program_t *prog_kmem = NULL;
3377 	void		(*mc_suspend)(void);
3378 	void		(*mc_resume)(void);
3379 	int		(*scf_fmem_start)(int, int);
3380 	int		(*scf_fmem_end)(void);
3381 	int		(*scf_fmem_cancel)(void);
3382 	uint64_t	(*scf_get_base_addr)(void);
3383 
3384 	if (!DRMACH_IS_MEM_ID(s_id))
3385 		return (drerr_new(0, EOPL_INAPPROP, NULL));
3386 	if (!DRMACH_IS_MEM_ID(t_id))
3387 		return (drerr_new(0, EOPL_INAPPROP, NULL));
3388 
3389 	for (i = 0; i < NCPU; i++) {
3390 		int lsb_id, onb_core_num, strand_id;
3391 		drmach_board_t *bp;
3392 
3393 		/*
3394 		 * this kind of CPU will spin in cache
3395 		 */
3396 		if (CPU_IN_SET(cpu_ready_set, i))
3397 			continue;
3398 
3399 		/*
3400 		 * Now check for any inactive CPU's that
3401 		 * have been hotadded.  This can only occur in
3402 		 * error condition in drmach_cpu_poweron().
3403 		 */
3404 		lsb_id = LSB_ID(i);
3405 		onb_core_num = ON_BOARD_CORE_NUM(i);
3406 		strand_id = STRAND_ID(i);
3407 		bp = drmach_get_board_by_bnum(lsb_id);
3408 		if (bp == NULL)
3409 			continue;
3410 		if (bp->cores[onb_core_num].core_hotadded &
3411 		    (1 << strand_id)) {
3412 		    if (!(bp->cores[onb_core_num].core_started &
3413 			(1 << strand_id))) {
3414 			return (drerr_new(1, EOPL_CPU_STATE, NULL));
3415 		    }
3416 		}
3417 	}
3418 
3419 	mc_suspend = (void (*)(void))
3420 	    modgetsymvalue("opl_mc_suspend", 0);
3421 	mc_resume = (void (*)(void))
3422 	    modgetsymvalue("opl_mc_resume", 0);
3423 
3424 	if (mc_suspend == NULL || mc_resume == NULL) {
3425 		return (drerr_new(1, EOPL_MC_OPL, NULL));
3426 	}
3427 
3428 	scf_fmem_start = (int (*)(int, int))
3429 	    modgetsymvalue("scf_fmem_start", 0);
3430 	if (scf_fmem_start == NULL) {
3431 		return (drerr_new(1, EOPL_SCF_FMEM, NULL));
3432 	}
3433 	scf_fmem_end = (int (*)(void))
3434 	    modgetsymvalue("scf_fmem_end", 0);
3435 	if (scf_fmem_end == NULL) {
3436 		return (drerr_new(1, EOPL_SCF_FMEM, NULL));
3437 	}
3438 	scf_fmem_cancel = (int (*)(void))
3439 	    modgetsymvalue("scf_fmem_cancel", 0);
3440 	if (scf_fmem_cancel == NULL) {
3441 		return (drerr_new(1, EOPL_SCF_FMEM, NULL));
3442 	}
3443 	scf_get_base_addr = (uint64_t (*)(void))
3444 	    modgetsymvalue("scf_get_base_addr", 0);
3445 	if (scf_get_base_addr == NULL) {
3446 		return (drerr_new(1, EOPL_SCF_FMEM, NULL));
3447 	}
3448 	s_mem = s_id;
3449 	t_mem = t_id;
3450 
3451 	s_bd = s_mem->dev.bp->bnum;
3452 	t_bd = t_mem->dev.bp->bnum;
3453 
3454 	/* calculate source and target base pa */
3455 
3456 	s_copybasepa = s_mem->slice_base;
3457 	t_copybasepa = t_mem->slice_base;
3458 
3459 	/* adjust copy memlist addresses to be relative to copy base pa */
3460 	x_ml = c_ml;
3461 	while (x_ml != NULL) {
3462 		x_ml->address -= s_copybasepa;
3463 		x_ml = x_ml->next;
3464 	}
3465 
3466 	/*
3467 	 * bp will be page aligned, since we're calling
3468 	 * kmem_zalloc() with an exact multiple of PAGESIZE.
3469 	 */
3470 
3471 	prog_kmem = (drmach_copy_rename_program_t *)kmem_zalloc(
3472 		DRMACH_FMEM_LOCKED_PAGES * PAGESIZE, KM_SLEEP);
3473 
3474 	prog_kmem->prog = prog_kmem;
3475 
3476 	/*
3477 	 * To avoid MTLB hit, we allocate a new VM space and remap
3478 	 * the kmem_alloc buffer to that address.  This solves
3479 	 * 2 problems we found:
3480 	 * - the kmem_alloc buffer can be just a chunk inside
3481 	 *   a much larger, e.g. 4MB buffer and MTLB will occur
3482 	 *   if there are both a 4MB and a 8K TLB mapping to
3483 	 *   the same VA range.
3484 	 * - the kmem mapping got dropped into the TLB by other
3485 	 *   strands, unintentionally.
3486 	 * Note that the pointers like data, critical, memlist_buffer,
3487 	 * and stat inside the copy rename structure are mapped to this
3488 	 * alternate VM space so we must make sure we lock the TLB mapping
3489 	 * whenever we access data pointed to by these pointers.
3490 	 */
3491 
3492 	prog = prog_kmem->locked_prog = vmem_alloc(heap_arena,
3493 		DRMACH_FMEM_LOCKED_PAGES * PAGESIZE, VM_SLEEP);
3494 	wp = bp = (caddr_t)prog;
3495 
3496 	/* Now remap prog_kmem to prog */
3497 	drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);
3498 
3499 	/* All pointers in prog are based on the alternate mapping */
3500 	prog->data = (drmach_copy_rename_data_t *)roundup(((uint64_t)prog +
3501 		sizeof (drmach_copy_rename_program_t)), sizeof (void *));
3502 
3503 	ASSERT(((uint64_t)prog->data + sizeof (drmach_copy_rename_data_t))
3504 		<= ((uint64_t)prog + PAGESIZE));
3505 
3506 	prog->critical = (drmach_copy_rename_critical_t *)
3507 		(wp + DRMACH_FMEM_CRITICAL_PAGE * PAGESIZE);
3508 
3509 	prog->memlist_buffer = (caddr_t)(wp +
3510 		DRMACH_FMEM_MLIST_PAGE * PAGESIZE);
3511 
3512 	prog->stat = (drmach_cr_stat_t *)(wp +
3513 		DRMACH_FMEM_STAT_PAGE * PAGESIZE);
3514 
3515 	/* LINTED */
3516 	ASSERT(sizeof (drmach_cr_stat_t)
3517 		<= ((DRMACH_FMEM_LOCKED_PAGES - DRMACH_FMEM_STAT_PAGE)
3518 		* PAGESIZE));
3519 
3520 	prog->critical->scf_reg_base = (uint64_t)-1;
3521 	prog->critical->scf_td[0] = (s_bd & 0xff);
3522 	prog->critical->scf_td[1] = (t_bd & 0xff);
3523 	for (i = 2; i < 15; i++) {
3524 		prog->critical->scf_td[i]   = 0;
3525 	}
3526 	prog->critical->scf_td[15] = ((0xaa + s_bd + t_bd) & 0xff);
3527 
3528 	bp = (caddr_t)prog->critical;
3529 	len = sizeof (drmach_copy_rename_critical_t);
3530 	wp = (caddr_t)roundup((uint64_t)bp + len, sizeof (void *));
3531 
3532 	len = (uint_t)((ulong_t)drmach_copy_rename_end -
3533 		(ulong_t)drmach_copy_rename_prog__relocatable);
3534 
3535 	/*
3536 	 * We always leave 1K nop's to prevent the processor from
3537 	 * speculative execution that causes memory access
3538 	 */
3539 	wp = wp + len + 1024;
3540 
3541 	len = (uint_t)((ulong_t)drmach_fmem_exec_script_end -
3542 		(ulong_t)drmach_fmem_exec_script);
3543 	/* this is the entry point of the loop script */
3544 	wp = wp + len + 1024;
3545 
3546 	len = (uint_t)((ulong_t)drmach_fmem_exec_script -
3547 		(ulong_t)drmach_fmem_loop_script);
3548 	wp = wp + len + 1024;
3549 
3550 	/* now we make sure there is 1K extra */
3551 
3552 	if ((wp - bp) > PAGESIZE) {
3553 		err = drerr_new(1, EOPL_FMEM_SETUP, NULL);
3554 		goto out;
3555 	}
3556 
3557 	bp = (caddr_t)prog->critical;
3558 	len = sizeof (drmach_copy_rename_critical_t);
3559 	wp = (caddr_t)roundup((uint64_t)bp + len, sizeof (void *));
3560 
3561 	prog->critical->run = (int (*)())(wp);
3562 	len = (uint_t)((ulong_t)drmach_copy_rename_end -
3563 		(ulong_t)drmach_copy_rename_prog__relocatable);
3564 
3565 	bcopy((caddr_t)drmach_copy_rename_prog__relocatable, wp, len);
3566 
3567 	wp = (caddr_t)roundup((uint64_t)wp + len, 1024);
3568 
3569 	prog->critical->fmem = (int (*)())(wp);
3570 	len = (int)((ulong_t)drmach_fmem_exec_script_end -
3571 		(ulong_t)drmach_fmem_exec_script);
3572 	bcopy((caddr_t)drmach_fmem_exec_script, wp, len);
3573 
3574 	len = (int)((ulong_t)drmach_fmem_exec_script_end -
3575 		(ulong_t)drmach_fmem_exec_script);
3576 	wp = (caddr_t)roundup((uint64_t)wp + len, 1024);
3577 
3578 	prog->critical->loop = (int (*)())(wp);
3579 	len = (int)((ulong_t)drmach_fmem_exec_script -
3580 		(ulong_t)drmach_fmem_loop_script);
3581 	bcopy((caddr_t)drmach_fmem_loop_script, (void *)wp, len);
3582 	len = (int)((ulong_t)drmach_fmem_loop_script_rtn-
3583 		(ulong_t)drmach_fmem_loop_script);
3584 	prog->critical->loop_rtn = (void (*)()) (wp+len);
3585 
3586 	/* now we are committed, call SCF, soft suspend mac patrol */
3587 	if ((*scf_fmem_start)(s_bd, t_bd)) {
3588 		err = drerr_new(1, EOPL_SCF_FMEM_START, NULL);
3589 		goto out;
3590 	}
3591 	prog->data->scf_fmem_end = scf_fmem_end;
3592 	prog->data->scf_fmem_cancel = scf_fmem_cancel;
3593 	prog->data->scf_get_base_addr = scf_get_base_addr;
3594 	prog->data->fmem_status.op |= OPL_FMEM_SCF_START;
3595 	/* soft suspend mac patrol */
3596 	(*mc_suspend)();
3597 	prog->data->fmem_status.op |= OPL_FMEM_MC_SUSPEND;
3598 	prog->data->mc_resume = mc_resume;
3599 
3600 	prog->critical->inst_loop_ret  =
3601 		*(uint64_t *)(prog->critical->loop_rtn);
3602 
3603 	/*
3604 	 * 0x30800000 is op code "ba,a	+0"
3605 	 */
3606 
3607 	*(uint_t *)(prog->critical->loop_rtn) = (uint_t)(0x30800000);
3608 
3609 	/*
3610 	 * set the value of SCF FMEM TIMEOUT
3611 	 */
3612 	prog->critical->delay = fmem_timeout * system_clock_freq;
3613 
3614 	prog->data->s_mem = (drmachid_t)s_mem;
3615 	prog->data->t_mem = (drmachid_t)t_mem;
3616 
3617 	cpuid = CPU->cpu_id;
3618 	prog->data->cpuid = cpuid;
3619 	prog->data->cpu_ready_set = cpu_ready_set;
3620 	prog->data->cpu_slave_set = cpu_ready_set;
3621 	prog->data->slowest_cpuid = (processorid_t)-1;
3622 	prog->data->copy_wait_time = 0;
3623 	CPUSET_DEL(prog->data->cpu_slave_set, cpuid);
3624 
3625 	for (i = 0; i < NCPU; i++) {
3626 		prog->data->cpu_ml[i] = NULL;
3627 	}
3628 
3629 	active_cpus = 0;
3630 	if (drmach_disable_mcopy) {
3631 		active_cpus = 1;
3632 		CPUSET_ADD(prog->data->cpu_copy_set, cpuid);
3633 	} else {
3634 		for (i = 0; i < NCPU; i++) {
3635 			if (CPU_IN_SET(cpu_ready_set, i) &&
3636 				CPU_ACTIVE(cpu[i])) {
3637 				CPUSET_ADD(prog->data->cpu_copy_set, i);
3638 				active_cpus++;
3639 			}
3640 		}
3641 	}
3642 
3643 	drmach_setup_memlist(prog);
3644 
3645 	x_ml = c_ml;
3646 	sz = 0;
3647 	while (x_ml != NULL) {
3648 		sz += x_ml->size;
3649 		x_ml = x_ml->next;
3650 	}
3651 
3652 	copy_sz = sz/active_cpus;
3653 	copy_sz = roundup(copy_sz, MMU_PAGESIZE4M);
3654 
3655 	while (sz > copy_sz*active_cpus) {
3656 		copy_sz += MMU_PAGESIZE4M;
3657 	}
3658 
3659 	prog->data->stick_freq = system_clock_freq;
3660 	prog->data->copy_delay = ((copy_sz / min_copy_size_per_sec) + 2) *
3661 		system_clock_freq;
3662 
3663 	x_ml = c_ml;
3664 	c_addr = x_ml->address;
3665 	c_size = x_ml->size;
3666 
3667 	for (i = 0; i < NCPU; i++) {
3668 		prog->stat->nbytes[i] = 0;
3669 		if (!CPU_IN_SET(prog->data->cpu_copy_set, i)) {
3670 			continue;
3671 		}
3672 		sz = copy_sz;
3673 
3674 		while (sz) {
3675 			if (c_size > sz) {
3676 				prog->data->cpu_ml[i] =
3677 					drmach_memlist_add_span(prog,
3678 					prog->data->cpu_ml[i],
3679 					c_addr, sz);
3680 				c_addr += sz;
3681 				c_size -= sz;
3682 				break;
3683 			} else {
3684 				sz -= c_size;
3685 				prog->data->cpu_ml[i] = drmach_memlist_add_span(
3686 					prog, prog->data->cpu_ml[i],
3687 						c_addr, c_size);
3688 				x_ml = x_ml->next;
3689 				if (x_ml != NULL) {
3690 					c_addr = x_ml->address;
3691 					c_size = x_ml->size;
3692 				} else {
3693 					goto end;
3694 				}
3695 			}
3696 		}
3697 	}
3698 end:
3699 	prog->data->s_copybasepa = s_copybasepa;
3700 	prog->data->t_copybasepa = t_copybasepa;
3701 	prog->data->c_ml = c_ml;
3702 	*pgm_id = prog_kmem;
3703 
3704 	/* Unmap the alternate space.  It will have to be remapped again */
3705 	drmach_unlock_critical((caddr_t)prog);
3706 	return (NULL);
3707 out:
3708 	if (prog != NULL) {
3709 		drmach_unlock_critical((caddr_t)prog);
3710 		vmem_free(heap_arena, prog,
3711 			DRMACH_FMEM_LOCKED_PAGES * PAGESIZE);
3712 	}
3713 	if (prog_kmem != NULL) {
3714 		kmem_free(prog_kmem, DRMACH_FMEM_LOCKED_PAGES * PAGESIZE);
3715 	}
3716 	return (err);
3717 }
3718 
3719 sbd_error_t *
3720 drmach_copy_rename_fini(drmachid_t id)
3721 {
3722 	drmach_copy_rename_program_t	*prog = id;
3723 	sbd_error_t			*err = NULL;
3724 	int				rv;
3725 
3726 	/*
3727 	 * Note that we have to delay calling SCF to find out the
3728 	 * status of the FMEM operation here because SCF cannot
3729 	 * respond while it is suspended.
3730 	 * This create a small window when we are sure about the
3731 	 * base address of the system board.
3732 	 * If there is any call to mc-opl to get memory unum,
3733 	 * mc-opl will return UNKNOWN as the unum.
3734 	 */
3735 
3736 	/*
3737 	 * we have to remap again because all the pointer like data,
3738 	 * critical in prog are based on the alternate vmem space.
3739 	 */
3740 	(void) drmach_lock_critical((caddr_t)prog, (caddr_t)prog->locked_prog);
3741 
3742 	if (prog->data->c_ml != NULL)
3743 		memlist_delete(prog->data->c_ml);
3744 
3745 	if ((prog->data->fmem_status.op &
3746 		(OPL_FMEM_SCF_START| OPL_FMEM_MC_SUSPEND)) !=
3747 		(OPL_FMEM_SCF_START | OPL_FMEM_MC_SUSPEND)) {
3748 		cmn_err(CE_PANIC, "drmach_copy_rename_fini: "
3749 			"invalid op code %x\n",
3750 				prog->data->fmem_status.op);
3751 	}
3752 
3753 	/* possible ops are SCF_START, MC_SUSPEND */
3754 	if (prog->critical->fmem_issued) {
3755 		if (prog->data->fmem_status.error != FMEM_NO_ERROR)
3756 			cmn_err(CE_PANIC, "scf fmem request failed. "
3757 				"error code = 0x%x.",
3758 				prog->data->fmem_status.error);
3759 		rv = (*prog->data->scf_fmem_end)();
3760 		if (rv) {
3761 			cmn_err(CE_PANIC, "scf_fmem_end() failed rv=%d", rv);
3762 		}
3763 		/*
3764 		 * If we get here, rename is successful.
3765 		 * Do all the copy rename post processing.
3766 		 */
3767 		drmach_swap_pa((drmach_mem_t *)prog->data->s_mem,
3768 			(drmach_mem_t *)prog->data->t_mem);
3769 	} else {
3770 		if (prog->data->fmem_status.error != 0) {
3771 			cmn_err(CE_WARN, "Kernel Migration fails. 0x%x",
3772 				prog->data->fmem_status.error);
3773 			err = drerr_new(1, EOPL_FMEM_ERROR, "FMEM error = 0x%x",
3774 				prog->data->fmem_status.error);
3775 		}
3776 		rv = (*prog->data->scf_fmem_cancel)();
3777 		if (rv) {
3778 		    cmn_err(CE_WARN, "scf_fmem_cancel() failed rv=0x%x", rv);
3779 		    if (!err)
3780 			err = drerr_new(1, EOPL_SCF_FMEM_CANCEL,
3781 			    "rv = 0x%x", rv);
3782 		}
3783 	}
3784 	/* soft resume mac patrol */
3785 	(*prog->data->mc_resume)();
3786 
3787 	drmach_unlock_critical((caddr_t)prog->locked_prog);
3788 
3789 	vmem_free(heap_arena, prog->locked_prog,
3790 		DRMACH_FMEM_LOCKED_PAGES * PAGESIZE);
3791 	kmem_free(prog, DRMACH_FMEM_LOCKED_PAGES * PAGESIZE);
3792 	return (err);
3793 }
3794 
3795 /*ARGSUSED*/
3796 static void
3797 drmach_copy_rename_slave(struct regs *rp, drmachid_t id)
3798 {
3799 	drmach_copy_rename_program_t	*prog =
3800 		(drmach_copy_rename_program_t *)id;
3801 	register int			cpuid;
3802 	extern void			drmach_flush();
3803 	extern void			membar_sync_il();
3804 	extern void			drmach_flush_icache();
3805 	on_trap_data_t			otd;
3806 
3807 	cpuid = CPU->cpu_id;
3808 
3809 	if (on_trap(&otd, OT_DATA_EC)) {
3810 		no_trap();
3811 		prog->data->error[cpuid] = FMEM_COPY_ERROR;
3812 		prog->critical->stat[cpuid] = FMEM_LOOP_EXIT;
3813 		drmach_flush_icache();
3814 		membar_sync_il();
3815 		return;
3816 	}
3817 
3818 
3819 	/*
3820 	 * jmp drmach_copy_rename_prog().
3821 	 */
3822 
3823 	drmach_flush(prog->critical, PAGESIZE);
3824 	(void) prog->critical->run(prog, cpuid);
3825 	drmach_flush_icache();
3826 
3827 	no_trap();
3828 
3829 	prog->critical->stat[cpuid] = FMEM_LOOP_EXIT;
3830 
3831 	membar_sync_il();
3832 }
3833 
3834 static void
3835 drmach_swap_pa(drmach_mem_t *s_mem, drmach_mem_t *t_mem)
3836 {
3837 	uint64_t s_base, t_base;
3838 	drmach_board_t *s_board, *t_board;
3839 	struct memlist *ml;
3840 
3841 	s_board = s_mem->dev.bp;
3842 	t_board = t_mem->dev.bp;
3843 	if (s_board == NULL || t_board == NULL) {
3844 		cmn_err(CE_PANIC, "Cannot locate source or target board\n");
3845 		return;
3846 	}
3847 	s_base = s_mem->slice_base;
3848 	t_base = t_mem->slice_base;
3849 
3850 	s_mem->slice_base = t_base;
3851 	s_mem->base_pa = (s_mem->base_pa - s_base) + t_base;
3852 
3853 	for (ml = s_mem->memlist; ml; ml = ml->next) {
3854 		ml->address = ml->address - s_base + t_base;
3855 	}
3856 
3857 	t_mem->slice_base = s_base;
3858 	t_mem->base_pa = (t_mem->base_pa - t_base) + s_base;
3859 
3860 	for (ml = t_mem->memlist; ml; ml = ml->next) {
3861 		ml->address = ml->address - t_base + s_base;
3862 	}
3863 
3864 	/*
3865 	 * IKP has to update the sb-mem-ranges for mac patrol driver
3866 	 * when it resumes, it will re-read the sb-mem-range property
3867 	 * to get the new base address
3868 	 */
3869 	if (oplcfg_pa_swap(s_board->bnum, t_board->bnum) != 0)
3870 		cmn_err(CE_PANIC, "Could not update device nodes\n");
3871 }
3872 
3873 void
3874 drmach_copy_rename(drmachid_t id)
3875 {
3876 	drmach_copy_rename_program_t	*prog_kmem = id;
3877 	drmach_copy_rename_program_t	*prog;
3878 	cpuset_t	cpuset;
3879 	int		cpuid;
3880 	uint64_t	inst;
3881 	register int	rtn;
3882 	extern int	in_sync;
3883 	int		old_in_sync;
3884 	extern void	drmach_sys_trap();
3885 	extern void	drmach_flush();
3886 	extern void	drmach_flush_icache();
3887 	extern uint64_t	patch_inst(uint64_t *, uint64_t);
3888 	on_trap_data_t	otd;
3889 
3890 
3891 	prog = prog_kmem->locked_prog;
3892 
3893 
3894 	/*
3895 	 * We must immediately drop in the TLB because all pointers
3896 	 * are based on the alternate vmem space.
3897 	 */
3898 
3899 	(void) drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);
3900 
3901 	/*
3902 	 * we call scf to get the base address here becuase if scf
3903 	 * has not been suspended yet, the active path can be changing and
3904 	 * sometimes it is not even mapped.  We call the interface when
3905 	 * the OS has been quiesced.
3906 	 */
3907 	prog->critical->scf_reg_base = (*prog->data->scf_get_base_addr)();
3908 
3909 	if (prog->critical->scf_reg_base == (uint64_t)-1 ||
3910 		prog->critical->scf_reg_base == NULL) {
3911 		prog->data->fmem_status.error = FMEM_SCF_ERR;
3912 		drmach_unlock_critical((caddr_t)prog);
3913 		return;
3914 	}
3915 
3916 	cpuset = prog->data->cpu_ready_set;
3917 
3918 	for (cpuid = 0; cpuid < NCPU; cpuid++) {
3919 		if (CPU_IN_SET(cpuset, cpuid)) {
3920 			prog->critical->stat[cpuid] = FMEM_LOOP_START;
3921 			prog->data->error[cpuid] = FMEM_NO_ERROR;
3922 		}
3923 	}
3924 
3925 	old_in_sync = in_sync;
3926 	in_sync = 1;
3927 	cpuid = CPU->cpu_id;
3928 
3929 	CPUSET_DEL(cpuset, cpuid);
3930 
3931 	for (cpuid = 0; cpuid < NCPU; cpuid++) {
3932 		if (CPU_IN_SET(cpuset, cpuid)) {
3933 			xc_one(cpuid, (xcfunc_t *)drmach_lock_critical,
3934 				(uint64_t)prog_kmem, (uint64_t)prog);
3935 		}
3936 	}
3937 
3938 	cpuid = CPU->cpu_id;
3939 
3940 	xt_some(cpuset, (xcfunc_t *)drmach_sys_trap,
3941 				(uint64_t)drmach_copy_rename_slave,
3942 				(uint64_t)prog);
3943 	xt_sync(cpuset);
3944 
3945 	if (on_trap(&otd, OT_DATA_EC)) {
3946 		rtn = FMEM_COPY_ERROR;
3947 		drmach_flush_icache();
3948 		goto done;
3949 	}
3950 
3951 	/*
3952 	 * jmp drmach_copy_rename_prog().
3953 	 */
3954 
3955 	drmach_flush(prog->critical, PAGESIZE);
3956 	rtn = prog->critical->run(prog, cpuid);
3957 	drmach_flush_icache();
3958 
3959 
3960 done:
3961 	no_trap();
3962 	if (rtn == FMEM_HW_ERROR) {
3963 		kpreempt_enable();
3964 		prom_panic("URGENT_ERROR_TRAP is "
3965 			"detected during FMEM.\n");
3966 	}
3967 
3968 	/*
3969 	 * In normal case, all slave CPU's are still spinning in
3970 	 * the assembly code.  The master has to patch the instruction
3971 	 * to get them out.
3972 	 * In error case, e.g. COPY_ERROR, some slave CPU's might
3973 	 * have aborted and already returned and sset LOOP_EXIT status.
3974 	 * Some CPU might still be copying.
3975 	 * In any case, some delay is necessary to give them
3976 	 * enough time to set the LOOP_EXIT status.
3977 	 */
3978 
3979 	for (;;) {
3980 		inst = patch_inst((uint64_t *)prog->critical->loop_rtn,
3981 			prog->critical->inst_loop_ret);
3982 		if (prog->critical->inst_loop_ret == inst) {
3983 			break;
3984 		}
3985 	}
3986 
3987 	for (cpuid = 0; cpuid < NCPU; cpuid++) {
3988 		uint64_t	last, now;
3989 		if (!CPU_IN_SET(cpuset, cpuid)) {
3990 			continue;
3991 		}
3992 		last = prog->stat->nbytes[cpuid];
3993 		/*
3994 		 * Wait for all CPU to exit.
3995 		 * However we do not want an infinite loop
3996 		 * so we detect hangup situation here.
3997 		 * If the slave CPU is still copying data,
3998 		 * we will continue to wait.
3999 		 * In error cases, the master has already set
4000 		 * fmem_status.error to abort the copying.
4001 		 * 1 m.s delay for them to abort copying and
4002 		 * return to drmach_copy_rename_slave to set
4003 		 * FMEM_LOOP_EXIT status should be enough.
4004 		 */
4005 		for (;;) {
4006 			if (prog->critical->stat[cpuid] == FMEM_LOOP_EXIT)
4007 				break;
4008 			drmach_sleep_il();
4009 			drv_usecwait(1000);
4010 			now = prog->stat->nbytes[cpuid];
4011 			if (now <= last) {
4012 			    drv_usecwait(1000);
4013 			    if (prog->critical->stat[cpuid] == FMEM_LOOP_EXIT)
4014 				break;
4015 			    cmn_err(CE_PANIC,
4016 				"CPU %d hang during Copy Rename", cpuid);
4017 			}
4018 			last = now;
4019 		}
4020 		if (prog->data->error[cpuid] == FMEM_HW_ERROR) {
4021 			prom_panic("URGENT_ERROR_TRAP is "
4022 				"detected during FMEM.\n");
4023 		}
4024 	}
4025 
4026 	/*
4027 	 * This must be done after all strands have exit.
4028 	 * Removing the TLB entry will affect both strands
4029 	 * in the same core.
4030 	 */
4031 
4032 	for (cpuid = 0; cpuid < NCPU; cpuid++) {
4033 		if (CPU_IN_SET(cpuset, cpuid)) {
4034 			xc_one(cpuid, (xcfunc_t *)drmach_unlock_critical,
4035 				(uint64_t)prog, 0);
4036 		}
4037 	}
4038 
4039 	in_sync = old_in_sync;
4040 
4041 	/*
4042 	 * we should unlock before the following lock to keep the kpreempt
4043 	 * count correct.
4044 	 */
4045 	(void) drmach_unlock_critical((caddr_t)prog);
4046 
4047 	/*
4048 	 * we must remap again.  TLB might have been removed in above xcall.
4049 	 */
4050 
4051 	(void) drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);
4052 
4053 	if (prog->data->fmem_status.error == 0)
4054 		prog->data->fmem_status.error = rtn;
4055 
4056 	if (prog->data->copy_wait_time > 0) {
4057 		DRMACH_PR("Unexpected long wait time %ld seconds "
4058 			"during copy rename on CPU %d\n",
4059 			prog->data->copy_wait_time/prog->data->stick_freq,
4060 			prog->data->slowest_cpuid);
4061 	}
4062 	drmach_unlock_critical((caddr_t)prog);
4063 }
4064