3118 drmach_copy_rename_prog__relocatable(drmach_copy_rename_program_t *prog,  in drmach_copy_rename_prog__relocatable()  argument
3145 	prog->critical->stat[cpuid] = FMEM_LOOP_COPY_READY;  in drmach_copy_rename_prog__relocatable()
3148 	if (prog->data->cpuid == cpuid) {  in drmach_copy_rename_prog__relocatable()
3152 			if (CPU_IN_SET(prog->data->cpu_slave_set, i)) {  in drmach_copy_rename_prog__relocatable()
3155 					if (prog->critical->stat[i] ==  in drmach_copy_rename_prog__relocatable()
3159 					DR_DELAY_IL(1, prog->data->stick_freq);  in drmach_copy_rename_prog__relocatable()
3163 					prog->data->fmem_status.error =  in drmach_copy_rename_prog__relocatable()
3169 		prog->data->fmem_status.stat = FMEM_LOOP_COPY_READY;  in drmach_copy_rename_prog__relocatable()
3174 			if (prog->data->fmem_status.stat ==  in drmach_copy_rename_prog__relocatable()
3178 			if (prog->data->fmem_status.error) {  in drmach_copy_rename_prog__relocatable()
3179 				prog->data->error[cpuid] = EOPL_FMEM_TERMINATE;  in drmach_copy_rename_prog__relocatable()
3182 			DR_DELAY_IL(1, prog->data->stick_freq);  in drmach_copy_rename_prog__relocatable()
3189 	if (CPU_IN_SET(prog->data->cpu_copy_set, cpuid)) {  in drmach_copy_rename_prog__relocatable()
3190 		for (ml = prog->data->cpu_ml[cpuid]; ml; ml = ml->ml_next) {  in drmach_copy_rename_prog__relocatable()
3194 			s_pa = prog->data->s_copybasepa + ml->ml_address;  in drmach_copy_rename_prog__relocatable()
3195 			t_pa = prog->data->t_copybasepa + ml->ml_address;  in drmach_copy_rename_prog__relocatable()
3203 				if (prog->data->fmem_status.error !=  in drmach_copy_rename_prog__relocatable()
3205 					prog->data->error[cpuid] =  in drmach_copy_rename_prog__relocatable()
3230 				prog->stat->nbytes[cpuid] += 32;  in drmach_copy_rename_prog__relocatable()
3240 		prog->critical->stat[cpuid] = FMEM_LOOP_COPY_DONE;  in drmach_copy_rename_prog__relocatable()
3254 	if (prog->data->cpuid == cpuid) {  in drmach_copy_rename_prog__relocatable()
3257 		limit = copy_start + prog->data->copy_delay;  in drmach_copy_rename_prog__relocatable()
3259 			if (!CPU_IN_SET(prog->data->cpu_slave_set, i))  in drmach_copy_rename_prog__relocatable()
3267 				if (prog->critical->stat[i] ==  in drmach_copy_rename_prog__relocatable()
3272 				if (prog->data->error[i] ==  in drmach_copy_rename_prog__relocatable()
3274 					prog->data->fmem_status.error =  in drmach_copy_rename_prog__relocatable()
3286 				prog->data->slowest_cpuid = i;  in drmach_copy_rename_prog__relocatable()
3287 				prog->data->copy_wait_time = curr - copy_start;  in drmach_copy_rename_prog__relocatable()
3290 				last = prog->stat->nbytes[i];  in drmach_copy_rename_prog__relocatable()
3292 				DR_DELAY_IL(1, prog->data->stick_freq);  in drmach_copy_rename_prog__relocatable()
3294 				now = prog->stat->nbytes[i];  in drmach_copy_rename_prog__relocatable()
3300 					DR_DELAY_IL(1, prog->data->stick_freq);  in drmach_copy_rename_prog__relocatable()
3301 					if (prog->critical->stat[i] ==  in drmach_copy_rename_prog__relocatable()
3305 					if (prog->data->error[i] ==  in drmach_copy_rename_prog__relocatable()
3307 						prog->data-> fmem_status.error =  in drmach_copy_rename_prog__relocatable()
3312 					prog->data->copy_rename_count++;  in drmach_copy_rename_prog__relocatable()
3313 					if (prog->data->copy_rename_count  in drmach_copy_rename_prog__relocatable()
3317 						prog->data->fmem_status.error =  in drmach_copy_rename_prog__relocatable()
3325 		prog->critical->stat[cpuid] = FMEM_LOOP_FMEM_READY;  in drmach_copy_rename_prog__relocatable()
3326 		prog->data->fmem_status.stat  = FMEM_LOOP_FMEM_READY;  in drmach_copy_rename_prog__relocatable()
3329 		flush_instr_mem_il((void*) (prog->critical));  in drmach_copy_rename_prog__relocatable()
3333 		rtn = prog->critical->fmem((void *)prog->critical, PAGESIZE);  in drmach_copy_rename_prog__relocatable()
3336 		flush_instr_mem_il((void*) (prog->critical));  in drmach_copy_rename_prog__relocatable()
3340 		rtn = prog->critical->loop((void *)(prog->critical), PAGESIZE,  in drmach_copy_rename_prog__relocatable()
3341 		    (void *)&(prog->critical->stat[cpuid]));  in drmach_copy_rename_prog__relocatable()
3342 		prog->data->error[cpuid] = rtn;  in drmach_copy_rename_prog__relocatable()
3435 	drmach_copy_rename_program_t *prog = NULL;  in drmach_copy_rename_init()  local
3536 	prog_kmem->prog = prog_kmem;  in drmach_copy_rename_init()
3554 	prog = prog_kmem->locked_prog = vmem_alloc(heap_arena,  in drmach_copy_rename_init()
3556 	wp = bp = (caddr_t)prog;  in drmach_copy_rename_init()
3559 	drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);  in drmach_copy_rename_init()
3562 	prog->data = (drmach_copy_rename_data_t *)roundup(((uint64_t)prog +  in drmach_copy_rename_init()
3565 	ASSERT(((uint64_t)prog->data + sizeof (drmach_copy_rename_data_t))  in drmach_copy_rename_init()
3566 	    <= ((uint64_t)prog + PAGESIZE));  in drmach_copy_rename_init()
3568 	prog->critical = (drmach_copy_rename_critical_t *)  in drmach_copy_rename_init()
3571 	prog->memlist_buffer = (caddr_t)(wp + DRMACH_FMEM_MLIST_PAGE *  in drmach_copy_rename_init()
3574 	prog->stat = (drmach_cr_stat_t *)(wp + DRMACH_FMEM_STAT_PAGE *  in drmach_copy_rename_init()
3581 	prog->critical->scf_reg_base = (uint64_t)-1;  in drmach_copy_rename_init()
3582 	prog->critical->scf_td[0] = (s_bd & 0xff);  in drmach_copy_rename_init()
3583 	prog->critical->scf_td[1] = (t_bd & 0xff);  in drmach_copy_rename_init()
3585 		prog->critical->scf_td[i]   = 0;  in drmach_copy_rename_init()
3587 	prog->critical->scf_td[15] = ((0xaa + s_bd + t_bd) & 0xff);  in drmach_copy_rename_init()
3589 	bp = (caddr_t)prog->critical;  in drmach_copy_rename_init()
3618 	bp = (caddr_t)prog->critical;  in drmach_copy_rename_init()
3622 	prog->critical->run = (int (*)())(wp);  in drmach_copy_rename_init()
3630 	prog->critical->fmem = (int (*)())(wp);  in drmach_copy_rename_init()
3639 	prog->critical->loop = (int (*)())(wp);  in drmach_copy_rename_init()
3645 	prog->critical->loop_rtn = (void (*)()) (wp+len);  in drmach_copy_rename_init()
3647 	prog->data->fmem_status.error = ESBD_NOERROR;  in drmach_copy_rename_init()
3654 	prog->data->scf_fmem_end = scf_fmem_end;  in drmach_copy_rename_init()
3655 	prog->data->scf_fmem_cancel = scf_fmem_cancel;  in drmach_copy_rename_init()
3656 	prog->data->scf_get_base_addr = scf_get_base_addr;  in drmach_copy_rename_init()
3657 	prog->data->fmem_status.op |= OPL_FMEM_SCF_START;  in drmach_copy_rename_init()
3661 	prog->data->fmem_status.op |= OPL_FMEM_MC_SUSPEND;  in drmach_copy_rename_init()
3662 	prog->data->mc_resume = mc_resume;  in drmach_copy_rename_init()
3664 	prog->critical->inst_loop_ret  =  in drmach_copy_rename_init()
3665 	    *(uint64_t *)(prog->critical->loop_rtn);  in drmach_copy_rename_init()
3671 	*(uint_t *)(prog->critical->loop_rtn) = (uint_t)(0x30800000);  in drmach_copy_rename_init()
3676 	prog->critical->delay = fmem_timeout * system_clock_freq;  in drmach_copy_rename_init()
3678 	prog->data->s_mem = (drmachid_t)s_mem;  in drmach_copy_rename_init()
3679 	prog->data->t_mem = (drmachid_t)t_mem;  in drmach_copy_rename_init()
3682 	prog->data->cpuid = cpuid;  in drmach_copy_rename_init()
3683 	prog->data->cpu_ready_set = cpu_ready_set;  in drmach_copy_rename_init()
3684 	prog->data->cpu_slave_set = cpu_ready_set;  in drmach_copy_rename_init()
3685 	prog->data->slowest_cpuid = (processorid_t)-1;  in drmach_copy_rename_init()
3686 	prog->data->copy_wait_time = 0;  in drmach_copy_rename_init()
3687 	prog->data->copy_rename_count = 0;  in drmach_copy_rename_init()
3688 	CPUSET_DEL(prog->data->cpu_slave_set, cpuid);  in drmach_copy_rename_init()
3691 		prog->data->cpu_ml[i] = NULL;  in drmach_copy_rename_init()
3701 	max_elms = drmach_setup_memlist(prog);  in drmach_copy_rename_init()
3710 		CPUSET_ADD(prog->data->cpu_copy_set, cpuid);  in drmach_copy_rename_init()
3738 				    !CPU_IN_SET(prog->data->cpu_copy_set,  in drmach_copy_rename_init()
3740 					CPUSET_ADD(prog->data->cpu_copy_set, i);  in drmach_copy_rename_init()
3771 	prog->data->stick_freq = system_clock_freq;  in drmach_copy_rename_init()
3772 	prog->data->copy_delay = ((copy_sz / min_copy_size_per_sec) + 2) *  in drmach_copy_rename_init()
3780 		prog->stat->nbytes[i] = 0;  in drmach_copy_rename_init()
3781 		if (!CPU_IN_SET(prog->data->cpu_copy_set, i)) {  in drmach_copy_rename_init()
3788 				if ((prog->data->cpu_ml[i] =  in drmach_copy_rename_init()
3789 				    drmach_memlist_add_span(prog,  in drmach_copy_rename_init()
3790 				    prog->data->cpu_ml[i],  in drmach_copy_rename_init()
3805 				if ((prog->data->cpu_ml[i] =  in drmach_copy_rename_init()
3806 				    drmach_memlist_add_span(prog,  in drmach_copy_rename_init()
3807 				    prog->data->cpu_ml[i],  in drmach_copy_rename_init()
3829 	prog->data->s_copybasepa = s_copybasepa;  in drmach_copy_rename_init()
3830 	prog->data->t_copybasepa = t_copybasepa;  in drmach_copy_rename_init()
3831 	prog->data->c_ml = c_ml;  in drmach_copy_rename_init()
3835 	drmach_unlock_critical((caddr_t)prog);  in drmach_copy_rename_init()
3840 	rv = (*prog->data->scf_fmem_cancel)();  in drmach_copy_rename_init()
3845 	if (prog != NULL) {  in drmach_copy_rename_init()
3846 		drmach_unlock_critical((caddr_t)prog);  in drmach_copy_rename_init()
3847 		vmem_free(heap_arena, prog, DRMACH_FMEM_LOCKED_PAGES *  in drmach_copy_rename_init()
3859 	drmach_copy_rename_program_t	*prog = id;  in drmach_copy_rename_fini()  local
3878 	(void) drmach_lock_critical((caddr_t)prog, (caddr_t)prog->locked_prog);  in drmach_copy_rename_fini()
3880 	if (prog->data->c_ml != NULL)  in drmach_copy_rename_fini()
3881 		memlist_delete(prog->data->c_ml);  in drmach_copy_rename_fini()
3883 	if ((prog->data->fmem_status.op &  in drmach_copy_rename_fini()
3887 		    "code %x\n", prog->data->fmem_status.op);  in drmach_copy_rename_fini()
3890 	fmem_error = prog->data->fmem_status.error;  in drmach_copy_rename_fini()
3896 	if (prog->critical->fmem_issued) {  in drmach_copy_rename_fini()
3901 		rv = (*prog->data->scf_fmem_end)();  in drmach_copy_rename_fini()
3909 		drmach_swap_pa((drmach_mem_t *)prog->data->s_mem,  in drmach_copy_rename_fini()
3910 		    (drmach_mem_t *)prog->data->t_mem);  in drmach_copy_rename_fini()
3912 		rv = (*prog->data->scf_fmem_cancel)();  in drmach_copy_rename_fini()
3923 	(*prog->data->mc_resume)();  in drmach_copy_rename_fini()
3925 	drmach_unlock_critical((caddr_t)prog->locked_prog);  in drmach_copy_rename_fini()
3927 	vmem_free(heap_arena, prog->locked_prog,  in drmach_copy_rename_fini()
3929 	kmem_free(prog, DRMACH_FMEM_LOCKED_PAGES * PAGESIZE);  in drmach_copy_rename_fini()
3937 	drmach_copy_rename_program_t	*prog =  in drmach_copy_rename_slave()  local
3949 		prog->data->error[cpuid] = EOPL_FMEM_COPY_ERROR;  in drmach_copy_rename_slave()
3950 		prog->critical->stat[cpuid] = FMEM_LOOP_EXIT;  in drmach_copy_rename_slave()
3961 	drmach_flush(prog->critical, PAGESIZE);  in drmach_copy_rename_slave()
3962 	(void) prog->critical->run(prog, cpuid);  in drmach_copy_rename_slave()
3967 	prog->critical->stat[cpuid] = FMEM_LOOP_EXIT;  in drmach_copy_rename_slave()
4015 	drmach_copy_rename_program_t	*prog;  in drmach_copy_rename()  local
4029 	prog = prog_kmem->locked_prog;  in drmach_copy_rename()
4037 	(void) drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);  in drmach_copy_rename()
4045 	prog->critical->scf_reg_base = (*prog->data->scf_get_base_addr)();  in drmach_copy_rename()
4047 	if (prog->critical->scf_reg_base == (uint64_t)-1 ||  in drmach_copy_rename()
4048 	    prog->critical->scf_reg_base == 0) {  in drmach_copy_rename()
4049 		prog->data->fmem_status.error = EOPL_FMEM_SCF_ERR;  in drmach_copy_rename()
4050 		drmach_unlock_critical((caddr_t)prog);  in drmach_copy_rename()
4054 	cpuset = prog->data->cpu_ready_set;  in drmach_copy_rename()
4058 			prog->critical->stat[cpuid] = FMEM_LOOP_START;  in drmach_copy_rename()
4059 			prog->data->error[cpuid] = ESBD_NOERROR;  in drmach_copy_rename()
4072 			    (uint64_t)prog_kmem, (uint64_t)prog);  in drmach_copy_rename()
4079 	    (uint64_t)drmach_copy_rename_slave, (uint64_t)prog);  in drmach_copy_rename()
4092 	drmach_flush(prog->critical, PAGESIZE);  in drmach_copy_rename()
4093 	rtn = prog->critical->run(prog, cpuid);  in drmach_copy_rename()
4117 		inst = patch_inst((uint64_t *)prog->critical->loop_rtn,  in drmach_copy_rename()
4118 		    prog->critical->inst_loop_ret);  in drmach_copy_rename()
4119 		if (prog->critical->inst_loop_ret == inst) {  in drmach_copy_rename()
4129 		last = prog->stat->nbytes[cpuid];  in drmach_copy_rename()
4143 			if (prog->critical->stat[cpuid] == FMEM_LOOP_EXIT)  in drmach_copy_rename()
4147 			now = prog->stat->nbytes[cpuid];  in drmach_copy_rename()
4150 				if (prog->critical->stat[cpuid] ==  in drmach_copy_rename()
4158 		if (prog->data->error[cpuid] == EOPL_FMEM_HW_ERROR) {  in drmach_copy_rename()
4173 			    (uint64_t)prog, 0);  in drmach_copy_rename()
4183 	(void) drmach_unlock_critical((caddr_t)prog);  in drmach_copy_rename()
4189 	(void) drmach_lock_critical((caddr_t)prog_kmem, (caddr_t)prog);  in drmach_copy_rename()
4191 	if (prog->data->fmem_status.error == ESBD_NOERROR)  in drmach_copy_rename()
4192 		prog->data->fmem_status.error = rtn;  in drmach_copy_rename()
4194 	if (prog->data->copy_wait_time > 0) {  in drmach_copy_rename()
4197 		    prog->data->copy_wait_time/prog->data->stick_freq,  in drmach_copy_rename()
4198 		    prog->data->slowest_cpuid);  in drmach_copy_rename()
4200 	drmach_unlock_critical((caddr_t)prog);  in drmach_copy_rename()