1 // SPDX-License-Identifier: GPL-2.0
40 #include <asm/access-regs.h>
41 #include <asm/asm-offsets.h>
99 static unsigned int smp_max_threads __initdata = -1U;
135 		cc = __pcpu_sigp(pcpu->address, order, parm, NULL);  in pcpu_sigp_retry()
148 	if (__pcpu_sigp(pcpu->address, SIGP_SENSE,  in pcpu_stopped()
156 	if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,  in pcpu_running()
178 	if (test_and_set_bit(ec_bit, &pcpu->ec_mask))  in pcpu_ec_call()
180 	pcpu->ec_clk = get_tod_clock_fast();  in pcpu_ec_call()
196 	memset((char *) lc + 512, 0, sizeof(*lc) - 512);  in pcpu_alloc_lowcore()
197 	lc->async_stack = async_stack + STACK_INIT_OFFSET;  in pcpu_alloc_lowcore()
198 	lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;  in pcpu_alloc_lowcore()
199 	lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET;  in pcpu_alloc_lowcore()
200 	lc->cpu_nr = cpu;  in pcpu_alloc_lowcore()
201 	lc->spinlock_lockval = arch_spin_lockval(cpu);  in pcpu_alloc_lowcore()
202 	lc->spinlock_index = 0;  in pcpu_alloc_lowcore()
203 	lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);  in pcpu_alloc_lowcore()
204 	lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);  in pcpu_alloc_lowcore()
205 	lc->preempt_count = PREEMPT_DISABLED;  in pcpu_alloc_lowcore()
206 	if (nmi_alloc_mcesa(&lc->mcesad))  in pcpu_alloc_lowcore()
215 	nmi_free_mcesa(&lc->mcesad);  in pcpu_alloc_lowcore()
221 	return -ENOMEM;  in pcpu_alloc_lowcore()
230 	nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET;  in pcpu_free_lowcore()
231 	async_stack = lc->async_stack - STACK_INIT_OFFSET;  in pcpu_free_lowcore()
232 	mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET;  in pcpu_free_lowcore()
236 	nmi_free_mcesa(&lc->mcesad);  in pcpu_free_lowcore()
250 	lc->cpu_nr = cpu;  in pcpu_prepare_secondary()
251 	lc->pcpu = (unsigned long)pcpu;  in pcpu_prepare_secondary()
252 	lc->restart_flags = RESTART_FLAG_CTLREGS;  in pcpu_prepare_secondary()
253 	lc->spinlock_lockval = arch_spin_lockval(cpu);  in pcpu_prepare_secondary()
254 	lc->spinlock_index = 0;  in pcpu_prepare_secondary()
255 	lc->percpu_offset = __per_cpu_offset[cpu];  in pcpu_prepare_secondary()
256 	lc->kernel_asce = get_lowcore()->kernel_asce;  in pcpu_prepare_secondary()
257 	lc->user_asce = s390_invalid_asce;  in pcpu_prepare_secondary()
258 	lc->user_timer = lc->system_timer =  in pcpu_prepare_secondary()
259 		lc->steal_timer = lc->avg_steal_timer = 0;  in pcpu_prepare_secondary()
261 	memcpy(lc->cregs_save_area, abs_lc->cregs_save_area, sizeof(lc->cregs_save_area));  in pcpu_prepare_secondary()
263 	lc->cregs_save_area[1] = lc->user_asce;  in pcpu_prepare_secondary()
264 	lc->cregs_save_area[7] = lc->user_asce;  in pcpu_prepare_secondary()
265 	save_access_regs((unsigned int *) lc->access_regs_save_area);  in pcpu_prepare_secondary()
274 	lc->kernel_stack = (unsigned long)task_stack_page(tsk) + STACK_INIT_OFFSET;  in pcpu_attach_task()
275 	lc->current_task = (unsigned long)tsk;  in pcpu_attach_task()
276 	lc->lpp = LPP_MAGIC;  in pcpu_attach_task()
277 	lc->current_pid = tsk->pid;  in pcpu_attach_task()
278 	lc->user_timer = tsk->thread.user_timer;  in pcpu_attach_task()
279 	lc->guest_timer = tsk->thread.guest_timer;  in pcpu_attach_task()
280 	lc->system_timer = tsk->thread.system_timer;  in pcpu_attach_task()
281 	lc->hardirq_timer = tsk->thread.hardirq_timer;  in pcpu_attach_task()
282 	lc->softirq_timer = tsk->thread.softirq_timer;  in pcpu_attach_task()
283 	lc->steal_timer = 0;  in pcpu_attach_task()
291 	lc->restart_stack = lc->kernel_stack;  in pcpu_start_fn()
292 	lc->restart_fn = (unsigned long) func;  in pcpu_start_fn()
293 	lc->restart_data = (unsigned long) data;  in pcpu_start_fn()
294 	lc->restart_source = -1U;  in pcpu_start_fn()
318 	if (pcpu->address == source_cpu) {  in pcpu_delegate()
327 		lc->restart_stack = stack;  in pcpu_delegate()
328 		lc->restart_fn = (unsigned long)func;  in pcpu_delegate()
329 		lc->restart_data = (unsigned long)data;  in pcpu_delegate()
330 		lc->restart_source = source_cpu;  in pcpu_delegate()
333 		abs_lc->restart_stack = stack;  in pcpu_delegate()
334 		abs_lc->restart_fn = (unsigned long)func;  in pcpu_delegate()
335 		abs_lc->restart_data = (unsigned long)data;  in pcpu_delegate()
336 		abs_lc->restart_source = source_cpu;  in pcpu_delegate()
344 		: : "d" (pcpu->address), "d" (source_cpu),  in pcpu_delegate()
351  * Enable additional logical cpus for multi-threading.
377 	if (ipl_pcpu->address == stap())  in smp_call_ipl_cpu()
380 	pcpu_delegate(ipl_pcpu, 0, func, data, lc->nodat_stack);  in smp_call_ipl_cpu()
390 	return -1;  in smp_find_processor_id()
436 		set_bit(ec_stop_cpu, &pcpu->ec_mask);  in smp_emergency_stop()
437 		while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,  in smp_emergency_stop()
551 	pa = __pa(&lc->floating_pt_save_area);  in smp_store_status()
552 	if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,  in smp_store_status()
554 		return -EIO;  in smp_store_status()
557 	pa = lc->mcesad & MCESA_ORIGIN_MASK;  in smp_store_status()
559 		pa |= lc->mcesad & MCESA_LC_MASK;  in smp_store_status()
560 	if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,  in smp_store_status()
562 		return -EIO;  in smp_store_status()
572  *    with sigp stop-and-store-status. The boot CPU state is located in
575  * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory)
578  *    with sigp stop-and-store-status. The firmware or the boot-loader
581  * 3) kdump or stand-alone kdump for DASD
584  *    with sigp stop-and-store-status. The kexec code or the boot-loader
627 	/* Set multi-threading state to the previous system. */  in smp_save_dump_secondary_cpus()
698 			info->core[info->configured].core_id =  in smp_get_core_info()
700 			info->configured++;  in smp_get_core_info()
702 		info->combined = info->configured;  in smp_get_core_info()
714 	if (sclp.has_core_type && core->type != boot_core_type)  in smp_add_core()
717 	address = core->core_id << smp_cpu_mt_shift;  in smp_add_core()
722 		pcpu->address = address + i;  in smp_add_core()
724 			pcpu->state = CPU_STATE_CONFIGURED;  in smp_add_core()
726 			pcpu->state = CPU_STATE_STANDBY;  in smp_add_core()
758 		for (i = 0; i < info->configured; i++) {  in __smp_rescan_cpus()
759 			core = &info->core[i];  in __smp_rescan_cpus()
760 			if (core->core_id == core_id) {  in __smp_rescan_cpus()
766 	for (i = 0; i < info->combined; i++) {  in __smp_rescan_cpus()
767 		configured = i < info->configured;  in __smp_rescan_cpus()
768 		nr += smp_add_core(&info->core[i], &avail, configured, early);  in __smp_rescan_cpus()
787 		for (cpu = 0; cpu < info->combined; cpu++)  in smp_detect_cpus()
788 			if (info->core[cpu].core_id == address) {  in smp_detect_cpus()
790 				boot_core_type = info->core[cpu].type;  in smp_detect_cpus()
793 		if (cpu >= info->combined)  in smp_detect_cpus()
797 	/* Set multi-threading state for the current system */  in smp_detect_cpus()
799 	mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;  in smp_detect_cpus()
805 	for (cpu = 0; cpu < info->combined; cpu++) {  in smp_detect_cpus()
807 		    info->core[cpu].type != boot_core_type)  in smp_detect_cpus()
809 		if (cpu < info->configured)  in smp_detect_cpus()
826 	lc->last_update_clock = get_tod_clock();  in smp_start_secondary()
827 	lc->restart_stack = (unsigned long)restart_stack;  in smp_start_secondary()
828 	lc->restart_fn = (unsigned long)do_restart;  in smp_start_secondary()
829 	lc->restart_data = 0;  in smp_start_secondary()
830 	lc->restart_source = -1U;  in smp_start_secondary()
831 	lc->restart_flags = 0;  in smp_start_secondary()
832 	restore_access_regs(lc->access_regs_save_area);  in smp_start_secondary()
858 	if (pcpu->state != CPU_STATE_CONFIGURED)  in __cpu_up()
859 		return -EIO;  in __cpu_up()
862 		return -EIO;  in __cpu_up()
925 	pcpu->flags = 0;  in __cpu_die()
964 	lc->percpu_offset = __per_cpu_offset[0];  in smp_prepare_boot_cpu()
966 	ipl_pcpu->state = CPU_STATE_CONFIGURED;  in smp_prepare_boot_cpu()
967 	lc->pcpu = (unsigned long)ipl_pcpu;  in smp_prepare_boot_cpu()
976 	lc->cpu_nr = 0;  in smp_setup_processor_id()
978 	lc->spinlock_lockval = arch_spin_lockval(0);  in smp_setup_processor_id()
979 	lc->spinlock_index = 0;  in smp_setup_processor_id()
994 				  struct device_attribute *attr, char *buf)  in cpu_configure_show()  argument
999 	count = sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).state);  in cpu_configure_show()
1005 				   struct device_attribute *attr,  in cpu_configure_store()  argument
1013 		return -EINVAL;  in cpu_configure_store()
1015 		return -EINVAL;  in cpu_configure_store()
1018 	rc = -EBUSY;  in cpu_configure_store()
1020 	cpu = dev->id;  in cpu_configure_store()
1029 		if (pcpu->state != CPU_STATE_CONFIGURED)  in cpu_configure_store()
1031 		rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);  in cpu_configure_store()
1044 		if (pcpu->state != CPU_STATE_STANDBY)  in cpu_configure_store()
1046 		rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);  in cpu_configure_store()
1069 				struct device_attribute *attr, char *buf)  in show_cpu_address()  argument
1071 	return sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).address);  in show_cpu_address()
1076 	&dev_attr_configure.attr,
1077 	&dev_attr_address.attr,
1086 	&dev_attr_idle_count.attr,
1087 	&dev_attr_idle_time_us.attr,
1099 	return sysfs_create_group(&c->dev.kobj, &cpu_online_attr_group);  in smp_cpu_online()
1106 	sysfs_remove_group(&c->dev.kobj, &cpu_online_attr_group);  in smp_cpu_pre_down()
1120 	c->hotpluggable = arch_cpu_is_hotpluggable(cpu);  in arch_register_cpu()
1124 	rc = sysfs_create_group(&c->dev.kobj, &cpu_common_attr_group);  in arch_register_cpu()
1133 	sysfs_remove_group(&c->dev.kobj, &cpu_common_attr_group);  in arch_register_cpu()
1147 		return -ENOMEM;  in smp_rescan_cpus()
1157 				  struct device_attribute *attr,  in rescan_store()  argument