Lines Matching +full:current +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
64 * been used to perform kernel mode NEON in the meantime.
67 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
68 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
70 * the most recently, or NULL if kernel mode NEON has been performed after that.
79 * indicate whether or not the userland FPSIMD state of the current task is
81 * CPU currently contain the most recent userland FPSIMD state of the current
84 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
97 * - the task gets scheduled in; if both the task's fpsimd_cpu field
98 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
102 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
104 * fpsimd_cpu field is set to the id of the current CPU, the current
108 * - the task executes an ordinary syscall; upon return to userland, the
112 * - the task executes a syscall which executes some NEON instructions; this is
114 * register contents to memory, clears the fpsimd_last_state per-cpu variable
117 * - the task gets preempted after kernel_neon_end() is called; as we have not
251 return task->thread.vl[type]; in task_get_vl()
257 task->thread.vl[type] = vl; in task_set_vl()
263 return task->thread.vl_onexec[type]; in task_get_vl_onexec()
269 task->thread.vl_onexec[type] = vl; in task_set_vl_onexec()
276 * and the SVE registers for both streaming and non-streaming modes.
312 * * FPSIMD only - FP_STATE_FPSIMD:
314 * When the FPSIMD only state stored task->thread.fp_type is set to
315 * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
316 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
317 * logically zero but not stored anywhere; P0-P15 and FFR are not
322 * task->thread.sve_state does not need to be non-NULL, valid or any
326 * * SVE state - FP_STATE_SVE:
328 * When the full SVE state is stored task->thread.fp_type is set to
329 * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
330 * corresponding Zn), P0-P15 and FFR are encoded in in
331 * task->thread.sve_state, formatted appropriately for vector
332 * length task->thread.sve_vl or, if SVCR.SM is set,
333 * task->thread.sme_vl. The storage for the vector registers in
334 * task->thread.uw.fpsimd_state should be ignored.
336 * task->thread.sve_state must point to a valid buffer at least
338 * task->thread.uw.fpsimd_state.vregs should be considered stale
341 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
347 * Update current's FPSIMD/SVE registers from thread_struct.
363 switch (current->thread.fp_type) { in task_fpsimd_load()
369 if (!thread_sm_enabled(¤t->thread)) in task_fpsimd_load()
373 sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); in task_fpsimd_load()
395 unsigned long sme_vl = task_get_sme_vl(current); in task_fpsimd_load()
399 sme_set_vq(sve_vq_from_vl(sme_vl) - 1); in task_fpsimd_load()
401 write_sysreg_s(current->thread.svcr, SYS_SVCR); in task_fpsimd_load()
403 if (thread_za_enabled(¤t->thread)) in task_fpsimd_load()
404 sme_load_state(current->thread.sme_state, in task_fpsimd_load()
407 if (thread_sm_enabled(¤t->thread)) in task_fpsimd_load()
412 write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); in task_fpsimd_load()
415 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); in task_fpsimd_load()
416 sve_load_state(sve_pffr(¤t->thread), in task_fpsimd_load()
417 ¤t->thread.uw.fpsimd_state.fpsr, in task_fpsimd_load()
420 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); in task_fpsimd_load()
421 fpsimd_load_state(¤t->thread.uw.fpsimd_state); in task_fpsimd_load()
428 * current context is the context last bound to the CPU stored in
430 * than the host thread for the VM pointed to by current. This means
432 * than via current, if we are saving KVM state then it will have
433 * ensured that the type of registers to save is set in last->to_save.
451 *(last->fpmr) = read_sysreg_s(SYS_FPMR); in fpsimd_save_user_state()
462 if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) || in fpsimd_save_user_state()
463 last->to_save == FP_STATE_SVE) { in fpsimd_save_user_state()
466 vl = last->sve_vl; in fpsimd_save_user_state()
470 u64 *svcr = last->svcr; in fpsimd_save_user_state()
475 sme_save_state(last->sme_state, in fpsimd_save_user_state()
478 /* If we are in streaming mode override regular SVE. */ in fpsimd_save_user_state()
482 vl = last->sme_vl; in fpsimd_save_user_state()
490 * Can't save the user regs, so current would in fpsimd_save_user_state()
491 * re-enter user with corrupt state. in fpsimd_save_user_state()
498 sve_save_state((char *)last->sve_state + in fpsimd_save_user_state()
500 &last->st->fpsr, save_ffr); in fpsimd_save_user_state()
501 *last->fp_type = FP_STATE_SVE; in fpsimd_save_user_state()
503 fpsimd_save_state(last->st); in fpsimd_save_user_state()
504 *last->fp_type = FP_STATE_FPSIMD; in fpsimd_save_user_state()
510 * We're on a slow path, so some sanity-checks are included.
519 int max_vl = info->max_vl; in find_supported_vector_length()
522 vl = info->min_vl; in find_supported_vector_length()
525 max_vl = info->min_vl; in find_supported_vector_length()
529 if (vl < info->min_vl) in find_supported_vector_length()
530 vl = info->min_vl; in find_supported_vector_length()
532 bit = find_next_bit(info->vq_map, SVE_VQ_MAX, in find_supported_vector_length()
542 struct vl_info *info = table->extra1; in vec_proc_do_default_vl()
543 enum vec_type type = info->type; in vec_proc_do_default_vl()
555 /* Writing -1 has the special meaning "set to max": */ in vec_proc_do_default_vl()
556 if (vl == -1) in vec_proc_do_default_vl()
557 vl = info->max_vl; in vec_proc_do_default_vl()
560 return -EINVAL; in vec_proc_do_default_vl()
569 .mode = 0644,
579 return -EINVAL; in sve_sysctl_init()
592 .mode = 0644,
602 return -EINVAL; in sme_sysctl_init()
612 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
639 *p = arm64_cpu_to_le128(fst->vregs[i]); in __fpsimd_to_sve()
644 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
645 * task->thread.sve_state.
647 * Task can be a non-runnable task, or current. In the latter case,
650 * task->thread.sve_state must point to at least sve_state_size(task)
652 * task->thread.uw.fpsimd_state must be up to date before calling this
658 void *sst = task->thread.sve_state; in fpsimd_to_sve()
659 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; in fpsimd_to_sve()
664 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); in fpsimd_to_sve()
669 * Transfer the SVE state in task->thread.sve_state to
670 * task->thread.uw.fpsimd_state.
672 * Task can be a non-runnable task, or current. In the latter case,
675 * task->thread.sve_state must point to at least sve_state_size(task)
677 * task->thread.sve_state must be up to date before calling this function.
682 void const *sst = task->thread.sve_state; in sve_to_fpsimd()
683 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; in sve_to_fpsimd()
690 vl = thread_get_cur_vl(&task->thread); in sve_to_fpsimd()
694 fst->vregs[i] = arm64_le128_to_cpu(*p); in sve_to_fpsimd()
700 memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs)); in __fpsimd_zero_vregs()
708 if (!thread_sm_enabled(&task->thread)) in task_smstop_sm()
711 __fpsimd_zero_vregs(&task->thread.uw.fpsimd_state); in task_smstop_sm()
712 task->thread.uw.fpsimd_state.fpsr = 0x0800009f; in task_smstop_sm()
714 task->thread.uw.fpmr = 0; in task_smstop_sm()
716 task->thread.svcr &= ~SVCR_SM_MASK; in task_smstop_sm()
717 task->thread.fp_type = FP_STATE_FPSIMD; in task_smstop_sm()
729 kfree(task->thread.sve_state); in sve_free()
730 task->thread.sve_state = NULL; in sve_free()
734 * Ensure that task->thread.sve_state is allocated and sufficiently large.
737 * task->thread.sve_state with new data. The memory is always zeroed
745 if (task->thread.sve_state) { in sve_alloc()
747 memset(task->thread.sve_state, 0, in sve_alloc()
753 task->thread.sve_state = in sve_alloc()
758 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the
766 if (task->thread.fp_type == FP_STATE_SVE) in fpsimd_sync_from_effective_state()
772 * with respect to task->thread.uw.fpsimd_state, zeroing any effective
773 * non-FPSIMD (S)SVE state.
781 void *sst = task->thread.sve_state; in fpsimd_sync_to_effective_state_zeropad()
782 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; in fpsimd_sync_to_effective_state_zeropad()
784 if (task->thread.fp_type != FP_STATE_SVE) in fpsimd_sync_to_effective_state_zeropad()
787 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); in fpsimd_sync_to_effective_state_zeropad()
826 if (task == current) in change_live_vector_length()
837 kfree(task->thread.sve_state); in change_live_vector_length()
838 task->thread.sve_state = sve_state; in change_live_vector_length()
842 task->thread.svcr &= ~SVCR_ZA_MASK; in change_live_vector_length()
843 kfree(task->thread.sme_state); in change_live_vector_length()
844 task->thread.sme_state = sme_state; in change_live_vector_length()
852 return -ENOMEM; in change_live_vector_length()
863 return -EINVAL; in vec_set_vector_length()
866 return -EINVAL; in vec_set_vector_length()
869 * Clamp to the maximum vector length that VL-agnostic code in vec_set_vector_length()
881 return -ENOMEM; in vec_set_vector_length()
897 * Encode the current vector length and flags for return.
908 ret = task_get_vl_onexec(current, type); in vec_prctl_status()
910 ret = task_get_vl(current, type); in vec_prctl_status()
928 return -EINVAL; in sve_set_current_vl()
930 ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags); in sve_set_current_vl()
941 return -EINVAL; in sve_get_current_vl()
957 return -EINVAL; in sme_set_current_vl()
959 ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags); in sme_set_current_vl()
970 return -EINVAL; in sme_get_current_vl()
983 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { in vec_probe_vqs()
984 write_vl(info->type, vq - 1); /* self-syncing */ in vec_probe_vqs()
986 switch (info->type) { in vec_probe_vqs()
1014 vec_probe_vqs(info, info->vq_map); in vec_init_vq_map()
1015 bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX); in vec_init_vq_map()
1020 * those not supported by the current CPU.
1021 * This function is called during the bring-up of early secondary CPUs only.
1029 bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX); in vec_update_vq_map()
1030 bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map, in vec_update_vq_map()
1035 * Check whether the current CPU supports all VQs in the committed set.
1036 * This function is called during the bring-up of late secondary CPUs only.
1047 if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) { in vec_verify_vq_map()
1049 info->name, smp_processor_id()); in vec_verify_vq_map()
1050 return -EINVAL; in vec_verify_vq_map()
1065 bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX); in vec_verify_vq_map()
1076 if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) { in vec_verify_vq_map()
1078 info->name, smp_processor_id()); in vec_verify_vq_map()
1079 return -EINVAL; in vec_verify_vq_map()
1134 * The SVE architecture mandates support for 128-bit vectors, in sve_setup()
1138 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map))) in sve_setup()
1139 set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map); in sve_setup()
1141 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); in sve_setup()
1142 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); in sve_setup()
1150 bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map, in sve_setup()
1155 /* No non-virtualisable VLs found */ in sve_setup()
1156 info->max_virtualisable_vl = SVE_VQ_MAX; in sve_setup()
1157 else if (WARN_ON(b == SVE_VQ_MAX - 1)) in sve_setup()
1159 info->max_virtualisable_vl = SVE_VQ_MIN; in sve_setup()
1161 info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); in sve_setup()
1163 if (info->max_virtualisable_vl > info->max_vl) in sve_setup()
1164 info->max_virtualisable_vl = info->max_vl; in sve_setup()
1167 info->name, info->max_vl); in sve_setup()
1169 info->name, get_sve_default_vl()); in sve_setup()
1174 info->name); in sve_setup()
1194 * Ensure that task->thread.sme_state is allocated and sufficiently large.
1197 * task->thread.sme_state with new data. The memory is always zeroed
1204 if (task->thread.sme_state) { in sme_alloc()
1206 memset(task->thread.sme_state, 0, in sme_alloc()
1212 task->thread.sme_state = in sme_alloc()
1218 kfree(task->thread.sme_state); in sme_free()
1219 task->thread.sme_state = NULL; in sme_free()
1268 min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX); in sme_setup()
1279 info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit)); in sme_setup()
1281 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); in sme_setup()
1282 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); in sme_setup()
1284 WARN_ON(info->min_vl > info->max_vl); in sme_setup()
1289 * grow the signal frame when in streaming mode, otherwise the in sme_setup()
1295 info->min_vl); in sme_setup()
1297 info->max_vl); in sme_setup()
1324 * is not shared with FPSIMD. If (as is likely) the current in sve_init_regs()
1326 * update our metadata for the current task including in sve_init_regs()
1327 * disabling the trap, otherwise update our in-memory copy. in sve_init_regs()
1328 * We are guaranteed to not be in streaming mode, we can only in sve_init_regs()
1329 * take a SVE trap when not in streaming mode and we can't be in sve_init_regs()
1330 * in streaming mode when taking a SME trap. in sve_init_regs()
1334 sve_vq_from_vl(task_get_sve_vl(current)) - 1; in sve_init_regs()
1339 fpsimd_to_sve(current); in sve_init_regs()
1340 current->thread.fp_type = FP_STATE_SVE; in sve_init_regs()
1341 fpsimd_flush_task_state(current); in sve_init_regs()
1348 * Storage is allocated for the full SVE state, the current FPSIMD
1360 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sve_acc()
1364 sve_alloc(current, true); in do_sve_acc()
1365 if (!current->thread.sve_state) { in do_sve_acc()
1376 * Even if the task can have used streaming mode we can only in do_sve_acc()
1377 * generate SVE access traps in normal SVE mode and in do_sve_acc()
1378 * transitioning out of streaming mode may discard any in do_sve_acc()
1379 * streaming mode state. Always clear the high bits to avoid in do_sve_acc()
1390 * Storage is allocated for the full SVE and SME state, the current
1402 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sme_acc()
1408 * is being used in the wrong mode, report as SIGILL. in do_sme_acc()
1411 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sme_acc()
1415 sve_alloc(current, false); in do_sme_acc()
1416 sme_alloc(current, true); in do_sme_acc()
1417 if (!current->thread.sve_state || !current->thread.sme_state) { in do_sme_acc()
1430 sve_vq_from_vl(task_get_sme_vl(current)) - 1; in do_sme_acc()
1435 fpsimd_flush_task_state(current); in do_sme_acc()
1448 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_fpsimd_acc()
1460 * Raise a SIGFPE for the current process.
1481 current); in do_fpsimd_exc()
1489 * Elide the load if this CPU holds the most recent kernel mode in fpsimd_load_kernel_state()
1490 * FPSIMD context of the current task. in fpsimd_load_kernel_state()
1492 if (last->st == &task->thread.kernel_fpsimd_state && in fpsimd_load_kernel_state()
1493 task->thread.kernel_fpsimd_cpu == smp_processor_id()) in fpsimd_load_kernel_state()
1496 fpsimd_load_state(&task->thread.kernel_fpsimd_state); in fpsimd_load_kernel_state()
1502 .st = &task->thread.kernel_fpsimd_state, in fpsimd_save_kernel_state()
1506 fpsimd_save_state(&task->thread.kernel_fpsimd_state); in fpsimd_save_kernel_state()
1509 task->thread.kernel_fpsimd_cpu = smp_processor_id(); in fpsimd_save_kernel_state()
1523 * Leaving streaming mode enabled will cause issues for any kernel in fpsimd_flush_cpu_state()
1524 * NEON and leaving streaming mode or ZA enabled may increase power in fpsimd_flush_cpu_state()
1544 fpsimd_save_kernel_state(current); in fpsimd_thread_switch()
1555 * loaded with user mode FPSIMD state and so wrong_task and in fpsimd_thread_switch()
1559 &next->thread.uw.fpsimd_state; in fpsimd_thread_switch()
1560 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); in fpsimd_thread_switch()
1581 vl = task_get_vl_onexec(current, type); in fpsimd_flush_thread_vl()
1592 task_set_vl(current, type, vl); in fpsimd_flush_thread_vl()
1599 task_set_vl_onexec(current, type, 0); in fpsimd_flush_thread_vl()
1612 fpsimd_flush_task_state(current); in fpsimd_flush_thread()
1613 memset(¤t->thread.uw.fpsimd_state, 0, in fpsimd_flush_thread()
1614 sizeof(current->thread.uw.fpsimd_state)); in fpsimd_flush_thread()
1620 sve_state = current->thread.sve_state; in fpsimd_flush_thread()
1621 current->thread.sve_state = NULL; in fpsimd_flush_thread()
1630 sme_state = current->thread.sme_state; in fpsimd_flush_thread()
1631 current->thread.sme_state = NULL; in fpsimd_flush_thread()
1634 current->thread.svcr = 0; in fpsimd_flush_thread()
1638 current->thread.uw.fpmr = 0; in fpsimd_flush_thread()
1640 current->thread.fp_type = FP_STATE_FPSIMD; in fpsimd_flush_thread()
1648 * Save the userland FPSIMD state of 'current' to memory, but only if the state
1649 * currently held in the registers does in fact belong to 'current'
1662 * Associate current's FPSIMD context with this cpu
1671 last->st = ¤t->thread.uw.fpsimd_state; in fpsimd_bind_task_to_cpu()
1672 last->sve_state = current->thread.sve_state; in fpsimd_bind_task_to_cpu()
1673 last->sme_state = current->thread.sme_state; in fpsimd_bind_task_to_cpu()
1674 last->sve_vl = task_get_sve_vl(current); in fpsimd_bind_task_to_cpu()
1675 last->sme_vl = task_get_sme_vl(current); in fpsimd_bind_task_to_cpu()
1676 last->svcr = ¤t->thread.svcr; in fpsimd_bind_task_to_cpu()
1677 last->fpmr = ¤t->thread.uw.fpmr; in fpsimd_bind_task_to_cpu()
1678 last->fp_type = ¤t->thread.fp_type; in fpsimd_bind_task_to_cpu()
1679 last->to_save = FP_STATE_CURRENT; in fpsimd_bind_task_to_cpu()
1680 current->thread.fpsimd_cpu = smp_processor_id(); in fpsimd_bind_task_to_cpu()
1712 * Load the userland FPSIMD state of 'current' from memory, but only if the
1714 * state of 'current'. This is called when we are preparing to return to
1726 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(), in fpsimd_restore_current_state()
1758 current->thread.uw.fpsimd_state = *state; in fpsimd_update_current_state()
1759 if (current->thread.fp_type == FP_STATE_SVE) in fpsimd_update_current_state()
1760 fpsimd_to_sve(current); in fpsimd_update_current_state()
1769 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1776 t->thread.fpsimd_cpu = NR_CPUS; in fpsimd_flush_task_state()
1797 fpsimd_flush_task_state(current); in fpsimd_save_and_flush_current_state()
1821 * Kernel-side NEON support functions
1849 fpsimd_save_kernel_state(current); in kernel_neon_begin()
1854 * Set the thread flag so that the kernel mode FPSIMD state in kernel_neon_begin()
1858 * On non-PREEMPT_RT, softirqs may interrupt task level kernel in kernel_neon_begin()
1859 * mode FPSIMD, but the task will not be preemptible so setting in kernel_neon_begin()
1867 * mode in task context. So in this case, setting the flag here in kernel_neon_begin()
1882 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1896 * If we are returning from a nested use of kernel mode FPSIMD, restore in kernel_neon_end()
1897 * the task context kernel mode FPSIMD state. This can only happen when in kernel_neon_end()
1898 * running in softirq context on non-PREEMPT_RT. in kernel_neon_end()
1902 fpsimd_load_kernel_state(current); in kernel_neon_end()
1925 * Do not use them for any other purpose -- if tempted to do so, you are
1959 * exist in streaming mode. in __efi_fpsimd_begin()
1995 * Restore streaming mode; EFI calls are in __efi_fpsimd_end()
1997 * streaming mode. in __efi_fpsimd_end()
2007 * exist in streaming mode. in __efi_fpsimd_end()
2093 pr_notice("Floating-point is not implemented\n"); in fpsimd_init()