50 static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)  in create_rstor_token()  argument
55 	if (!IS_ALIGNED(ssp, 8))  in create_rstor_token()
58 	addr = ssp - SS_FRAME_SIZE;  in create_rstor_token()
61 	 * SSP is aligned, so reserved bits and mode bit are a zero, just mark  in create_rstor_token()
64 	ssp |= BIT(0);  in create_rstor_token()
66 	if (write_user_shstk_64((u64 __user *)addr, (u64)ssp))  in create_rstor_token()
79  * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The
86  * and INCSSP. In addition to modifying SSP, INCSSP also reads from the
90  * READ_ONCE(ssp);       // read+discard top element on stack
91  * ssp += nr_to_pop * 8; // move the shadow stack
92  * READ_ONCE(ssp-8);     // read+discard last popped stack element
94  * The maximum distance INCSSP can move the SSP is 2040 bytes, before
96  * to prevent any operation from shifting the SSP to an adjacent stack,
238 	unsigned long long ssp;  in get_user_shstk_addr()  local
242 	rdmsrq(MSR_IA32_PL3_SSP, ssp);  in get_user_shstk_addr()
246 	return ssp;  in get_user_shstk_addr()
252 	u64 ssp;  in shstk_pop()  local
259 	rdmsrq(MSR_IA32_PL3_SSP, ssp);  in shstk_pop()
260 	if (val && get_user(*val, (__user u64 *)ssp))  in shstk_pop()
263 		wrmsrq(MSR_IA32_PL3_SSP, ssp + SS_FRAME_SIZE);  in shstk_pop()
271 	u64 ssp;  in shstk_push()  local
279 	rdmsrq(MSR_IA32_PL3_SSP, ssp);  in shstk_push()
280 	ssp -= SS_FRAME_SIZE;  in shstk_push()
281 	ret = write_user_shstk_64((__user void *)ssp, val);  in shstk_push()
283 		wrmsrq(MSR_IA32_PL3_SSP, ssp);  in shstk_push()
320 static int shstk_push_sigframe(unsigned long *ssp)  in shstk_push_sigframe()  argument
322 	unsigned long target_ssp = *ssp;  in shstk_push_sigframe()
328 	*ssp -= SS_FRAME_SIZE;  in shstk_push_sigframe()
329 	if (put_shstk_data((void __user *)*ssp, target_ssp))  in shstk_push_sigframe()
335 static int shstk_pop_sigframe(unsigned long *ssp)  in shstk_pop_sigframe()  argument
343 	 * It is possible for the SSP to be off the end of a shadow stack by 4  in shstk_pop_sigframe()
348 	if (!IS_ALIGNED(*ssp, 8))  in shstk_pop_sigframe()
351 	need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp;  in shstk_pop_sigframe()
356 	err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp);  in shstk_pop_sigframe()
361 		vma = find_vma(current->mm, *ssp);  in shstk_pop_sigframe()
370 	/* Restore SSP aligned? */  in shstk_pop_sigframe()
374 	/* SSP in userspace? */  in shstk_pop_sigframe()
378 	*ssp = token_addr;  in shstk_pop_sigframe()
390 	unsigned long ssp;  in setup_signal_shadow_stack()  local
400 	ssp = get_user_shstk_addr();  in setup_signal_shadow_stack()
401 	if (unlikely(!ssp))  in setup_signal_shadow_stack()
404 	err = shstk_push_sigframe(&ssp);  in setup_signal_shadow_stack()
409 	ssp -= SS_FRAME_SIZE;  in setup_signal_shadow_stack()
410 	err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer);  in setup_signal_shadow_stack()
415 	wrmsrq(MSR_IA32_PL3_SSP, ssp);  in setup_signal_shadow_stack()
423 	unsigned long ssp;  in restore_signal_shadow_stack()  local
430 	ssp = get_user_shstk_addr();  in restore_signal_shadow_stack()
431 	if (unlikely(!ssp))  in restore_signal_shadow_stack()
434 	err = shstk_pop_sigframe(&ssp);  in restore_signal_shadow_stack()
439 	wrmsrq(MSR_IA32_PL3_SSP, ssp);  in restore_signal_shadow_stack()
623 	unsigned long ssp;  in shstk_update_last_frame()  local
628 	ssp = get_user_shstk_addr();  in shstk_update_last_frame()
629 	return write_user_shstk_64((u64 __user *)ssp, (u64)val);  in shstk_update_last_frame()