1ec6347bbSDan Williams // SPDX-License-Identifier: GPL-2.0 2ec6347bbSDan Williams /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ 3ec6347bbSDan Williams 4ec6347bbSDan Williams #include <linux/jump_label.h> 5ec6347bbSDan Williams #include <linux/uaccess.h> 6ec6347bbSDan Williams #include <linux/export.h> 7*61b258b0SAlexander Potapenko #include <linux/instrumented.h> 8ec6347bbSDan Williams #include <linux/string.h> 9ec6347bbSDan Williams #include <linux/types.h> 10ec6347bbSDan Williams 11ec6347bbSDan Williams #include <asm/mce.h> 12ec6347bbSDan Williams 13ec6347bbSDan Williams #ifdef CONFIG_X86_MCE 14ec6347bbSDan Williams static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key); 15ec6347bbSDan Williams 16ec6347bbSDan Williams void enable_copy_mc_fragile(void) 17ec6347bbSDan Williams { 18ec6347bbSDan Williams static_branch_inc(©_mc_fragile_key); 19ec6347bbSDan Williams } 20ec6347bbSDan Williams #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key)) 21ec6347bbSDan Williams 22ec6347bbSDan Williams /* 23ec6347bbSDan Williams * Similar to copy_user_handle_tail, probe for the write fault point, or 24ec6347bbSDan Williams * source exception point. 25ec6347bbSDan Williams */ 26ec6347bbSDan Williams __visible notrace unsigned long 27ec6347bbSDan Williams copy_mc_fragile_handle_tail(char *to, char *from, unsigned len) 28ec6347bbSDan Williams { 29ec6347bbSDan Williams for (; len; --len, to++, from++) 30ec6347bbSDan Williams if (copy_mc_fragile(to, from, 1)) 31ec6347bbSDan Williams break; 32ec6347bbSDan Williams return len; 33ec6347bbSDan Williams } 34ec6347bbSDan Williams #else 35ec6347bbSDan Williams /* 36ec6347bbSDan Williams * No point in doing careful copying, or consulting a static key when 37ec6347bbSDan Williams * there is no #MC handler in the CONFIG_X86_MCE=n case. 38ec6347bbSDan Williams */ 39ec6347bbSDan Williams void enable_copy_mc_fragile(void) 40ec6347bbSDan Williams { 41ec6347bbSDan Williams } 42ec6347bbSDan Williams #define copy_mc_fragile_enabled (0) 43ec6347bbSDan Williams #endif 44ec6347bbSDan Williams 455da8e4a6SDan Williams unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len); 465da8e4a6SDan Williams 47ec6347bbSDan Williams /** 48ec6347bbSDan Williams * copy_mc_to_kernel - memory copy that handles source exceptions 49ec6347bbSDan Williams * 50ec6347bbSDan Williams * @dst: destination address 51ec6347bbSDan Williams * @src: source address 52ec6347bbSDan Williams * @len: number of bytes to copy 53ec6347bbSDan Williams * 545da8e4a6SDan Williams * Call into the 'fragile' version on systems that benefit from avoiding 555da8e4a6SDan Williams * corner case poison consumption scenarios, For example, accessing 565da8e4a6SDan Williams * poison across 2 cachelines with a single instruction. Almost all 575da8e4a6SDan Williams * other uses case can use copy_mc_enhanced_fast_string() for a fast 585da8e4a6SDan Williams * recoverable copy, or fallback to plain memcpy. 59ec6347bbSDan Williams * 60ec6347bbSDan Williams * Return 0 for success, or number of bytes not copied if there was an 61ec6347bbSDan Williams * exception. 62ec6347bbSDan Williams */ 63ec6347bbSDan Williams unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len) 64ec6347bbSDan Williams { 65*61b258b0SAlexander Potapenko unsigned long ret; 66*61b258b0SAlexander Potapenko 67*61b258b0SAlexander Potapenko if (copy_mc_fragile_enabled) { 68*61b258b0SAlexander Potapenko instrument_memcpy_before(dst, src, len); 69*61b258b0SAlexander Potapenko ret = copy_mc_fragile(dst, src, len); 70*61b258b0SAlexander Potapenko instrument_memcpy_after(dst, src, len, ret); 71*61b258b0SAlexander Potapenko return ret; 72*61b258b0SAlexander Potapenko } 73*61b258b0SAlexander Potapenko if (static_cpu_has(X86_FEATURE_ERMS)) { 74*61b258b0SAlexander Potapenko instrument_memcpy_before(dst, src, len); 75*61b258b0SAlexander Potapenko ret = copy_mc_enhanced_fast_string(dst, src, len); 76*61b258b0SAlexander Potapenko instrument_memcpy_after(dst, src, len, ret); 77*61b258b0SAlexander Potapenko return ret; 78*61b258b0SAlexander Potapenko } 79ec6347bbSDan Williams memcpy(dst, src, len); 80ec6347bbSDan Williams return 0; 81ec6347bbSDan Williams } 82ec6347bbSDan Williams EXPORT_SYMBOL_GPL(copy_mc_to_kernel); 83ec6347bbSDan Williams 84066baf92SDavid Howells unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len) 85ec6347bbSDan Williams { 86ec6347bbSDan Williams unsigned long ret; 87ec6347bbSDan Williams 885da8e4a6SDan Williams if (copy_mc_fragile_enabled) { 89*61b258b0SAlexander Potapenko instrument_copy_to_user(dst, src, len); 90ec6347bbSDan Williams __uaccess_begin(); 91066baf92SDavid Howells ret = copy_mc_fragile((__force void *)dst, src, len); 92ec6347bbSDan Williams __uaccess_end(); 93ec6347bbSDan Williams return ret; 94ec6347bbSDan Williams } 955da8e4a6SDan Williams 965da8e4a6SDan Williams if (static_cpu_has(X86_FEATURE_ERMS)) { 97*61b258b0SAlexander Potapenko instrument_copy_to_user(dst, src, len); 985da8e4a6SDan Williams __uaccess_begin(); 99066baf92SDavid Howells ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len); 1005da8e4a6SDan Williams __uaccess_end(); 1015da8e4a6SDan Williams return ret; 1025da8e4a6SDan Williams } 1035da8e4a6SDan Williams 104066baf92SDavid Howells return copy_user_generic((__force void *)dst, src, len); 1055da8e4a6SDan Williams } 106