1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_STRING_64_H 3 #define _ASM_X86_STRING_64_H 4 5 #ifdef __KERNEL__ 6 #include <linux/jump_label.h> 7 8 /* Written 2002 by Andi Kleen */ 9 10 /* Even with __builtin_ the compiler may decide to use the out of line 11 function. */ 12 13 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 14 #include <linux/kmsan_string.h> 15 #endif 16 17 #define __HAVE_ARCH_MEMCPY 1 18 extern void *memcpy(void *to, const void *from, size_t len); 19 extern void *__memcpy(void *to, const void *from, size_t len); 20 21 #define __HAVE_ARCH_MEMSET 22 void *memset(void *s, int c, size_t n); 23 void *__memset(void *s, int c, size_t n); 24 KCFI_REFERENCE(__memset); 25 26 /* 27 * KMSAN needs to instrument as much code as possible. Use C versions of 28 * memsetXX() from lib/string.c under KMSAN. 29 */ 30 #if !defined(CONFIG_KMSAN) 31 #define __HAVE_ARCH_MEMSET16 32 static inline void *memset16(uint16_t *s, uint16_t v, size_t n) 33 { 34 const auto s0 = s; 35 asm volatile ( 36 "rep stosw" 37 : "+D" (s), "+c" (n) 38 : "a" (v) 39 : "memory" 40 ); 41 return s0; 42 } 43 44 #define __HAVE_ARCH_MEMSET32 45 static inline void *memset32(uint32_t *s, uint32_t v, size_t n) 46 { 47 const auto s0 = s; 48 asm volatile ( 49 "rep stosl" 50 : "+D" (s), "+c" (n) 51 : "a" (v) 52 : "memory" 53 ); 54 return s0; 55 } 56 57 #define __HAVE_ARCH_MEMSET64 58 static inline void *memset64(uint64_t *s, uint64_t v, size_t n) 59 { 60 const auto s0 = s; 61 asm volatile ( 62 "rep stosq" 63 : "+D" (s), "+c" (n) 64 : "a" (v) 65 : "memory" 66 ); 67 return s0; 68 } 69 #endif 70 71 #define __HAVE_ARCH_MEMMOVE 72 void *memmove(void *dest, const void *src, size_t count); 73 void *__memmove(void *dest, const void *src, size_t count); 74 KCFI_REFERENCE(__memmove); 75 76 int memcmp(const void *cs, const void *ct, size_t count); 77 size_t strlen(const char *s); 78 char *strcpy(char *dest, const char *src); 79 char *strcat(char *dest, const char *src); 80 int strcmp(const char *cs, const char *ct); 81 82 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 83 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 84 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); 85 86 static __always_inline void movnti_4(void *dst, const void *src) 87 { 88 asm volatile("movntil %1, %0" 89 : "=m"(*(u32 *)dst) 90 : "r"(*(const u32 *)src) 91 : "memory"); 92 } 93 94 static __always_inline void movnti_8(void *dst, const void *src) 95 { 96 asm volatile("movntiq %1, %0" 97 : "=m"(*(u64 *)dst) 98 : "r"(*(const u64 *)src) 99 : "memory"); 100 } 101 102 static __always_inline void movnti_16(void *dst, const void *src) 103 { 104 movnti_8(dst, src); 105 movnti_8(dst + 8, src + 8); 106 } 107 108 static __always_inline void movnti_32(void *dst, const void *src) 109 { 110 movnti_16(dst, src); 111 movnti_16(dst + 16, src + 16); 112 } 113 114 static __always_inline void movnti_64(void *dst, const void *src) 115 { 116 movnti_32(dst, src); 117 movnti_32(dst + 32, src + 32); 118 } 119 120 static __always_inline void memcpy_flushcache(void *dst, const void *src, 121 size_t cnt) 122 { 123 if (!__builtin_constant_p(cnt)) 124 return __memcpy_flushcache(dst, src, cnt); 125 126 /* 127 * The relevant fixed-size copies here are the x86_64 struct page sizes: 128 * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well 129 * instead of sending those nearby fixed-size cases back to 130 * __memcpy_flushcache(). 131 */ 132 switch (cnt) { 133 case 4: movnti_4(dst, src); break; 134 case 8: movnti_8(dst, src); break; 135 case 16: movnti_16(dst, src); break; 136 case 32: movnti_32(dst, src); break; 137 case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break; 138 case 64: movnti_64(dst, src); break; 139 case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break; 140 case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break; 141 default: __memcpy_flushcache(dst, src, cnt); break; 142 } 143 } 144 145 #define memcpy_nontemporal memcpy_nontemporal 146 /* 147 * Reuse the existing x86 flushcache backend as the non-temporal copy 148 * primitive. 149 */ 150 static __always_inline void memcpy_nontemporal(void *dst, const void *src, 151 size_t cnt) 152 { 153 memcpy_flushcache(dst, src, cnt); 154 } 155 156 #endif 157 158 #endif /* __KERNEL__ */ 159 160 #endif /* _ASM_X86_STRING_64_H */ 161