1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_STRING_64_H 3 #define _ASM_X86_STRING_64_H 4 5 #ifdef __KERNEL__ 6 7 /* Written 2002 by Andi Kleen */ 8 9 /* Even with __builtin_ the compiler may decide to use the out of line 10 function. */ 11 12 #if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) 13 #include <linux/kmsan_string.h> 14 #endif 15 16 #define __HAVE_ARCH_MEMCPY 1 17 extern void *memcpy(void *to, const void *from, size_t len); 18 extern void *__memcpy(void *to, const void *from, size_t len); 19 20 #define __HAVE_ARCH_MEMSET 21 void *memset(void *s, int c, size_t n); 22 void *__memset(void *s, int c, size_t n); 23 KCFI_REFERENCE(__memset); 24 25 /* 26 * KMSAN needs to instrument as much code as possible. Use C versions of 27 * memsetXX() from lib/string.c under KMSAN. 28 */ 29 #if !defined(CONFIG_KMSAN) 30 #define __HAVE_ARCH_MEMSET16 31 static inline void *memset16(uint16_t *s, uint16_t v, size_t n) 32 { 33 const auto s0 = s; 34 asm volatile ( 35 "rep stosw" 36 : "+D" (s), "+c" (n) 37 : "a" (v) 38 : "memory" 39 ); 40 return s0; 41 } 42 43 #define __HAVE_ARCH_MEMSET32 44 static inline void *memset32(uint32_t *s, uint32_t v, size_t n) 45 { 46 const auto s0 = s; 47 asm volatile ( 48 "rep stosl" 49 : "+D" (s), "+c" (n) 50 : "a" (v) 51 : "memory" 52 ); 53 return s0; 54 } 55 56 #define __HAVE_ARCH_MEMSET64 57 static inline void *memset64(uint64_t *s, uint64_t v, size_t n) 58 { 59 const auto s0 = s; 60 asm volatile ( 61 "rep stosq" 62 : "+D" (s), "+c" (n) 63 : "a" (v) 64 : "memory" 65 ); 66 return s0; 67 } 68 #endif 69 70 #define __HAVE_ARCH_MEMMOVE 71 void *memmove(void *dest, const void *src, size_t count); 72 void *__memmove(void *dest, const void *src, size_t count); 73 KCFI_REFERENCE(__memmove); 74 75 int memcmp(const void *cs, const void *ct, size_t count); 76 size_t strlen(const char *s); 77 char *strcpy(char *dest, const char *src); 78 char *strcat(char *dest, const char *src); 79 int strcmp(const char *cs, const char *ct); 80 81 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 82 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 83 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); 84 85 static __always_inline void movnti_4(void *dst, const void *src) 86 { 87 asm volatile("movntil %1, %0" 88 : "=m"(*(u32 *)dst) 89 : "r"(*(const u32 *)src) 90 : "memory"); 91 } 92 93 static __always_inline void movnti_8(void *dst, const void *src) 94 { 95 asm volatile("movntiq %1, %0" 96 : "=m"(*(u64 *)dst) 97 : "r"(*(const u64 *)src) 98 : "memory"); 99 } 100 101 static __always_inline void movnti_16(void *dst, const void *src) 102 { 103 movnti_8(dst, src); 104 movnti_8(dst + 8, src + 8); 105 } 106 107 static __always_inline void movnti_32(void *dst, const void *src) 108 { 109 movnti_16(dst, src); 110 movnti_16(dst + 16, src + 16); 111 } 112 113 static __always_inline void movnti_64(void *dst, const void *src) 114 { 115 movnti_32(dst, src); 116 movnti_32(dst + 32, src + 32); 117 } 118 119 static __always_inline void memcpy_flushcache(void *dst, const void *src, 120 size_t cnt) 121 { 122 if (!__builtin_constant_p(cnt)) 123 return __memcpy_flushcache(dst, src, cnt); 124 125 /* 126 * The relevant fixed-size copies here are the x86_64 struct page sizes: 127 * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well 128 * instead of sending those nearby fixed-size cases back to 129 * __memcpy_flushcache(). 130 */ 131 switch (cnt) { 132 case 4: movnti_4(dst, src); break; 133 case 8: movnti_8(dst, src); break; 134 case 16: movnti_16(dst, src); break; 135 case 32: movnti_32(dst, src); break; 136 case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break; 137 case 64: movnti_64(dst, src); break; 138 case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break; 139 case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break; 140 default: __memcpy_flushcache(dst, src, cnt); break; 141 } 142 } 143 144 #define memcpy_nontemporal memcpy_nontemporal 145 /* 146 * Reuse the existing x86 flushcache backend as the non-temporal copy 147 * primitive. 148 */ 149 static __always_inline void memcpy_nontemporal(void *dst, const void *src, 150 size_t cnt) 151 { 152 memcpy_flushcache(dst, src, cnt); 153 } 154 155 #endif 156 157 #endif /* __KERNEL__ */ 158 159 #endif /* _ASM_X86_STRING_64_H */ 160