1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> 4 * Copyright 2002 Andi Kleen, SuSE Labs. 5 * 6 * Functions to copy from and to user space. 7 */ 8 9#include <linux/linkage.h> 10#include <asm/current.h> 11#include <asm/asm-offsets.h> 12#include <asm/thread_info.h> 13#include <asm/cpufeatures.h> 14#include <asm/alternative.h> 15#include <asm/asm.h> 16#include <asm/smap.h> 17#include <asm/export.h> 18#include <asm/trapnr.h> 19 20.macro ALIGN_DESTINATION 21 /* check for bad alignment of destination */ 22 movl %edi,%ecx 23 andl $7,%ecx 24 jz 102f /* already aligned */ 25 subl $8,%ecx 26 negl %ecx 27 subl %ecx,%edx 28100: movb (%rsi),%al 29101: movb %al,(%rdi) 30 incq %rsi 31 incq %rdi 32 decl %ecx 33 jnz 100b 34102: 35 36 _ASM_EXTABLE_CPY(100b, .Lcopy_user_handle_align) 37 _ASM_EXTABLE_CPY(101b, .Lcopy_user_handle_align) 38.endm 39 40/* 41 * copy_user_generic_unrolled - memory copy with exception handling. 42 * This version is for CPUs like P4 that don't have efficient micro 43 * code for rep movsq 44 * 45 * Input: 46 * rdi destination 47 * rsi source 48 * rdx count 49 * 50 * Output: 51 * eax uncopied bytes or 0 if successful. 52 */ 53SYM_FUNC_START(copy_user_generic_unrolled) 54 ASM_STAC 55 cmpl $8,%edx 56 jb .Lcopy_user_short_string_bytes 57 ALIGN_DESTINATION 58 movl %edx,%ecx 59 andl $63,%edx 60 shrl $6,%ecx 61 jz copy_user_short_string 621: movq (%rsi),%r8 632: movq 1*8(%rsi),%r9 643: movq 2*8(%rsi),%r10 654: movq 3*8(%rsi),%r11 665: movq %r8,(%rdi) 676: movq %r9,1*8(%rdi) 687: movq %r10,2*8(%rdi) 698: movq %r11,3*8(%rdi) 709: movq 4*8(%rsi),%r8 7110: movq 5*8(%rsi),%r9 7211: movq 6*8(%rsi),%r10 7312: movq 7*8(%rsi),%r11 7413: movq %r8,4*8(%rdi) 7514: movq %r9,5*8(%rdi) 7615: movq %r10,6*8(%rdi) 7716: movq %r11,7*8(%rdi) 78 leaq 64(%rsi),%rsi 79 leaq 64(%rdi),%rdi 80 decl %ecx 81 jnz 1b 82 jmp copy_user_short_string 83 8430: shll $6,%ecx 85 addl %ecx,%edx 86 jmp .Lcopy_user_handle_tail 87 88 _ASM_EXTABLE_CPY(1b, 30b) 89 _ASM_EXTABLE_CPY(2b, 30b) 90 _ASM_EXTABLE_CPY(3b, 30b) 91 _ASM_EXTABLE_CPY(4b, 30b) 92 _ASM_EXTABLE_CPY(5b, 30b) 93 _ASM_EXTABLE_CPY(6b, 30b) 94 _ASM_EXTABLE_CPY(7b, 30b) 95 _ASM_EXTABLE_CPY(8b, 30b) 96 _ASM_EXTABLE_CPY(9b, 30b) 97 _ASM_EXTABLE_CPY(10b, 30b) 98 _ASM_EXTABLE_CPY(11b, 30b) 99 _ASM_EXTABLE_CPY(12b, 30b) 100 _ASM_EXTABLE_CPY(13b, 30b) 101 _ASM_EXTABLE_CPY(14b, 30b) 102 _ASM_EXTABLE_CPY(15b, 30b) 103 _ASM_EXTABLE_CPY(16b, 30b) 104SYM_FUNC_END(copy_user_generic_unrolled) 105EXPORT_SYMBOL(copy_user_generic_unrolled) 106 107/* Some CPUs run faster using the string copy instructions. 108 * This is also a lot simpler. Use them when possible. 109 * 110 * Only 4GB of copy is supported. This shouldn't be a problem 111 * because the kernel normally only writes from/to page sized chunks 112 * even if user space passed a longer buffer. 113 * And more would be dangerous because both Intel and AMD have 114 * errata with rep movsq > 4GB. If someone feels the need to fix 115 * this please consider this. 116 * 117 * Input: 118 * rdi destination 119 * rsi source 120 * rdx count 121 * 122 * Output: 123 * eax uncopied bytes or 0 if successful. 124 */ 125SYM_FUNC_START(copy_user_generic_string) 126 ASM_STAC 127 cmpl $8,%edx 128 jb 2f /* less than 8 bytes, go to byte copy loop */ 129 ALIGN_DESTINATION 130 movl %edx,%ecx 131 shrl $3,%ecx 132 andl $7,%edx 1331: rep movsq 1342: movl %edx,%ecx 1353: rep movsb 136 xorl %eax,%eax 137 ASM_CLAC 138 RET 139 14011: leal (%rdx,%rcx,8),%ecx 14112: movl %ecx,%edx /* ecx is zerorest also */ 142 jmp .Lcopy_user_handle_tail 143 144 _ASM_EXTABLE_CPY(1b, 11b) 145 _ASM_EXTABLE_CPY(3b, 12b) 146SYM_FUNC_END(copy_user_generic_string) 147EXPORT_SYMBOL(copy_user_generic_string) 148 149/* 150 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions. 151 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled. 152 * 153 * Input: 154 * rdi destination 155 * rsi source 156 * rdx count 157 * 158 * Output: 159 * eax uncopied bytes or 0 if successful. 160 */ 161SYM_FUNC_START(copy_user_enhanced_fast_string) 162 ASM_STAC 163 /* CPUs without FSRM should avoid rep movsb for short copies */ 164 ALTERNATIVE "cmpl $64, %edx; jb copy_user_short_string", "", X86_FEATURE_FSRM 165 movl %edx,%ecx 1661: rep movsb 167 xorl %eax,%eax 168 ASM_CLAC 169 RET 170 17112: movl %ecx,%edx /* ecx is zerorest also */ 172 jmp .Lcopy_user_handle_tail 173 174 _ASM_EXTABLE_CPY(1b, 12b) 175SYM_FUNC_END(copy_user_enhanced_fast_string) 176EXPORT_SYMBOL(copy_user_enhanced_fast_string) 177 178/* 179 * Try to copy last bytes and clear the rest if needed. 180 * Since protection fault in copy_from/to_user is not a normal situation, 181 * it is not necessary to optimize tail handling. 182 * Don't try to copy the tail if machine check happened 183 * 184 * Input: 185 * eax trap number written by ex_handler_copy() 186 * rdi destination 187 * rsi source 188 * rdx count 189 * 190 * Output: 191 * eax uncopied bytes or 0 if successful. 192 */ 193SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail) 194 cmp $X86_TRAP_MC,%eax 195 je 3f 196 197 movl %edx,%ecx 1981: rep movsb 1992: mov %ecx,%eax 200 ASM_CLAC 201 RET 202 2033: 204 movl %edx,%eax 205 ASM_CLAC 206 RET 207 208 _ASM_EXTABLE_CPY(1b, 2b) 209 210.Lcopy_user_handle_align: 211 addl %ecx,%edx /* ecx is zerorest also */ 212 jmp .Lcopy_user_handle_tail 213 214SYM_CODE_END(.Lcopy_user_handle_tail) 215 216/* 217 * Finish memcpy of less than 64 bytes. #AC should already be set. 218 * 219 * Input: 220 * rdi destination 221 * rsi source 222 * rdx count (< 64) 223 * 224 * Output: 225 * eax uncopied bytes or 0 if successful. 226 */ 227SYM_CODE_START_LOCAL(copy_user_short_string) 228 movl %edx,%ecx 229 andl $7,%edx 230 shrl $3,%ecx 231 jz .Lcopy_user_short_string_bytes 23218: movq (%rsi),%r8 23319: movq %r8,(%rdi) 234 leaq 8(%rsi),%rsi 235 leaq 8(%rdi),%rdi 236 decl %ecx 237 jnz 18b 238.Lcopy_user_short_string_bytes: 239 andl %edx,%edx 240 jz 23f 241 movl %edx,%ecx 24221: movb (%rsi),%al 24322: movb %al,(%rdi) 244 incq %rsi 245 incq %rdi 246 decl %ecx 247 jnz 21b 24823: xor %eax,%eax 249 ASM_CLAC 250 RET 251 25240: leal (%rdx,%rcx,8),%edx 253 jmp 60f 25450: movl %ecx,%edx /* ecx is zerorest also */ 25560: jmp .Lcopy_user_handle_tail 256 257 _ASM_EXTABLE_CPY(18b, 40b) 258 _ASM_EXTABLE_CPY(19b, 40b) 259 _ASM_EXTABLE_CPY(21b, 50b) 260 _ASM_EXTABLE_CPY(22b, 50b) 261SYM_CODE_END(copy_user_short_string) 262 263/* 264 * copy_user_nocache - Uncached memory copy with exception handling 265 * This will force destination out of cache for more performance. 266 * 267 * Note: Cached memory copy is used when destination or size is not 268 * naturally aligned. That is: 269 * - Require 8-byte alignment when size is 8 bytes or larger. 270 * - Require 4-byte alignment when size is 4 bytes. 271 */ 272SYM_FUNC_START(__copy_user_nocache) 273 ASM_STAC 274 275 /* If size is less than 8 bytes, go to 4-byte copy */ 276 cmpl $8,%edx 277 jb .L_4b_nocache_copy_entry 278 279 /* If destination is not 8-byte aligned, "cache" copy to align it */ 280 ALIGN_DESTINATION 281 282 /* Set 4x8-byte copy count and remainder */ 283 movl %edx,%ecx 284 andl $63,%edx 285 shrl $6,%ecx 286 jz .L_8b_nocache_copy_entry /* jump if count is 0 */ 287 288 /* Perform 4x8-byte nocache loop-copy */ 289.L_4x8b_nocache_copy_loop: 2901: movq (%rsi),%r8 2912: movq 1*8(%rsi),%r9 2923: movq 2*8(%rsi),%r10 2934: movq 3*8(%rsi),%r11 2945: movnti %r8,(%rdi) 2956: movnti %r9,1*8(%rdi) 2967: movnti %r10,2*8(%rdi) 2978: movnti %r11,3*8(%rdi) 2989: movq 4*8(%rsi),%r8 29910: movq 5*8(%rsi),%r9 30011: movq 6*8(%rsi),%r10 30112: movq 7*8(%rsi),%r11 30213: movnti %r8,4*8(%rdi) 30314: movnti %r9,5*8(%rdi) 30415: movnti %r10,6*8(%rdi) 30516: movnti %r11,7*8(%rdi) 306 leaq 64(%rsi),%rsi 307 leaq 64(%rdi),%rdi 308 decl %ecx 309 jnz .L_4x8b_nocache_copy_loop 310 311 /* Set 8-byte copy count and remainder */ 312.L_8b_nocache_copy_entry: 313 movl %edx,%ecx 314 andl $7,%edx 315 shrl $3,%ecx 316 jz .L_4b_nocache_copy_entry /* jump if count is 0 */ 317 318 /* Perform 8-byte nocache loop-copy */ 319.L_8b_nocache_copy_loop: 32020: movq (%rsi),%r8 32121: movnti %r8,(%rdi) 322 leaq 8(%rsi),%rsi 323 leaq 8(%rdi),%rdi 324 decl %ecx 325 jnz .L_8b_nocache_copy_loop 326 327 /* If no byte left, we're done */ 328.L_4b_nocache_copy_entry: 329 andl %edx,%edx 330 jz .L_finish_copy 331 332 /* If destination is not 4-byte aligned, go to byte copy: */ 333 movl %edi,%ecx 334 andl $3,%ecx 335 jnz .L_1b_cache_copy_entry 336 337 /* Set 4-byte copy count (1 or 0) and remainder */ 338 movl %edx,%ecx 339 andl $3,%edx 340 shrl $2,%ecx 341 jz .L_1b_cache_copy_entry /* jump if count is 0 */ 342 343 /* Perform 4-byte nocache copy: */ 34430: movl (%rsi),%r8d 34531: movnti %r8d,(%rdi) 346 leaq 4(%rsi),%rsi 347 leaq 4(%rdi),%rdi 348 349 /* If no bytes left, we're done: */ 350 andl %edx,%edx 351 jz .L_finish_copy 352 353 /* Perform byte "cache" loop-copy for the remainder */ 354.L_1b_cache_copy_entry: 355 movl %edx,%ecx 356.L_1b_cache_copy_loop: 35740: movb (%rsi),%al 35841: movb %al,(%rdi) 359 incq %rsi 360 incq %rdi 361 decl %ecx 362 jnz .L_1b_cache_copy_loop 363 364 /* Finished copying; fence the prior stores */ 365.L_finish_copy: 366 xorl %eax,%eax 367 ASM_CLAC 368 sfence 369 RET 370 371.L_fixup_4x8b_copy: 372 shll $6,%ecx 373 addl %ecx,%edx 374 jmp .L_fixup_handle_tail 375.L_fixup_8b_copy: 376 lea (%rdx,%rcx,8),%rdx 377 jmp .L_fixup_handle_tail 378.L_fixup_4b_copy: 379 lea (%rdx,%rcx,4),%rdx 380 jmp .L_fixup_handle_tail 381.L_fixup_1b_copy: 382 movl %ecx,%edx 383.L_fixup_handle_tail: 384 sfence 385 jmp .Lcopy_user_handle_tail 386 387 _ASM_EXTABLE_CPY(1b, .L_fixup_4x8b_copy) 388 _ASM_EXTABLE_CPY(2b, .L_fixup_4x8b_copy) 389 _ASM_EXTABLE_CPY(3b, .L_fixup_4x8b_copy) 390 _ASM_EXTABLE_CPY(4b, .L_fixup_4x8b_copy) 391 _ASM_EXTABLE_CPY(5b, .L_fixup_4x8b_copy) 392 _ASM_EXTABLE_CPY(6b, .L_fixup_4x8b_copy) 393 _ASM_EXTABLE_CPY(7b, .L_fixup_4x8b_copy) 394 _ASM_EXTABLE_CPY(8b, .L_fixup_4x8b_copy) 395 _ASM_EXTABLE_CPY(9b, .L_fixup_4x8b_copy) 396 _ASM_EXTABLE_CPY(10b, .L_fixup_4x8b_copy) 397 _ASM_EXTABLE_CPY(11b, .L_fixup_4x8b_copy) 398 _ASM_EXTABLE_CPY(12b, .L_fixup_4x8b_copy) 399 _ASM_EXTABLE_CPY(13b, .L_fixup_4x8b_copy) 400 _ASM_EXTABLE_CPY(14b, .L_fixup_4x8b_copy) 401 _ASM_EXTABLE_CPY(15b, .L_fixup_4x8b_copy) 402 _ASM_EXTABLE_CPY(16b, .L_fixup_4x8b_copy) 403 _ASM_EXTABLE_CPY(20b, .L_fixup_8b_copy) 404 _ASM_EXTABLE_CPY(21b, .L_fixup_8b_copy) 405 _ASM_EXTABLE_CPY(30b, .L_fixup_4b_copy) 406 _ASM_EXTABLE_CPY(31b, .L_fixup_4b_copy) 407 _ASM_EXTABLE_CPY(40b, .L_fixup_1b_copy) 408 _ASM_EXTABLE_CPY(41b, .L_fixup_1b_copy) 409SYM_FUNC_END(__copy_user_nocache) 410EXPORT_SYMBOL(__copy_user_nocache) 411