1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 6#include <linux/export.h> 7#include <asm/alternative-asm.h> 8#include <asm/asm.h> 9#include <asm/asmmacro.h> 10#include <asm/asm-extable.h> 11#include <asm/cpu.h> 12#include <asm/regdef.h> 13#include <asm/unwind_hints.h> 14 15SYM_FUNC_START(__copy_user) 16 /* 17 * Some CPUs support hardware unaligned access 18 */ 19 ALTERNATIVE "b __copy_user_generic", \ 20 "b __copy_user_fast", CPU_FEATURE_UAL 21SYM_FUNC_END(__copy_user) 22 23EXPORT_SYMBOL(__copy_user) 24 25/* 26 * unsigned long __copy_user_generic(void *to, const void *from, size_t n) 27 * 28 * a0: to 29 * a1: from 30 * a2: n 31 */ 32SYM_FUNC_START(__copy_user_generic) 33 beqz a2, 3f 34 351: ld.b t0, a1, 0 362: st.b t0, a0, 0 37 addi.d a0, a0, 1 38 addi.d a1, a1, 1 39 addi.d a2, a2, -1 40 bgtz a2, 1b 41 423: move a0, a2 43 jr ra 44 45 _asm_extable 1b, 3b 46 _asm_extable 2b, 3b 47SYM_FUNC_END(__copy_user_generic) 48 49/* 50 * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n) 51 * 52 * a0: to 53 * a1: from 54 * a2: n 55 */ 56SYM_FUNC_START(__copy_user_fast) 57 sltui t0, a2, 9 58 bnez t0, .Lsmall 59 600: ld.d t0, a1, 0 611: st.d t0, a0, 0 62 add.d a3, a1, a2 63 add.d a2, a0, a2 64 65 /* align up destination address */ 66 andi t1, a0, 7 67 sub.d t0, zero, t1 68 addi.d t0, t0, 8 69 add.d a1, a1, t0 70 add.d a0, a0, t0 71 72 addi.d a4, a3, -64 73 bgeu a1, a4, .Llt64 74 75 /* copy 64 bytes at a time */ 76.Lloop64: 772: ld.d t0, a1, 0 783: ld.d t1, a1, 8 794: ld.d t2, a1, 16 805: ld.d t3, a1, 24 816: ld.d t4, a1, 32 827: ld.d t5, a1, 40 838: ld.d t6, a1, 48 849: ld.d t7, a1, 56 8510: st.d t0, a0, 0 8611: st.d t1, a0, 8 8712: st.d t2, a0, 16 8813: st.d t3, a0, 24 8914: st.d t4, a0, 32 9015: st.d t5, a0, 40 9116: st.d t6, a0, 48 9217: st.d t7, a0, 56 93 addi.d a1, a1, 64 94 addi.d a0, a0, 64 95 bltu a1, a4, .Lloop64 96 97 /* copy the remaining bytes */ 98.Llt64: 99 addi.d a4, a3, -32 100 bgeu a1, a4, .Llt32 10118: ld.d t0, a1, 0 10219: ld.d t1, a1, 8 10320: ld.d t2, a1, 16 10421: ld.d t3, a1, 24 10522: st.d t0, a0, 0 10623: st.d t1, a0, 8 10724: st.d t2, a0, 16 10825: st.d t3, a0, 24 109 addi.d a1, a1, 32 110 addi.d a0, a0, 32 111 112.Llt32: 113 addi.d a4, a3, -16 114 bgeu a1, a4, .Llt16 11526: ld.d t0, a1, 0 11627: ld.d t1, a1, 8 11728: st.d t0, a0, 0 11829: st.d t1, a0, 8 119 addi.d a1, a1, 16 120 addi.d a0, a0, 16 121 122.Llt16: 123 addi.d a4, a3, -8 124 bgeu a1, a4, .Llt8 12530: ld.d t0, a1, 0 12631: st.d t0, a0, 0 127 addi.d a1, a1, 8 128 addi.d a0, a0, 8 129 130.Llt8: 13132: ld.d t0, a3, -8 13233: st.d t0, a2, -8 133 134 /* return */ 135 move a0, zero 136 jr ra 137 138 .align 5 139.Lsmall: 140 pcaddi t0, 8 141 slli.d a3, a2, 5 142 add.d t0, t0, a3 143 jr t0 144 145 .align 5 146 move a0, zero 147 jr ra 148 149 .align 5 15034: ld.b t0, a1, 0 15135: st.b t0, a0, 0 152 move a0, zero 153 jr ra 154 155 .align 5 15636: ld.h t0, a1, 0 15737: st.h t0, a0, 0 158 move a0, zero 159 jr ra 160 161 .align 5 16238: ld.h t0, a1, 0 16339: ld.b t1, a1, 2 16440: st.h t0, a0, 0 16541: st.b t1, a0, 2 166 move a0, zero 167 jr ra 168 169 .align 5 17042: ld.w t0, a1, 0 17143: st.w t0, a0, 0 172 move a0, zero 173 jr ra 174 175 .align 5 17644: ld.w t0, a1, 0 17745: ld.b t1, a1, 4 17846: st.w t0, a0, 0 17947: st.b t1, a0, 4 180 move a0, zero 181 jr ra 182 183 .align 5 18448: ld.w t0, a1, 0 18549: ld.h t1, a1, 4 18650: st.w t0, a0, 0 18751: st.h t1, a0, 4 188 move a0, zero 189 jr ra 190 191 .align 5 19252: ld.w t0, a1, 0 19353: ld.w t1, a1, 3 19454: st.w t0, a0, 0 19555: st.w t1, a0, 3 196 move a0, zero 197 jr ra 198 199 .align 5 20056: ld.d t0, a1, 0 20157: st.d t0, a0, 0 202 move a0, zero 203 jr ra 204 205 /* fixup and ex_table */ 206.Llarge_fixup: 207 sub.d a2, a2, a0 208 209.Lsmall_fixup: 21058: ld.b t0, a1, 0 21159: st.b t0, a0, 0 212 addi.d a0, a0, 1 213 addi.d a1, a1, 1 214 addi.d a2, a2, -1 215 bgt a2, zero, 58b 216 217.Lexit: 218 move a0, a2 219 jr ra 220 221 _asm_extable 0b, .Lsmall_fixup 222 _asm_extable 1b, .Lsmall_fixup 223 _asm_extable 2b, .Llarge_fixup 224 _asm_extable 3b, .Llarge_fixup 225 _asm_extable 4b, .Llarge_fixup 226 _asm_extable 5b, .Llarge_fixup 227 _asm_extable 6b, .Llarge_fixup 228 _asm_extable 7b, .Llarge_fixup 229 _asm_extable 8b, .Llarge_fixup 230 _asm_extable 9b, .Llarge_fixup 231 _asm_extable 10b, .Llarge_fixup 232 _asm_extable 11b, .Llarge_fixup 233 _asm_extable 12b, .Llarge_fixup 234 _asm_extable 13b, .Llarge_fixup 235 _asm_extable 14b, .Llarge_fixup 236 _asm_extable 15b, .Llarge_fixup 237 _asm_extable 16b, .Llarge_fixup 238 _asm_extable 17b, .Llarge_fixup 239 _asm_extable 18b, .Llarge_fixup 240 _asm_extable 19b, .Llarge_fixup 241 _asm_extable 20b, .Llarge_fixup 242 _asm_extable 21b, .Llarge_fixup 243 _asm_extable 22b, .Llarge_fixup 244 _asm_extable 23b, .Llarge_fixup 245 _asm_extable 24b, .Llarge_fixup 246 _asm_extable 25b, .Llarge_fixup 247 _asm_extable 26b, .Llarge_fixup 248 _asm_extable 27b, .Llarge_fixup 249 _asm_extable 28b, .Llarge_fixup 250 _asm_extable 29b, .Llarge_fixup 251 _asm_extable 30b, .Llarge_fixup 252 _asm_extable 31b, .Llarge_fixup 253 _asm_extable 32b, .Llarge_fixup 254 _asm_extable 33b, .Llarge_fixup 255 _asm_extable 34b, .Lexit 256 _asm_extable 35b, .Lexit 257 _asm_extable 36b, .Lsmall_fixup 258 _asm_extable 37b, .Lsmall_fixup 259 _asm_extable 38b, .Lsmall_fixup 260 _asm_extable 39b, .Lsmall_fixup 261 _asm_extable 40b, .Lsmall_fixup 262 _asm_extable 41b, .Lsmall_fixup 263 _asm_extable 42b, .Lsmall_fixup 264 _asm_extable 43b, .Lsmall_fixup 265 _asm_extable 44b, .Lsmall_fixup 266 _asm_extable 45b, .Lsmall_fixup 267 _asm_extable 46b, .Lsmall_fixup 268 _asm_extable 47b, .Lsmall_fixup 269 _asm_extable 48b, .Lsmall_fixup 270 _asm_extable 49b, .Lsmall_fixup 271 _asm_extable 50b, .Lsmall_fixup 272 _asm_extable 51b, .Lsmall_fixup 273 _asm_extable 52b, .Lsmall_fixup 274 _asm_extable 53b, .Lsmall_fixup 275 _asm_extable 54b, .Lsmall_fixup 276 _asm_extable 55b, .Lsmall_fixup 277 _asm_extable 56b, .Lsmall_fixup 278 _asm_extable 57b, .Lsmall_fixup 279 _asm_extable 58b, .Lexit 280 _asm_extable 59b, .Lexit 281SYM_FUNC_END(__copy_user_fast) 282 283STACK_FRAME_NON_STANDARD __copy_user_fast 284