1#include <linux/linkage.h> 2#include <linux/export.h> 3#include <asm/asm.h> 4#include <asm/asm-extable.h> 5#include <asm/csr.h> 6#include <asm/hwcap.h> 7#include <asm/alternative-macros.h> 8 9 .macro fixup op reg addr lbl 10100: 11 \op \reg, \addr 12 _asm_extable 100b, \lbl 13 .endm 14 15SYM_FUNC_START(__asm_copy_to_user) 16#ifdef CONFIG_RISCV_ISA_V 17 ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) 18 REG_L t0, riscv_v_usercopy_threshold 19 bltu a2, t0, fallback_scalar_usercopy 20 li a3, 1 21 tail enter_vector_usercopy 22#endif 23SYM_FUNC_END(__asm_copy_to_user) 24EXPORT_SYMBOL(__asm_copy_to_user) 25SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) 26EXPORT_SYMBOL(__asm_copy_from_user) 27 28SYM_FUNC_START(fallback_scalar_usercopy) 29 /* Enable access to user memory */ 30 li t6, SR_SUM 31 csrs CSR_STATUS, t6 32 mv t6, ra 33 34 call fallback_scalar_usercopy_sum_enabled 35 36 /* Disable access to user memory */ 37 mv ra, t6 38 li t6, SR_SUM 39 csrc CSR_STATUS, t6 40 ret 41SYM_FUNC_END(fallback_scalar_usercopy) 42 43SYM_FUNC_START(__asm_copy_to_user_sum_enabled) 44#ifdef CONFIG_RISCV_ISA_V 45 ALTERNATIVE("j fallback_scalar_usercopy_sum_enabled", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) 46 REG_L t0, riscv_v_usercopy_threshold 47 bltu a2, t0, fallback_scalar_usercopy_sum_enabled 48 li a3, 0 49 tail enter_vector_usercopy 50#endif 51SYM_FUNC_END(__asm_copy_to_user_sum_enabled) 52SYM_FUNC_ALIAS(__asm_copy_from_user_sum_enabled, __asm_copy_to_user_sum_enabled) 53EXPORT_SYMBOL(__asm_copy_from_user_sum_enabled) 54EXPORT_SYMBOL(__asm_copy_to_user_sum_enabled) 55 56SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) 57 /* 58 * Save the terminal address which will be used to compute the number 59 * of bytes copied in case of a fixup exception. 60 */ 61 add t5, a0, a2 62 63 /* 64 * Register allocation for code below: 65 * a0 - start of uncopied dst 66 * a1 - start of uncopied src 67 * a2 - size 68 * t0 - end of uncopied dst 69 */ 70 add t0, a0, a2 71 72 /* 73 * Use byte copy only if too small. 74 * SZREG holds 4 for RV32 and 8 for RV64 75 */ 76 li a3, 9*SZREG-1 /* size must >= (word_copy stride + SZREG-1) */ 77 bltu a2, a3, .Lbyte_copy_tail 78 79#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) 80 /* 81 * Copy first bytes until dst is aligned to word boundary. 82 * a0 - start of dst 83 * t1 - start of aligned dst 84 */ 85 addi t1, a0, SZREG-1 86 andi t1, t1, ~(SZREG-1) 87 /* dst is already aligned, skip */ 88 beq a0, t1, .Lskip_align_dst 891: 90 /* a5 - one byte for copying data */ 91 fixup lb a5, 0(a1), 10f 92 addi a1, a1, 1 /* src */ 93 fixup sb a5, 0(a0), 10f 94 addi a0, a0, 1 /* dst */ 95 bltu a0, t1, 1b /* t1 - start of aligned dst */ 96 97.Lskip_align_dst: 98 /* 99 * Now dst is aligned. 100 * Use shift-copy if src is misaligned. 101 * Use word-copy if both src and dst are aligned because 102 * can not use shift-copy which do not require shifting 103 */ 104 /* a1 - start of src */ 105 andi a3, a1, SZREG-1 106 bnez a3, .Lshift_copy 107#endif 108.Lword_copy: 109 /* 110 * Both src and dst are aligned, unrolled word copy 111 * 112 * a0 - start of aligned dst 113 * a1 - start of aligned src 114 * t0 - end of aligned dst 115 */ 116 addi t0, t0, -(8*SZREG) /* not to over run */ 1172: 118 fixup REG_L a4, 0(a1), 10f 119 fixup REG_L a5, SZREG(a1), 10f 120 fixup REG_L a6, 2*SZREG(a1), 10f 121 fixup REG_L a7, 3*SZREG(a1), 10f 122 fixup REG_L t1, 4*SZREG(a1), 10f 123 fixup REG_L t2, 5*SZREG(a1), 10f 124 fixup REG_L t3, 6*SZREG(a1), 10f 125 fixup REG_L t4, 7*SZREG(a1), 10f 126 fixup REG_S a4, 0(a0), 10f 127 fixup REG_S a5, SZREG(a0), 10f 128 fixup REG_S a6, 2*SZREG(a0), 10f 129 fixup REG_S a7, 3*SZREG(a0), 10f 130 fixup REG_S t1, 4*SZREG(a0), 10f 131 fixup REG_S t2, 5*SZREG(a0), 10f 132 fixup REG_S t3, 6*SZREG(a0), 10f 133 fixup REG_S t4, 7*SZREG(a0), 10f 134 addi a0, a0, 8*SZREG 135 addi a1, a1, 8*SZREG 136 bleu a0, t0, 2b 137 138 addi t0, t0, 8*SZREG /* revert to original value */ 139 j .Lbyte_copy_tail 140 141#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) 142.Lshift_copy: 143 144 /* 145 * Word copy with shifting. 146 * For misaligned copy we still perform aligned word copy, but 147 * we need to use the value fetched from the previous iteration and 148 * do some shifts. 149 * This is safe because reading is less than a word size. 150 * 151 * a0 - start of aligned dst 152 * a1 - start of src 153 * a3 - a1 & mask:(SZREG-1) 154 * t0 - end of uncopied dst 155 * t1 - end of aligned dst 156 */ 157 /* calculating aligned word boundary for dst */ 158 andi t1, t0, ~(SZREG-1) 159 /* Converting unaligned src to aligned src */ 160 andi a1, a1, ~(SZREG-1) 161 162 /* 163 * Calculate shifts 164 * t3 - prev shift 165 * t4 - current shift 166 */ 167 slli t3, a3, 3 /* converting bytes in a3 to bits */ 168 li a5, SZREG*8 169 sub t4, a5, t3 170 171 /* Load the first word to combine with second word */ 172 fixup REG_L a5, 0(a1), 10f 173 1743: 175 /* Main shifting copy 176 * 177 * a0 - start of aligned dst 178 * a1 - start of aligned src 179 * t1 - end of aligned dst 180 */ 181 182 /* At least one iteration will be executed */ 183 srl a4, a5, t3 184 fixup REG_L a5, SZREG(a1), 10f 185 addi a1, a1, SZREG 186 sll a2, a5, t4 187 or a2, a2, a4 188 fixup REG_S a2, 0(a0), 10f 189 addi a0, a0, SZREG 190 bltu a0, t1, 3b 191 192 /* Revert src to original unaligned value */ 193 add a1, a1, a3 194#endif 195 196.Lbyte_copy_tail: 197 /* 198 * Byte copy anything left. 199 * 200 * a0 - start of remaining dst 201 * a1 - start of remaining src 202 * t0 - end of remaining dst 203 */ 204 bgeu a0, t0, .Lout_copy_user /* check if end of copy */ 2054: 206 fixup lb a5, 0(a1), 10f 207 addi a1, a1, 1 /* src */ 208 fixup sb a5, 0(a0), 10f 209 addi a0, a0, 1 /* dst */ 210 bltu a0, t0, 4b /* t0 - end of dst */ 211 212.Lout_copy_user: 213 li a0, 0 214 ret 21510: 216 sub a0, t5, a0 217 ret 218SYM_FUNC_END(fallback_scalar_usercopy_sum_enabled) 219 220SYM_FUNC_START(__clear_user) 221 222 /* Enable access to user memory */ 223 li t6, SR_SUM 224 csrs CSR_STATUS, t6 225 226 add a3, a0, a1 227 addi t0, a0, SZREG-1 228 andi t1, a3, ~(SZREG-1) 229 andi t0, t0, ~(SZREG-1) 230 /* 231 * a3: terminal address of target region 232 * t0: lowest doubleword-aligned address in target region 233 * t1: highest doubleword-aligned address in target region 234 */ 235 bgeu t0, t1, 2f 236 bltu a0, t0, 4f 2371: 238 fixup REG_S, zero, (a0), 11f 239 addi a0, a0, SZREG 240 bltu a0, t1, 1b 2412: 242 bltu a0, a3, 5f 243 2443: 245 /* Disable access to user memory */ 246 csrc CSR_STATUS, t6 247 li a0, 0 248 ret 2494: /* Edge case: unalignment */ 250 fixup sb, zero, (a0), 11f 251 addi a0, a0, 1 252 bltu a0, t0, 4b 253 j 1b 2545: /* Edge case: remainder */ 255 fixup sb, zero, (a0), 11f 256 addi a0, a0, 1 257 bltu a0, a3, 5b 258 j 3b 259 260 /* Exception fixup code */ 26111: 262 /* Disable access to user memory */ 263 csrc CSR_STATUS, t6 264 sub a0, a3, a0 265 ret 266SYM_FUNC_END(__clear_user) 267EXPORT_SYMBOL(__clear_user) 268