1#include <linux/linkage.h> 2#include <linux/export.h> 3#include <asm/asm.h> 4#include <asm/asm-extable.h> 5#include <asm/csr.h> 6#include <asm/hwcap.h> 7#include <asm/alternative-macros.h> 8 9 .macro fixup op reg addr lbl 10100: 11 \op \reg, \addr 12 _asm_extable 100b, \lbl 13 .endm 14 15SYM_FUNC_START(__asm_copy_to_user) 16#ifdef CONFIG_RISCV_ISA_V 17 ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) 18 REG_L t0, riscv_v_usercopy_threshold 19 bltu a2, t0, fallback_scalar_usercopy 20 li a3, 1 21 tail enter_vector_usercopy 22#endif 23SYM_FUNC_END(__asm_copy_to_user) 24EXPORT_SYMBOL(__asm_copy_to_user) 25SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) 26EXPORT_SYMBOL(__asm_copy_from_user) 27 28SYM_FUNC_START(fallback_scalar_usercopy) 29 /* Enable access to user memory */ 30 li t6, SR_SUM 31 csrs CSR_STATUS, t6 32 mv t6, ra 33 34 call fallback_scalar_usercopy_sum_enabled 35 36 /* Disable access to user memory */ 37 mv ra, t6 38 li t6, SR_SUM 39 csrc CSR_STATUS, t6 40 ret 41SYM_FUNC_END(fallback_scalar_usercopy) 42 43SYM_FUNC_START(__asm_copy_to_user_sum_enabled) 44#ifdef CONFIG_RISCV_ISA_V 45 ALTERNATIVE("j fallback_scalar_usercopy_sum_enabled", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V) 46 REG_L t0, riscv_v_usercopy_threshold 47 bltu a2, t0, fallback_scalar_usercopy_sum_enabled 48 li a3, 0 49 tail enter_vector_usercopy 50#endif 51SYM_FUNC_END(__asm_copy_to_user_sum_enabled) 52SYM_FUNC_ALIAS(__asm_copy_from_user_sum_enabled, __asm_copy_to_user_sum_enabled) 53EXPORT_SYMBOL(__asm_copy_from_user_sum_enabled) 54EXPORT_SYMBOL(__asm_copy_to_user_sum_enabled) 55 56SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) 57 /* 58 * Save the terminal address which will be used to compute the number 59 * of bytes copied in case of a fixup exception. 60 */ 61 add t5, a0, a2 62 63 /* 64 * Register allocation for code below: 65 * a0 - start of uncopied dst 66 * a1 - start of uncopied src 67 * a2 - size 68 * t0 - end of uncopied dst 69 */ 70 add t0, a0, a2 71 72 /* 73 * Use byte copy only if too small. 74 * SZREG holds 4 for RV32 and 8 for RV64 75 */ 76 li a3, 9*SZREG-1 /* size must >= (word_copy stride + SZREG-1) */ 77 bltu a2, a3, .Lbyte_copy_tail 78 79 /* 80 * Copy first bytes until dst is aligned to word boundary. 81 * a0 - start of dst 82 * t1 - start of aligned dst 83 */ 84 addi t1, a0, SZREG-1 85 andi t1, t1, ~(SZREG-1) 86 /* dst is already aligned, skip */ 87 beq a0, t1, .Lskip_align_dst 881: 89 /* a5 - one byte for copying data */ 90 fixup lb a5, 0(a1), 10f 91 addi a1, a1, 1 /* src */ 92 fixup sb a5, 0(a0), 10f 93 addi a0, a0, 1 /* dst */ 94 bltu a0, t1, 1b /* t1 - start of aligned dst */ 95 96.Lskip_align_dst: 97 /* 98 * Now dst is aligned. 99 * Use shift-copy if src is misaligned. 100 * Use word-copy if both src and dst are aligned because 101 * can not use shift-copy which do not require shifting 102 */ 103 /* a1 - start of src */ 104 andi a3, a1, SZREG-1 105 bnez a3, .Lshift_copy 106 107.Lword_copy: 108 /* 109 * Both src and dst are aligned, unrolled word copy 110 * 111 * a0 - start of aligned dst 112 * a1 - start of aligned src 113 * t0 - end of aligned dst 114 */ 115 addi t0, t0, -(8*SZREG) /* not to over run */ 1162: 117 fixup REG_L a4, 0(a1), 10f 118 fixup REG_L a5, SZREG(a1), 10f 119 fixup REG_L a6, 2*SZREG(a1), 10f 120 fixup REG_L a7, 3*SZREG(a1), 10f 121 fixup REG_L t1, 4*SZREG(a1), 10f 122 fixup REG_L t2, 5*SZREG(a1), 10f 123 fixup REG_L t3, 6*SZREG(a1), 10f 124 fixup REG_L t4, 7*SZREG(a1), 10f 125 fixup REG_S a4, 0(a0), 10f 126 fixup REG_S a5, SZREG(a0), 10f 127 fixup REG_S a6, 2*SZREG(a0), 10f 128 fixup REG_S a7, 3*SZREG(a0), 10f 129 fixup REG_S t1, 4*SZREG(a0), 10f 130 fixup REG_S t2, 5*SZREG(a0), 10f 131 fixup REG_S t3, 6*SZREG(a0), 10f 132 fixup REG_S t4, 7*SZREG(a0), 10f 133 addi a0, a0, 8*SZREG 134 addi a1, a1, 8*SZREG 135 bleu a0, t0, 2b 136 137 addi t0, t0, 8*SZREG /* revert to original value */ 138 j .Lbyte_copy_tail 139 140.Lshift_copy: 141 142 /* 143 * Word copy with shifting. 144 * For misaligned copy we still perform aligned word copy, but 145 * we need to use the value fetched from the previous iteration and 146 * do some shifts. 147 * This is safe because reading is less than a word size. 148 * 149 * a0 - start of aligned dst 150 * a1 - start of src 151 * a3 - a1 & mask:(SZREG-1) 152 * t0 - end of uncopied dst 153 * t1 - end of aligned dst 154 */ 155 /* calculating aligned word boundary for dst */ 156 andi t1, t0, ~(SZREG-1) 157 /* Converting unaligned src to aligned src */ 158 andi a1, a1, ~(SZREG-1) 159 160 /* 161 * Calculate shifts 162 * t3 - prev shift 163 * t4 - current shift 164 */ 165 slli t3, a3, 3 /* converting bytes in a3 to bits */ 166 li a5, SZREG*8 167 sub t4, a5, t3 168 169 /* Load the first word to combine with second word */ 170 fixup REG_L a5, 0(a1), 10f 171 1723: 173 /* Main shifting copy 174 * 175 * a0 - start of aligned dst 176 * a1 - start of aligned src 177 * t1 - end of aligned dst 178 */ 179 180 /* At least one iteration will be executed */ 181 srl a4, a5, t3 182 fixup REG_L a5, SZREG(a1), 10f 183 addi a1, a1, SZREG 184 sll a2, a5, t4 185 or a2, a2, a4 186 fixup REG_S a2, 0(a0), 10f 187 addi a0, a0, SZREG 188 bltu a0, t1, 3b 189 190 /* Revert src to original unaligned value */ 191 add a1, a1, a3 192 193.Lbyte_copy_tail: 194 /* 195 * Byte copy anything left. 196 * 197 * a0 - start of remaining dst 198 * a1 - start of remaining src 199 * t0 - end of remaining dst 200 */ 201 bgeu a0, t0, .Lout_copy_user /* check if end of copy */ 2024: 203 fixup lb a5, 0(a1), 10f 204 addi a1, a1, 1 /* src */ 205 fixup sb a5, 0(a0), 10f 206 addi a0, a0, 1 /* dst */ 207 bltu a0, t0, 4b /* t0 - end of dst */ 208 209.Lout_copy_user: 210 li a0, 0 211 ret 21210: 213 sub a0, t5, a0 214 ret 215SYM_FUNC_END(fallback_scalar_usercopy_sum_enabled) 216 217SYM_FUNC_START(__clear_user) 218 219 /* Enable access to user memory */ 220 li t6, SR_SUM 221 csrs CSR_STATUS, t6 222 223 add a3, a0, a1 224 addi t0, a0, SZREG-1 225 andi t1, a3, ~(SZREG-1) 226 andi t0, t0, ~(SZREG-1) 227 /* 228 * a3: terminal address of target region 229 * t0: lowest doubleword-aligned address in target region 230 * t1: highest doubleword-aligned address in target region 231 */ 232 bgeu t0, t1, 2f 233 bltu a0, t0, 4f 2341: 235 fixup REG_S, zero, (a0), 11f 236 addi a0, a0, SZREG 237 bltu a0, t1, 1b 2382: 239 bltu a0, a3, 5f 240 2413: 242 /* Disable access to user memory */ 243 csrc CSR_STATUS, t6 244 li a0, 0 245 ret 2464: /* Edge case: unalignment */ 247 fixup sb, zero, (a0), 11f 248 addi a0, a0, 1 249 bltu a0, t0, 4b 250 j 1b 2515: /* Edge case: remainder */ 252 fixup sb, zero, (a0), 11f 253 addi a0, a0, 1 254 bltu a0, a3, 5b 255 j 3b 256 257 /* Exception fixup code */ 25811: 259 /* Disable access to user memory */ 260 csrc CSR_STATUS, t6 261 sub a0, a3, a0 262 ret 263SYM_FUNC_END(__clear_user) 264EXPORT_SYMBOL(__clear_user) 265