1#include <linux/linkage.h> 2#include <asm-generic/export.h> 3#include <asm/asm.h> 4#include <asm/asm-extable.h> 5#include <asm/csr.h> 6 7 .macro fixup op reg addr lbl 8100: 9 \op \reg, \addr 10 _asm_extable 100b, \lbl 11 .endm 12 13SYM_FUNC_START(__asm_copy_to_user) 14 15 /* Enable access to user memory */ 16 li t6, SR_SUM 17 csrs CSR_STATUS, t6 18 19 /* 20 * Save the terminal address which will be used to compute the number 21 * of bytes copied in case of a fixup exception. 22 */ 23 add t5, a0, a2 24 25 /* 26 * Register allocation for code below: 27 * a0 - start of uncopied dst 28 * a1 - start of uncopied src 29 * a2 - size 30 * t0 - end of uncopied dst 31 */ 32 add t0, a0, a2 33 34 /* 35 * Use byte copy only if too small. 36 * SZREG holds 4 for RV32 and 8 for RV64 37 */ 38 li a3, 9*SZREG /* size must be larger than size in word_copy */ 39 bltu a2, a3, .Lbyte_copy_tail 40 41 /* 42 * Copy first bytes until dst is aligned to word boundary. 43 * a0 - start of dst 44 * t1 - start of aligned dst 45 */ 46 addi t1, a0, SZREG-1 47 andi t1, t1, ~(SZREG-1) 48 /* dst is already aligned, skip */ 49 beq a0, t1, .Lskip_align_dst 501: 51 /* a5 - one byte for copying data */ 52 fixup lb a5, 0(a1), 10f 53 addi a1, a1, 1 /* src */ 54 fixup sb a5, 0(a0), 10f 55 addi a0, a0, 1 /* dst */ 56 bltu a0, t1, 1b /* t1 - start of aligned dst */ 57 58.Lskip_align_dst: 59 /* 60 * Now dst is aligned. 61 * Use shift-copy if src is misaligned. 62 * Use word-copy if both src and dst are aligned because 63 * can not use shift-copy which do not require shifting 64 */ 65 /* a1 - start of src */ 66 andi a3, a1, SZREG-1 67 bnez a3, .Lshift_copy 68 69.Lword_copy: 70 /* 71 * Both src and dst are aligned, unrolled word copy 72 * 73 * a0 - start of aligned dst 74 * a1 - start of aligned src 75 * t0 - end of aligned dst 76 */ 77 addi t0, t0, -(8*SZREG) /* not to over run */ 782: 79 fixup REG_L a4, 0(a1), 10f 80 fixup REG_L a5, SZREG(a1), 10f 81 fixup REG_L a6, 2*SZREG(a1), 10f 82 fixup REG_L a7, 3*SZREG(a1), 10f 83 fixup REG_L t1, 4*SZREG(a1), 10f 84 fixup REG_L t2, 5*SZREG(a1), 10f 85 fixup REG_L t3, 6*SZREG(a1), 10f 86 fixup REG_L t4, 7*SZREG(a1), 10f 87 fixup REG_S a4, 0(a0), 10f 88 fixup REG_S a5, SZREG(a0), 10f 89 fixup REG_S a6, 2*SZREG(a0), 10f 90 fixup REG_S a7, 3*SZREG(a0), 10f 91 fixup REG_S t1, 4*SZREG(a0), 10f 92 fixup REG_S t2, 5*SZREG(a0), 10f 93 fixup REG_S t3, 6*SZREG(a0), 10f 94 fixup REG_S t4, 7*SZREG(a0), 10f 95 addi a0, a0, 8*SZREG 96 addi a1, a1, 8*SZREG 97 bltu a0, t0, 2b 98 99 addi t0, t0, 8*SZREG /* revert to original value */ 100 j .Lbyte_copy_tail 101 102.Lshift_copy: 103 104 /* 105 * Word copy with shifting. 106 * For misaligned copy we still perform aligned word copy, but 107 * we need to use the value fetched from the previous iteration and 108 * do some shifts. 109 * This is safe because reading is less than a word size. 110 * 111 * a0 - start of aligned dst 112 * a1 - start of src 113 * a3 - a1 & mask:(SZREG-1) 114 * t0 - end of uncopied dst 115 * t1 - end of aligned dst 116 */ 117 /* calculating aligned word boundary for dst */ 118 andi t1, t0, ~(SZREG-1) 119 /* Converting unaligned src to aligned src */ 120 andi a1, a1, ~(SZREG-1) 121 122 /* 123 * Calculate shifts 124 * t3 - prev shift 125 * t4 - current shift 126 */ 127 slli t3, a3, 3 /* converting bytes in a3 to bits */ 128 li a5, SZREG*8 129 sub t4, a5, t3 130 131 /* Load the first word to combine with second word */ 132 fixup REG_L a5, 0(a1), 10f 133 1343: 135 /* Main shifting copy 136 * 137 * a0 - start of aligned dst 138 * a1 - start of aligned src 139 * t1 - end of aligned dst 140 */ 141 142 /* At least one iteration will be executed */ 143 srl a4, a5, t3 144 fixup REG_L a5, SZREG(a1), 10f 145 addi a1, a1, SZREG 146 sll a2, a5, t4 147 or a2, a2, a4 148 fixup REG_S a2, 0(a0), 10f 149 addi a0, a0, SZREG 150 bltu a0, t1, 3b 151 152 /* Revert src to original unaligned value */ 153 add a1, a1, a3 154 155.Lbyte_copy_tail: 156 /* 157 * Byte copy anything left. 158 * 159 * a0 - start of remaining dst 160 * a1 - start of remaining src 161 * t0 - end of remaining dst 162 */ 163 bgeu a0, t0, .Lout_copy_user /* check if end of copy */ 1644: 165 fixup lb a5, 0(a1), 10f 166 addi a1, a1, 1 /* src */ 167 fixup sb a5, 0(a0), 10f 168 addi a0, a0, 1 /* dst */ 169 bltu a0, t0, 4b /* t0 - end of dst */ 170 171.Lout_copy_user: 172 /* Disable access to user memory */ 173 csrc CSR_STATUS, t6 174 li a0, 0 175 ret 176 177 /* Exception fixup code */ 17810: 179 /* Disable access to user memory */ 180 csrc CSR_STATUS, t6 181 sub a0, t5, a0 182 ret 183SYM_FUNC_END(__asm_copy_to_user) 184EXPORT_SYMBOL(__asm_copy_to_user) 185SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user) 186EXPORT_SYMBOL(__asm_copy_from_user) 187 188 189SYM_FUNC_START(__clear_user) 190 191 /* Enable access to user memory */ 192 li t6, SR_SUM 193 csrs CSR_STATUS, t6 194 195 add a3, a0, a1 196 addi t0, a0, SZREG-1 197 andi t1, a3, ~(SZREG-1) 198 andi t0, t0, ~(SZREG-1) 199 /* 200 * a3: terminal address of target region 201 * t0: lowest doubleword-aligned address in target region 202 * t1: highest doubleword-aligned address in target region 203 */ 204 bgeu t0, t1, 2f 205 bltu a0, t0, 4f 2061: 207 fixup REG_S, zero, (a0), 11f 208 addi a0, a0, SZREG 209 bltu a0, t1, 1b 2102: 211 bltu a0, a3, 5f 212 2133: 214 /* Disable access to user memory */ 215 csrc CSR_STATUS, t6 216 li a0, 0 217 ret 2184: /* Edge case: unalignment */ 219 fixup sb, zero, (a0), 11f 220 addi a0, a0, 1 221 bltu a0, t0, 4b 222 j 1b 2235: /* Edge case: remainder */ 224 fixup sb, zero, (a0), 11f 225 addi a0, a0, 1 226 bltu a0, a3, 5b 227 j 3b 228 229 /* Exception fixup code */ 23011: 231 /* Disable access to user memory */ 232 csrc CSR_STATUS, t6 233 sub a0, a3, a0 234 ret 235SYM_FUNC_END(__clear_user) 236EXPORT_SYMBOL(__clear_user) 237