1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2023 SiFive
4 * Author: Andy Chiu <andy.chiu@sifive.com>
5 */
6 #include <linux/linkage.h>
7 #include <asm/asm.h>
8
9 #include <asm/vector.h>
10 #include <asm/simd.h>
11
12 #ifdef CONFIG_MMU
13 #include <asm/asm-prototypes.h>
14 #endif
15
16 #ifdef CONFIG_MMU
17 size_t riscv_v_usercopy_threshold = CONFIG_RISCV_ISA_V_UCOPY_THRESHOLD;
18 int __asm_vector_usercopy(void *dst, void *src, size_t n);
19 int __asm_vector_usercopy_sum_enabled(void *dst, void *src, size_t n);
20 int fallback_scalar_usercopy(void *dst, void *src, size_t n);
21 int fallback_scalar_usercopy_sum_enabled(void *dst, void *src, size_t n);
enter_vector_usercopy(void * dst,void * src,size_t n,bool enable_sum)22 asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n,
23 bool enable_sum)
24 {
25 size_t remain, copied;
26
27 /* skip has_vector() check because it has been done by the asm */
28 if (!may_use_simd())
29 goto fallback;
30
31 kernel_vector_begin();
32 remain = enable_sum ? __asm_vector_usercopy(dst, src, n) :
33 __asm_vector_usercopy_sum_enabled(dst, src, n);
34 kernel_vector_end();
35
36 if (remain) {
37 copied = n - remain;
38 dst += copied;
39 src += copied;
40 n = remain;
41 goto fallback;
42 }
43
44 return remain;
45
46 fallback:
47 return enable_sum ? fallback_scalar_usercopy(dst, src, n) :
48 fallback_scalar_usercopy_sum_enabled(dst, src, n);
49 }
50 #endif
51