1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #ifndef __ASM_PERCPU_H 6 #define __ASM_PERCPU_H 7 8 #include <asm/loongarch.h> 9 10 /* 11 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to 12 * the loading address of main kernel image, but far from where the modules are 13 * loaded. Tell the compiler this fact when using explicit relocs. 14 */ 15 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) && defined(CONFIG_64BIT) 16 # if __has_attribute(model) 17 # define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) 18 # else 19 # error compiler support for the model attribute is necessary when a recent assembler is used 20 # endif 21 #endif 22 23 /* Use r21 for fast access */ 24 register unsigned long __my_cpu_offset __asm__("$r21"); 25 26 static inline void set_my_cpu_offset(unsigned long off) 27 { 28 __my_cpu_offset = off; 29 csr_write(off, PERCPU_BASE_KS); 30 } 31 32 #define __my_cpu_offset \ 33 ({ \ 34 __asm__ __volatile__("":"+r"(__my_cpu_offset)); \ 35 __my_cpu_offset; \ 36 }) 37 38 #ifdef CONFIG_CPU_HAS_AMO 39 40 #define PERCPU_OP(op, asm_op, c_op) \ 41 static __always_inline unsigned long __percpu_##op(void *ptr, \ 42 unsigned long val, int size) \ 43 { \ 44 unsigned long ret; \ 45 \ 46 switch (size) { \ 47 case 4: \ 48 __asm__ __volatile__( \ 49 "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \ 50 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \ 51 : [val] "r" (val)); \ 52 break; \ 53 case 8: \ 54 __asm__ __volatile__( \ 55 "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \ 56 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \ 57 : [val] "r" (val)); \ 58 break; \ 59 default: \ 60 ret = 0; \ 61 BUILD_BUG(); \ 62 } \ 63 \ 64 return ret c_op val; \ 65 } 66 67 PERCPU_OP(add, add, +) 68 PERCPU_OP(and, and, &) 69 PERCPU_OP(or, or, |) 70 #undef PERCPU_OP 71 72 #endif 73 74 #ifdef CONFIG_64BIT 75 76 #define __pcpu_op_1(op) op ".b " 77 #define __pcpu_op_2(op) op ".h " 78 #define __pcpu_op_4(op) op ".w " 79 #define __pcpu_op_8(op) op ".d " 80 81 #define _percpu_read(size, _pcp) \ 82 ({ \ 83 typeof(_pcp) __pcp_ret; \ 84 \ 85 __asm__ __volatile__( \ 86 __pcpu_op_##size("ldx") "%[ret], $r21, %[ptr] \n" \ 87 : [ret] "=&r"(__pcp_ret) \ 88 : [ptr] "r"(&(_pcp)) \ 89 : "memory"); \ 90 \ 91 __pcp_ret; \ 92 }) 93 94 #define _percpu_write(size, _pcp, _val) \ 95 do { \ 96 __asm__ __volatile__( \ 97 __pcpu_op_##size("stx") "%[val], $r21, %[ptr] \n" \ 98 : \ 99 : [val] "r"(_val), [ptr] "r"(&(_pcp)) \ 100 : "memory"); \ 101 } while (0) 102 103 #endif 104 105 #define __percpu_xchg __arch_xchg 106 107 /* this_cpu_cmpxchg */ 108 #define _protect_cmpxchg_local(pcp, o, n) \ 109 ({ \ 110 typeof(*raw_cpu_ptr(&(pcp))) __ret; \ 111 preempt_disable_notrace(); \ 112 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ 113 preempt_enable_notrace(); \ 114 __ret; \ 115 }) 116 117 #define _pcp_protect(operation, pcp, val) \ 118 ({ \ 119 typeof(pcp) __retval; \ 120 preempt_disable_notrace(); \ 121 __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ 122 (val), sizeof(pcp)); \ 123 preempt_enable_notrace(); \ 124 __retval; \ 125 }) 126 127 #ifdef CONFIG_CPU_HAS_AMO 128 129 #define _percpu_add(pcp, val) \ 130 _pcp_protect(__percpu_add, pcp, val) 131 132 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val) 133 134 #define _percpu_and(pcp, val) \ 135 _pcp_protect(__percpu_and, pcp, val) 136 137 #define _percpu_or(pcp, val) \ 138 _pcp_protect(__percpu_or, pcp, val) 139 140 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) 141 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) 142 143 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) 144 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) 145 146 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) 147 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) 148 149 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) 150 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) 151 152 #endif 153 154 #ifdef CONFIG_64BIT 155 156 #define this_cpu_read_1(pcp) _percpu_read(1, pcp) 157 #define this_cpu_read_2(pcp) _percpu_read(2, pcp) 158 #define this_cpu_read_4(pcp) _percpu_read(4, pcp) 159 #define this_cpu_read_8(pcp) _percpu_read(8, pcp) 160 161 #define this_cpu_write_1(pcp, val) _percpu_write(1, pcp, val) 162 #define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val) 163 #define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val) 164 #define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val) 165 166 #endif 167 168 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \ 169 _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) 170 171 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) 172 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) 173 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) 174 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) 175 176 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 177 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 178 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 179 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 180 181 #include <asm-generic/percpu.h> 182 183 #endif /* __ASM_PERCPU_H */ 184