1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #ifndef CK_PR_PPC64_H 28 #define CK_PR_PPC64_H 29 30 #ifndef CK_PR_H 31 #error Do not include this file directly, use ck_pr.h 32 #endif 33 34 #include <ck_cc.h> 35 #include <ck_md.h> 36 37 /* 38 * The following represent supported atomic operations. 39 * These operations may be emulated. 40 */ 41 #include "ck_f_pr.h" 42 43 /* 44 * Minimum interface requirement met. 45 */ 46 #define CK_F_PR 47 48 /* 49 * This bounces the hardware thread from low to medium 50 * priority. I am unsure of the benefits of this approach 51 * but it is used by the Linux kernel. 52 */ 53 CK_CC_INLINE static void 54 ck_pr_stall(void) 55 { 56 57 __asm__ __volatile__("or 1, 1, 1;" 58 "or 2, 2, 2;" ::: "memory"); 59 return; 60 } 61 62 #define CK_PR_FENCE(T, I) \ 63 CK_CC_INLINE static void \ 64 ck_pr_fence_strict_##T(void) \ 65 { \ 66 __asm__ __volatile__(I ::: "memory"); \ 67 } 68 69 /* 70 * These are derived from: 71 * http://www.ibm.com/developerworks/systems/articles/powerpc.html 72 */ 73 CK_PR_FENCE(atomic, "lwsync") 74 CK_PR_FENCE(atomic_store, "lwsync") 75 CK_PR_FENCE(atomic_load, "sync") 76 CK_PR_FENCE(store_atomic, "lwsync") 77 CK_PR_FENCE(load_atomic, "lwsync") 78 CK_PR_FENCE(store, "lwsync") 79 CK_PR_FENCE(store_load, "sync") 80 CK_PR_FENCE(load, "lwsync") 81 CK_PR_FENCE(load_store, "lwsync") 82 CK_PR_FENCE(memory, "sync") 83 CK_PR_FENCE(acquire, "lwsync") 84 CK_PR_FENCE(release, "lwsync") 85 CK_PR_FENCE(acqrel, "lwsync") 86 CK_PR_FENCE(lock, "lwsync") 87 CK_PR_FENCE(unlock, "lwsync") 88 89 #undef CK_PR_FENCE 90 91 #define CK_PR_LOAD(S, M, T, C, I) \ 92 CK_CC_INLINE static T \ 93 ck_pr_md_load_##S(const M *target) \ 94 { \ 95 T r; \ 96 __asm__ __volatile__(I "%U1%X1 %0, %1" \ 97 : "=r" (r) \ 98 : "m" (*(const C *)target) \ 99 : "memory"); \ 100 return (r); \ 101 } 102 103 CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") 104 105 #define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 106 107 CK_PR_LOAD_S(64, uint64_t, "ld") 108 CK_PR_LOAD_S(32, uint32_t, "lwz") 109 CK_PR_LOAD_S(16, uint16_t, "lhz") 110 CK_PR_LOAD_S(8, uint8_t, "lbz") 111 CK_PR_LOAD_S(uint, unsigned int, "lwz") 112 CK_PR_LOAD_S(int, int, "lwz") 113 CK_PR_LOAD_S(short, short, "lhz") 114 CK_PR_LOAD_S(char, char, "lbz") 115 CK_PR_LOAD_S(double, double, "ld") 116 117 #undef CK_PR_LOAD_S 118 #undef CK_PR_LOAD 119 120 #define CK_PR_STORE(S, M, T, C, I) \ 121 CK_CC_INLINE static void \ 122 ck_pr_md_store_##S(M *target, T v) \ 123 { \ 124 __asm__ __volatile__(I "%U0%X0 %1, %0" \ 125 : "=m" (*(C *)target) \ 126 : "r" (v) \ 127 : "memory"); \ 128 return; \ 129 } 130 131 CK_PR_STORE(ptr, void, const void *, uint64_t, "std") 132 133 #define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 134 135 CK_PR_STORE_S(64, uint64_t, "std") 136 CK_PR_STORE_S(32, uint32_t, "stw") 137 CK_PR_STORE_S(16, uint16_t, "sth") 138 CK_PR_STORE_S(8, uint8_t, "stb") 139 CK_PR_STORE_S(uint, unsigned int, "stw") 140 CK_PR_STORE_S(int, int, "stw") 141 CK_PR_STORE_S(short, short, "sth") 142 CK_PR_STORE_S(char, char, "stb") 143 CK_PR_STORE_S(double, double, "std") 144 145 #undef CK_PR_STORE_S 146 #undef CK_PR_STORE 147 148 CK_CC_INLINE static bool 149 ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) 150 { 151 uint64_t previous; 152 153 __asm__ __volatile__("1:" 154 "ldarx %0, 0, %1;" 155 "cmpd 0, %0, %3;" 156 "bne- 2f;" 157 "stdcx. %2, 0, %1;" 158 "bne- 1b;" 159 "2:" 160 : "=&r" (previous) 161 : "r" (target), 162 "r" (set), 163 "r" (compare) 164 : "memory", "cc"); 165 166 *value = previous; 167 return (previous == compare); 168 } 169 170 CK_CC_INLINE static bool 171 ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) 172 { 173 void *previous; 174 175 __asm__ __volatile__("1:" 176 "ldarx %0, 0, %1;" 177 "cmpd 0, %0, %3;" 178 "bne- 2f;" 179 "stdcx. %2, 0, %1;" 180 "bne- 1b;" 181 "2:" 182 : "=&r" (previous) 183 : "r" (target), 184 "r" (set), 185 "r" (compare) 186 : "memory", "cc"); 187 188 ck_pr_md_store_ptr(value, previous); 189 return (previous == compare); 190 } 191 192 CK_CC_INLINE static bool 193 ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) 194 { 195 uint64_t previous; 196 197 __asm__ __volatile__("1:" 198 "ldarx %0, 0, %1;" 199 "cmpd 0, %0, %3;" 200 "bne- 2f;" 201 "stdcx. %2, 0, %1;" 202 "bne- 1b;" 203 "2:" 204 : "=&r" (previous) 205 : "r" (target), 206 "r" (set), 207 "r" (compare) 208 : "memory", "cc"); 209 210 return (previous == compare); 211 } 212 213 CK_CC_INLINE static bool 214 ck_pr_cas_ptr(void *target, void *compare, void *set) 215 { 216 void *previous; 217 218 __asm__ __volatile__("1:" 219 "ldarx %0, 0, %1;" 220 "cmpd 0, %0, %3;" 221 "bne- 2f;" 222 "stdcx. %2, 0, %1;" 223 "bne- 1b;" 224 "2:" 225 : "=&r" (previous) 226 : "r" (target), 227 "r" (set), 228 "r" (compare) 229 : "memory", "cc"); 230 231 return (previous == compare); 232 } 233 234 #define CK_PR_CAS(N, T) \ 235 CK_CC_INLINE static bool \ 236 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 237 { \ 238 T previous; \ 239 __asm__ __volatile__("1:" \ 240 "lwarx %0, 0, %1;" \ 241 "cmpw 0, %0, %3;" \ 242 "bne- 2f;" \ 243 "stwcx. %2, 0, %1;" \ 244 "bne- 1b;" \ 245 "2:" \ 246 : "=&r" (previous) \ 247 : "r" (target), \ 248 "r" (set), \ 249 "r" (compare) \ 250 : "memory", "cc"); \ 251 *value = previous; \ 252 return (previous == compare); \ 253 } \ 254 CK_CC_INLINE static bool \ 255 ck_pr_cas_##N(T *target, T compare, T set) \ 256 { \ 257 T previous; \ 258 __asm__ __volatile__("1:" \ 259 "lwarx %0, 0, %1;" \ 260 "cmpw 0, %0, %3;" \ 261 "bne- 2f;" \ 262 "stwcx. %2, 0, %1;" \ 263 "bne- 1b;" \ 264 "2:" \ 265 : "=&r" (previous) \ 266 : "r" (target), \ 267 "r" (set), \ 268 "r" (compare) \ 269 : "memory", "cc"); \ 270 return (previous == compare); \ 271 } 272 273 CK_PR_CAS(32, uint32_t) 274 CK_PR_CAS(uint, unsigned int) 275 CK_PR_CAS(int, int) 276 277 #undef CK_PR_CAS 278 279 #define CK_PR_FAS(N, M, T, W) \ 280 CK_CC_INLINE static T \ 281 ck_pr_fas_##N(M *target, T v) \ 282 { \ 283 T previous; \ 284 __asm__ __volatile__("1:" \ 285 "l" W "arx %0, 0, %1;" \ 286 "st" W "cx. %2, 0, %1;" \ 287 "bne- 1b;" \ 288 : "=&r" (previous) \ 289 : "r" (target), \ 290 "r" (v) \ 291 : "memory", "cc"); \ 292 return (previous); \ 293 } 294 295 CK_PR_FAS(64, uint64_t, uint64_t, "d") 296 CK_PR_FAS(32, uint32_t, uint32_t, "w") 297 CK_PR_FAS(double, double, double, "d") 298 CK_PR_FAS(ptr, void, void *, "d") 299 CK_PR_FAS(int, int, int, "w") 300 CK_PR_FAS(uint, unsigned int, unsigned int, "w") 301 302 #undef CK_PR_FAS 303 304 #define CK_PR_UNARY(O, N, M, T, I, W) \ 305 CK_CC_INLINE static void \ 306 ck_pr_##O##_##N(M *target) \ 307 { \ 308 T previous; \ 309 __asm__ __volatile__("1:" \ 310 "l" W "arx %0, 0, %1;" \ 311 I ";" \ 312 "st" W "cx. %0, 0, %1;" \ 313 "bne- 1b;" \ 314 : "=&r" (previous) \ 315 : "r" (target) \ 316 : "memory", "cc"); \ 317 return; \ 318 } 319 320 CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") 321 CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") 322 CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") 323 CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") 324 325 #define CK_PR_UNARY_S(S, T, W) \ 326 CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ 327 CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ 328 CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ 329 CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) 330 331 CK_PR_UNARY_S(64, uint64_t, "d") 332 CK_PR_UNARY_S(32, uint32_t, "w") 333 CK_PR_UNARY_S(uint, unsigned int, "w") 334 CK_PR_UNARY_S(int, int, "w") 335 336 #undef CK_PR_UNARY_S 337 #undef CK_PR_UNARY 338 339 #define CK_PR_BINARY(O, N, M, T, I, W) \ 340 CK_CC_INLINE static void \ 341 ck_pr_##O##_##N(M *target, T delta) \ 342 { \ 343 T previous; \ 344 __asm__ __volatile__("1:" \ 345 "l" W "arx %0, 0, %1;" \ 346 I " %0, %2, %0;" \ 347 "st" W "cx. %0, 0, %1;" \ 348 "bne- 1b;" \ 349 : "=&r" (previous) \ 350 : "r" (target), \ 351 "r" (delta) \ 352 : "memory", "cc"); \ 353 return; \ 354 } 355 356 CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") 357 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") 358 CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") 359 CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") 360 CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") 361 362 #define CK_PR_BINARY_S(S, T, W) \ 363 CK_PR_BINARY(and, S, T, T, "and", W) \ 364 CK_PR_BINARY(add, S, T, T, "add", W) \ 365 CK_PR_BINARY(or, S, T, T, "or", W) \ 366 CK_PR_BINARY(sub, S, T, T, "subf", W) \ 367 CK_PR_BINARY(xor, S, T, T, "xor", W) 368 369 CK_PR_BINARY_S(64, uint64_t, "d") 370 CK_PR_BINARY_S(32, uint32_t, "w") 371 CK_PR_BINARY_S(uint, unsigned int, "w") 372 CK_PR_BINARY_S(int, int, "w") 373 374 #undef CK_PR_BINARY_S 375 #undef CK_PR_BINARY 376 377 CK_CC_INLINE static void * 378 ck_pr_faa_ptr(void *target, uintptr_t delta) 379 { 380 uintptr_t previous, r; 381 382 __asm__ __volatile__("1:" 383 "ldarx %0, 0, %2;" 384 "add %1, %3, %0;" 385 "stdcx. %1, 0, %2;" 386 "bne- 1b;" 387 : "=&r" (previous), 388 "=&r" (r) 389 : "r" (target), 390 "r" (delta) 391 : "memory", "cc"); 392 393 return (void *)(previous); 394 } 395 396 #define CK_PR_FAA(S, T, W) \ 397 CK_CC_INLINE static T \ 398 ck_pr_faa_##S(T *target, T delta) \ 399 { \ 400 T previous, r; \ 401 __asm__ __volatile__("1:" \ 402 "l" W "arx %0, 0, %2;" \ 403 "add %1, %3, %0;" \ 404 "st" W "cx. %1, 0, %2;" \ 405 "bne- 1b;" \ 406 : "=&r" (previous), \ 407 "=&r" (r) \ 408 : "r" (target), \ 409 "r" (delta) \ 410 : "memory", "cc"); \ 411 return (previous); \ 412 } 413 414 CK_PR_FAA(64, uint64_t, "d") 415 CK_PR_FAA(32, uint32_t, "w") 416 CK_PR_FAA(uint, unsigned int, "w") 417 CK_PR_FAA(int, int, "w") 418 419 #undef CK_PR_FAA 420 421 #endif /* CK_PR_PPC64_H */ 422