1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #ifndef CK_PR_PPC64_H 28 #define CK_PR_PPC64_H 29 30 #ifndef CK_PR_H 31 #error Do not include this file directly, use ck_pr.h 32 #endif 33 34 #include <ck_cc.h> 35 #include <ck_md.h> 36 37 /* 38 * The following represent supported atomic operations. 39 * These operations may be emulated. 40 */ 41 #include "ck_f_pr.h" 42 43 /* 44 * Minimum interface requirement met. 45 */ 46 #define CK_F_PR 47 48 /* 49 * This bounces the hardware thread from low to medium 50 * priority. I am unsure of the benefits of this approach 51 * but it is used by the Linux kernel. 52 */ 53 CK_CC_INLINE static void 54 ck_pr_stall(void) 55 { 56 57 __asm__ __volatile__("or 1, 1, 1;" 58 "or 2, 2, 2;" ::: "memory"); 59 return; 60 } 61 62 #define CK_PR_FENCE(T, I) \ 63 CK_CC_INLINE static void \ 64 ck_pr_fence_strict_##T(void) \ 65 { \ 66 __asm__ __volatile__(I ::: "memory"); \ 67 } 68 69 /* 70 * These are derived from: 71 * http://www.ibm.com/developerworks/systems/articles/powerpc.html 72 */ 73 CK_PR_FENCE(atomic, "lwsync") 74 CK_PR_FENCE(atomic_store, "lwsync") 75 CK_PR_FENCE(atomic_load, "sync") 76 CK_PR_FENCE(store_atomic, "lwsync") 77 CK_PR_FENCE(load_atomic, "lwsync") 78 CK_PR_FENCE(store, "lwsync") 79 CK_PR_FENCE(store_load, "sync") 80 CK_PR_FENCE(load, "lwsync") 81 CK_PR_FENCE(load_store, "lwsync") 82 CK_PR_FENCE(memory, "sync") 83 CK_PR_FENCE(acquire, "lwsync") 84 CK_PR_FENCE(release, "lwsync") 85 CK_PR_FENCE(acqrel, "lwsync") 86 CK_PR_FENCE(lock, "lwsync") 87 CK_PR_FENCE(unlock, "lwsync") 88 89 #undef CK_PR_FENCE 90 91 #define CK_PR_LOAD(S, M, T, C, I) \ 92 CK_CC_INLINE static T \ 93 ck_pr_md_load_##S(const M *target) \ 94 { \ 95 T r; \ 96 __asm__ __volatile__(I "%U1%X1 %0, %1" \ 97 : "=r" (r) \ 98 : "m" (*(const C *)target) \ 99 : "memory"); \ 100 return (r); \ 101 } 102 103 CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") 104 105 #define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 106 107 CK_PR_LOAD_S(64, uint64_t, "ld") 108 CK_PR_LOAD_S(32, uint32_t, "lwz") 109 CK_PR_LOAD_S(16, uint16_t, "lhz") 110 CK_PR_LOAD_S(8, uint8_t, "lbz") 111 CK_PR_LOAD_S(uint, unsigned int, "lwz") 112 CK_PR_LOAD_S(int, int, "lwz") 113 CK_PR_LOAD_S(short, short, "lhz") 114 CK_PR_LOAD_S(char, char, "lbz") 115 #ifndef CK_PR_DISABLE_DOUBLE 116 CK_PR_LOAD_S(double, double, "ld") 117 #endif 118 119 #undef CK_PR_LOAD_S 120 #undef CK_PR_LOAD 121 122 #define CK_PR_STORE(S, M, T, C, I) \ 123 CK_CC_INLINE static void \ 124 ck_pr_md_store_##S(M *target, T v) \ 125 { \ 126 __asm__ __volatile__(I "%U0%X0 %1, %0" \ 127 : "=m" (*(C *)target) \ 128 : "r" (v) \ 129 : "memory"); \ 130 return; \ 131 } 132 133 CK_PR_STORE(ptr, void, const void *, uint64_t, "std") 134 135 #define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 136 137 CK_PR_STORE_S(64, uint64_t, "std") 138 CK_PR_STORE_S(32, uint32_t, "stw") 139 CK_PR_STORE_S(16, uint16_t, "sth") 140 CK_PR_STORE_S(8, uint8_t, "stb") 141 CK_PR_STORE_S(uint, unsigned int, "stw") 142 CK_PR_STORE_S(int, int, "stw") 143 CK_PR_STORE_S(short, short, "sth") 144 CK_PR_STORE_S(char, char, "stb") 145 #ifndef CK_PR_DISABLE_DOUBLE 146 CK_PR_STORE_S(double, double, "std") 147 #endif 148 149 #undef CK_PR_STORE_S 150 #undef CK_PR_STORE 151 152 CK_CC_INLINE static bool 153 ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) 154 { 155 uint64_t previous; 156 157 __asm__ __volatile__("1:" 158 "ldarx %0, 0, %1;" 159 "cmpd 0, %0, %3;" 160 "bne- 2f;" 161 "stdcx. %2, 0, %1;" 162 "bne- 1b;" 163 "2:" 164 : "=&r" (previous) 165 : "r" (target), 166 "r" (set), 167 "r" (compare) 168 : "memory", "cc"); 169 170 *value = previous; 171 return (previous == compare); 172 } 173 174 CK_CC_INLINE static bool 175 ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) 176 { 177 void *previous; 178 179 __asm__ __volatile__("1:" 180 "ldarx %0, 0, %1;" 181 "cmpd 0, %0, %3;" 182 "bne- 2f;" 183 "stdcx. %2, 0, %1;" 184 "bne- 1b;" 185 "2:" 186 : "=&r" (previous) 187 : "r" (target), 188 "r" (set), 189 "r" (compare) 190 : "memory", "cc"); 191 192 ck_pr_md_store_ptr(value, previous); 193 return (previous == compare); 194 } 195 196 CK_CC_INLINE static bool 197 ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) 198 { 199 uint64_t previous; 200 201 __asm__ __volatile__("1:" 202 "ldarx %0, 0, %1;" 203 "cmpd 0, %0, %3;" 204 "bne- 2f;" 205 "stdcx. %2, 0, %1;" 206 "bne- 1b;" 207 "2:" 208 : "=&r" (previous) 209 : "r" (target), 210 "r" (set), 211 "r" (compare) 212 : "memory", "cc"); 213 214 return (previous == compare); 215 } 216 217 CK_CC_INLINE static bool 218 ck_pr_cas_ptr(void *target, void *compare, void *set) 219 { 220 void *previous; 221 222 __asm__ __volatile__("1:" 223 "ldarx %0, 0, %1;" 224 "cmpd 0, %0, %3;" 225 "bne- 2f;" 226 "stdcx. %2, 0, %1;" 227 "bne- 1b;" 228 "2:" 229 : "=&r" (previous) 230 : "r" (target), 231 "r" (set), 232 "r" (compare) 233 : "memory", "cc"); 234 235 return (previous == compare); 236 } 237 238 #define CK_PR_CAS(N, T) \ 239 CK_CC_INLINE static bool \ 240 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 241 { \ 242 T previous; \ 243 __asm__ __volatile__("1:" \ 244 "lwarx %0, 0, %1;" \ 245 "cmpw 0, %0, %3;" \ 246 "bne- 2f;" \ 247 "stwcx. %2, 0, %1;" \ 248 "bne- 1b;" \ 249 "2:" \ 250 : "=&r" (previous) \ 251 : "r" (target), \ 252 "r" (set), \ 253 "r" (compare) \ 254 : "memory", "cc"); \ 255 *value = previous; \ 256 return (previous == compare); \ 257 } \ 258 CK_CC_INLINE static bool \ 259 ck_pr_cas_##N(T *target, T compare, T set) \ 260 { \ 261 T previous; \ 262 __asm__ __volatile__("1:" \ 263 "lwarx %0, 0, %1;" \ 264 "cmpw 0, %0, %3;" \ 265 "bne- 2f;" \ 266 "stwcx. %2, 0, %1;" \ 267 "bne- 1b;" \ 268 "2:" \ 269 : "=&r" (previous) \ 270 : "r" (target), \ 271 "r" (set), \ 272 "r" (compare) \ 273 : "memory", "cc"); \ 274 return (previous == compare); \ 275 } 276 277 CK_PR_CAS(32, uint32_t) 278 CK_PR_CAS(uint, unsigned int) 279 CK_PR_CAS(int, int) 280 281 #undef CK_PR_CAS 282 283 #define CK_PR_FAS(N, M, T, W) \ 284 CK_CC_INLINE static T \ 285 ck_pr_fas_##N(M *target, T v) \ 286 { \ 287 T previous; \ 288 __asm__ __volatile__("1:" \ 289 "l" W "arx %0, 0, %1;" \ 290 "st" W "cx. %2, 0, %1;" \ 291 "bne- 1b;" \ 292 : "=&r" (previous) \ 293 : "r" (target), \ 294 "r" (v) \ 295 : "memory", "cc"); \ 296 return (previous); \ 297 } 298 299 CK_PR_FAS(64, uint64_t, uint64_t, "d") 300 CK_PR_FAS(32, uint32_t, uint32_t, "w") 301 #ifndef CK_PR_DISABLE_DOUBLE 302 CK_PR_FAS(double, double, double, "d") 303 #endif 304 CK_PR_FAS(ptr, void, void *, "d") 305 CK_PR_FAS(int, int, int, "w") 306 CK_PR_FAS(uint, unsigned int, unsigned int, "w") 307 308 #undef CK_PR_FAS 309 310 #define CK_PR_UNARY(O, N, M, T, I, W) \ 311 CK_CC_INLINE static void \ 312 ck_pr_##O##_##N(M *target) \ 313 { \ 314 T previous; \ 315 __asm__ __volatile__("1:" \ 316 "l" W "arx %0, 0, %1;" \ 317 I ";" \ 318 "st" W "cx. %0, 0, %1;" \ 319 "bne- 1b;" \ 320 : "=&r" (previous) \ 321 : "r" (target) \ 322 : "memory", "cc"); \ 323 return; \ 324 } 325 326 CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") 327 CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") 328 CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") 329 CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") 330 331 #define CK_PR_UNARY_S(S, T, W) \ 332 CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ 333 CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ 334 CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ 335 CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) 336 337 CK_PR_UNARY_S(64, uint64_t, "d") 338 CK_PR_UNARY_S(32, uint32_t, "w") 339 CK_PR_UNARY_S(uint, unsigned int, "w") 340 CK_PR_UNARY_S(int, int, "w") 341 342 #undef CK_PR_UNARY_S 343 #undef CK_PR_UNARY 344 345 #define CK_PR_BINARY(O, N, M, T, I, W) \ 346 CK_CC_INLINE static void \ 347 ck_pr_##O##_##N(M *target, T delta) \ 348 { \ 349 T previous; \ 350 __asm__ __volatile__("1:" \ 351 "l" W "arx %0, 0, %1;" \ 352 I " %0, %2, %0;" \ 353 "st" W "cx. %0, 0, %1;" \ 354 "bne- 1b;" \ 355 : "=&r" (previous) \ 356 : "r" (target), \ 357 "r" (delta) \ 358 : "memory", "cc"); \ 359 return; \ 360 } 361 362 CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") 363 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") 364 CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") 365 CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") 366 CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") 367 368 #define CK_PR_BINARY_S(S, T, W) \ 369 CK_PR_BINARY(and, S, T, T, "and", W) \ 370 CK_PR_BINARY(add, S, T, T, "add", W) \ 371 CK_PR_BINARY(or, S, T, T, "or", W) \ 372 CK_PR_BINARY(sub, S, T, T, "subf", W) \ 373 CK_PR_BINARY(xor, S, T, T, "xor", W) 374 375 CK_PR_BINARY_S(64, uint64_t, "d") 376 CK_PR_BINARY_S(32, uint32_t, "w") 377 CK_PR_BINARY_S(uint, unsigned int, "w") 378 CK_PR_BINARY_S(int, int, "w") 379 380 #undef CK_PR_BINARY_S 381 #undef CK_PR_BINARY 382 383 CK_CC_INLINE static void * 384 ck_pr_faa_ptr(void *target, uintptr_t delta) 385 { 386 uintptr_t previous, r; 387 388 __asm__ __volatile__("1:" 389 "ldarx %0, 0, %2;" 390 "add %1, %3, %0;" 391 "stdcx. %1, 0, %2;" 392 "bne- 1b;" 393 : "=&r" (previous), 394 "=&r" (r) 395 : "r" (target), 396 "r" (delta) 397 : "memory", "cc"); 398 399 return (void *)(previous); 400 } 401 402 #define CK_PR_FAA(S, T, W) \ 403 CK_CC_INLINE static T \ 404 ck_pr_faa_##S(T *target, T delta) \ 405 { \ 406 T previous, r; \ 407 __asm__ __volatile__("1:" \ 408 "l" W "arx %0, 0, %2;" \ 409 "add %1, %3, %0;" \ 410 "st" W "cx. %1, 0, %2;" \ 411 "bne- 1b;" \ 412 : "=&r" (previous), \ 413 "=&r" (r) \ 414 : "r" (target), \ 415 "r" (delta) \ 416 : "memory", "cc"); \ 417 return (previous); \ 418 } 419 420 CK_PR_FAA(64, uint64_t, "d") 421 CK_PR_FAA(32, uint32_t, "w") 422 CK_PR_FAA(uint, unsigned int, "w") 423 CK_PR_FAA(int, int, "w") 424 425 #undef CK_PR_FAA 426 427 #endif /* CK_PR_PPC64_H */ 428