1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 #define dsb() __asm __volatile("dsb sy" : : : "memory") 34 #define dmb() __asm __volatile("dmb sy" : : : "memory") 35 36 #define mb() dmb() 37 #define wmb() dmb() 38 #define rmb() dmb() 39 40 static __inline void 41 atomic_add_32(volatile uint32_t *p, uint32_t val) 42 { 43 uint32_t tmp; 44 int res; 45 46 __asm __volatile( 47 "1: ldxr %w0, [%2] \n" 48 " add %w0, %w0, %w3 \n" 49 " stxr %w1, %w0, [%2] \n" 50 " cbnz %w1, 1b \n" 51 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 52 ); 53 } 54 55 static __inline void 56 atomic_clear_32(volatile uint32_t *p, uint32_t val) 57 { 58 uint32_t tmp; 59 int res; 60 61 __asm __volatile( 62 "1: ldxr %w0, [%2] \n" 63 " bic %w0, %w0, %w3 \n" 64 " stxr %w1, %w0, [%2] \n" 65 " cbnz %w1, 1b \n" 66 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 67 ); 68 } 69 70 static __inline int 71 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 72 { 73 uint32_t tmp; 74 int res; 75 76 __asm __volatile( 77 "1: mov %w1, #1 \n" 78 " ldxr %w0, [%2] \n" 79 " cmp %w0, %w3 \n" 80 " b.ne 2f \n" 81 " stxr %w1, %w4, [%2] \n" 82 " cbnz %w1, 1b \n" 83 "2:" 84 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 85 : : "cc" 86 ); 87 88 return (!res); 89 } 90 91 static __inline uint32_t 92 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 93 { 94 uint32_t tmp, ret; 95 int res; 96 97 __asm __volatile( 98 "1: ldxr %w4, [%2] \n" 99 " add %w0, %w4, %w3 \n" 100 " stxr %w1, %w0, [%2] \n" 101 " cbnz %w1, 1b \n" 102 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" 103 ); 104 105 return (ret); 106 } 107 108 static __inline uint32_t 109 atomic_readandclear_32(volatile uint32_t *p) 110 { 111 uint32_t tmp, ret; 112 int res; 113 114 __asm __volatile( 115 " mov %w0, #0 \n" 116 "1: ldxr %w3, [%2] \n" 117 " stxr %w1, %w0, [%2] \n" 118 " cbnz %w1, 1b \n" 119 : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" 120 ); 121 122 return (ret); 123 } 124 125 static __inline void 126 atomic_set_32(volatile uint32_t *p, uint32_t val) 127 { 128 uint32_t tmp; 129 int res; 130 131 __asm __volatile( 132 "1: ldxr %w0, [%2] \n" 133 " orr %w0, %w0, %w3 \n" 134 " stxr %w1, %w0, [%2] \n" 135 " cbnz %w1, 1b \n" 136 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 137 ); 138 } 139 140 static __inline void 141 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 142 { 143 uint32_t tmp; 144 int res; 145 146 __asm __volatile( 147 "1: ldxr %w0, [%2] \n" 148 " sub %w0, %w0, %w3 \n" 149 " stxr %w1, %w0, [%2] \n" 150 " cbnz %w1, 1b \n" 151 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 152 ); 153 } 154 155 #define atomic_add_int atomic_add_32 156 #define atomic_clear_int atomic_clear_32 157 #define atomic_cmpset_int atomic_cmpset_32 158 #define atomic_fetchadd_int atomic_fetchadd_32 159 #define atomic_readandclear_int atomic_readandclear_32 160 #define atomic_set_int atomic_set_32 161 #define atomic_subtract_int atomic_subtract_32 162 163 164 static __inline void 165 atomic_add_acq_32(volatile uint32_t *p, uint32_t val) 166 { 167 uint32_t tmp; 168 int res; 169 170 __asm __volatile( 171 "1: ldaxr %w0, [%2] \n" 172 " add %w0, %w0, %w3 \n" 173 " stlxr %w1, %w0, [%2] \n" 174 " cbnz %w1, 1b \n" 175 "2:" 176 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 177 ); 178 } 179 180 static __inline void 181 atomic_clear_acq_32(volatile uint32_t *p, uint32_t val) 182 { 183 uint32_t tmp; 184 int res; 185 186 __asm __volatile( 187 "1: ldaxr %w0, [%2] \n" 188 " bic %w0, %w0, %w3 \n" 189 " stlxr %w1, %w0, [%2] \n" 190 " cbnz %w1, 1b \n" 191 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 192 ); 193 } 194 195 static __inline int 196 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 197 { 198 uint32_t tmp; 199 int res; 200 201 __asm __volatile( 202 "1: mov %w1, #1 \n" 203 " ldaxr %w0, [%2] \n" 204 " cmp %w0, %w3 \n" 205 " b.ne 2f \n" 206 " stlxr %w1, %w4, [%2] \n" 207 " cbnz %w1, 1b \n" 208 "2:" 209 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 210 : : "cc", "memory" 211 ); 212 213 return (!res); 214 } 215 216 static __inline uint32_t 217 atomic_load_acq_32(volatile uint32_t *p) 218 { 219 uint32_t ret; 220 221 ret = *p; 222 dmb(); 223 224 return (ret); 225 } 226 227 static __inline void 228 atomic_set_acq_32(volatile uint32_t *p, uint32_t val) 229 { 230 uint32_t tmp; 231 int res; 232 233 __asm __volatile( 234 "1: ldaxr %w0, [%2] \n" 235 " orr %w0, %w0, %w3 \n" 236 " stlxr %w1, %w0, [%2] \n" 237 " cbnz %w1, 1b \n" 238 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 239 ); 240 } 241 242 static __inline void 243 atomic_subtract_acq_32(volatile uint32_t *p, uint32_t val) 244 { 245 uint32_t tmp; 246 int res; 247 248 __asm __volatile( 249 "1: ldaxr %w0, [%2] \n" 250 " sub %w0, %w0, %w3 \n" 251 " stlxr %w1, %w0, [%2] \n" 252 " cbnz %w1, 1b \n" 253 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 254 ); 255 } 256 257 static __inline void 258 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 259 { 260 261 dmb(); 262 *p = val; 263 } 264 265 #define atomic_add_acq_int atomic_add_acq_32 266 #define atomic_clear_acq_int atomic_add_acq_32 267 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 268 #define atomic_load_acq_int atomic_load_acq_32 269 #define atomic_set_acq_int atomic_set_acq_32 270 #define atomic_subtract_acq_int atomic_subtract_acq_32 271 272 /* The atomic functions currently are both acq and rel, we should fix this. */ 273 #define atomic_add_rel_32 atomic_add_acq_32 274 #define atomic_clear_rel_32 atomic_add_acq_32 275 #define atomic_cmpset_rel_32 atomic_cmpset_acq_32 276 #define atomic_set_rel_32 atomic_set_acq_32 277 #define atomic_subtract_rel_32 atomic_subtract_acq_32 278 279 #define atomic_add_rel_int atomic_add_rel_32 280 #define atomic_clear_rel_int atomic_add_rel_32 281 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 282 #define atomic_set_rel_int atomic_set_rel_32 283 #define atomic_subtract_rel_int atomic_subtract_rel_32 284 #define atomic_store_rel_int atomic_store_rel_32 285 286 287 static __inline void 288 atomic_add_64(volatile uint64_t *p, uint64_t val) 289 { 290 uint64_t tmp; 291 int res; 292 293 __asm __volatile( 294 "1: ldxr %0, [%2] \n" 295 " add %0, %0, %3 \n" 296 " stxr %w1, %0, [%2] \n" 297 " cbnz %w1, 1b \n" 298 : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (val) : : "cc" 299 ); 300 } 301 302 static __inline void 303 atomic_clear_64(volatile uint64_t *p, uint64_t val) 304 { 305 uint64_t tmp; 306 int res; 307 308 __asm __volatile( 309 "1: ldxr %0, [%2] \n" 310 " bic %0, %0, %3 \n" 311 " stxr %w1, %0, [%2] \n" 312 " cbnz %w1, 1b \n" 313 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 314 ); 315 } 316 317 static __inline int 318 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 319 { 320 uint64_t tmp; 321 int res; 322 323 __asm __volatile( 324 "1: mov %w1, #1 \n" 325 " ldxr %0, [%2] \n" 326 " cmp %0, %3 \n" 327 " b.ne 2f \n" 328 " stxr %w1, %4, [%2] \n" 329 " cbnz %w1, 1b \n" 330 "2:" 331 : "=&r" (tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 332 : : "cc", "memory" 333 ); 334 335 return (!res); 336 } 337 338 static __inline uint64_t 339 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 340 { 341 uint64_t tmp, ret; 342 int res; 343 344 __asm __volatile( 345 "1: ldxr %4, [%2] \n" 346 " add %0, %4, %3 \n" 347 " stxr %w1, %0, [%2] \n" 348 " cbnz %w1, 1b \n" 349 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" 350 ); 351 352 return (ret); 353 } 354 355 static __inline uint64_t 356 atomic_readandclear_64(volatile uint64_t *p) 357 { 358 uint64_t tmp, ret; 359 int res; 360 361 __asm __volatile( 362 " mov %0, #0 \n" 363 "1: ldxr %3, [%2] \n" 364 " stxr %w1, %0, [%2] \n" 365 " cbnz %w1, 1b \n" 366 : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" 367 ); 368 369 return (ret); 370 } 371 372 static __inline void 373 atomic_set_64(volatile uint64_t *p, uint64_t val) 374 { 375 uint64_t tmp; 376 int res; 377 378 __asm __volatile( 379 "1: ldxr %0, [%2] \n" 380 " orr %0, %0, %3 \n" 381 " stxr %w1, %0, [%2] \n" 382 " cbnz %w1, 1b \n" 383 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 384 ); 385 } 386 387 static __inline void 388 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 389 { 390 uint64_t tmp; 391 int res; 392 393 __asm __volatile( 394 "1: ldxr %0, [%2] \n" 395 " sub %0, %0, %3 \n" 396 " stxr %w1, %0, [%2] \n" 397 " cbnz %w1, 1b \n" 398 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 399 ); 400 } 401 402 static __inline uint64_t 403 atomic_swap_64(volatile uint64_t *p, uint64_t val) 404 { 405 uint64_t old; 406 int res; 407 408 __asm __volatile( 409 "1: ldxr %0, [%2] \n" 410 " stxr %w1, %3, [%2] \n" 411 " cbnz %w1, 1b \n" 412 : "=&r"(old), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 413 ); 414 415 return (old); 416 } 417 418 #define atomic_add_long atomic_add_64 419 #define atomic_clear_long atomic_clear_64 420 #define atomic_cmpset_long atomic_cmpset_64 421 #define atomic_fetchadd_long atomic_fetchadd_64 422 #define atomic_readandclear_long atomic_readandclear_64 423 #define atomic_set_long atomic_set_64 424 #define atomic_subtract_long atomic_subtract_64 425 426 #define atomic_add_ptr atomic_add_64 427 #define atomic_clear_ptr atomic_clear_64 428 #define atomic_cmpset_ptr atomic_cmpset_64 429 #define atomic_fetchadd_ptr atomic_fetchadd_64 430 #define atomic_readandclear_ptr atomic_readandclear_64 431 #define atomic_set_ptr atomic_set_64 432 #define atomic_subtract_ptr atomic_subtract_64 433 434 static __inline void 435 atomic_add_acq_64(volatile uint64_t *p, uint64_t val) 436 { 437 uint64_t tmp; 438 int res; 439 440 __asm __volatile( 441 "1: ldaxr %0, [%2] \n" 442 " add %0, %0, %3 \n" 443 " stlxr %w1, %0, [%2] \n" 444 " cbnz %w1, 1b \n" 445 "2:" 446 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 447 ); 448 } 449 450 static __inline void 451 atomic_clear_acq_64(volatile uint64_t *p, uint64_t val) 452 { 453 uint64_t tmp; 454 int res; 455 456 __asm __volatile( 457 "1: ldaxr %0, [%2] \n" 458 " bic %0, %0, %3 \n" 459 " stlxr %w1, %0, [%2] \n" 460 " cbnz %w1, 1b \n" 461 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 462 ); 463 } 464 465 static __inline int 466 atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 467 { 468 uint64_t tmp; 469 int res; 470 471 __asm __volatile( 472 "1: mov %w1, #1 \n" 473 " ldaxr %0, [%2] \n" 474 " cmp %0, %3 \n" 475 " b.ne 2f \n" 476 " stlxr %w1, %4, [%2] \n" 477 " cbnz %w1, 1b \n" 478 "2:" 479 : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval) 480 : : "cc", "memory" 481 ); 482 483 return (!res); 484 } 485 486 static __inline uint64_t 487 atomic_load_acq_64(volatile uint64_t *p) 488 { 489 uint64_t ret; 490 491 ret = *p; 492 dmb(); 493 494 return (ret); 495 } 496 497 static __inline void 498 atomic_set_acq_64(volatile uint64_t *p, uint64_t val) 499 { 500 uint64_t tmp; 501 int res; 502 503 __asm __volatile( 504 "1: ldaxr %0, [%2] \n" 505 " orr %0, %0, %3 \n" 506 " stlxr %w1, %0, [%2] \n" 507 " cbnz %w1, 1b \n" 508 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 509 ); 510 } 511 512 static __inline void 513 atomic_subtract_acq_64(volatile uint64_t *p, uint64_t val) 514 { 515 uint64_t tmp; 516 int res; 517 518 __asm __volatile( 519 "1: ldaxr %0, [%2] \n" 520 " sub %0, %0, %3 \n" 521 " stlxr %w1, %0, [%2] \n" 522 " cbnz %w1, 1b \n" 523 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 524 ); 525 } 526 527 static __inline void 528 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 529 { 530 531 dmb(); 532 *p = val; 533 } 534 535 #define atomic_add_acq_long atomic_add_acq_64 536 #define atomic_clear_acq_long atomic_add_acq_64 537 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 538 #define atomic_load_acq_long atomic_load_acq_64 539 #define atomic_set_acq_long atomic_set_acq_64 540 #define atomic_subtract_acq_long atomic_subtract_acq_64 541 542 #define atomic_add_acq_ptr atomic_add_acq_64 543 #define atomic_clear_acq_ptr atomic_add_acq_64 544 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 545 #define atomic_load_acq_ptr atomic_load_acq_64 546 #define atomic_set_acq_ptr atomic_set_acq_64 547 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 548 549 /* 550 * TODO: The atomic functions currently are both acq and rel, we should fix 551 * this. 552 */ 553 #define atomic_add_rel_64 atomic_add_acq_64 554 #define atomic_clear_rel_64 atomic_add_acq_64 555 #define atomic_cmpset_rel_64 atomic_cmpset_acq_64 556 #define atomic_set_rel_64 atomic_set_acq_64 557 #define atomic_subtract_rel_64 atomic_subtract_acq_64 558 559 #define atomic_add_rel_long atomic_add_rel_64 560 #define atomic_clear_rel_long atomic_add_rel_64 561 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 562 #define atomic_set_rel_long atomic_set_rel_64 563 #define atomic_subtract_rel_long atomic_subtract_rel_64 564 #define atomic_store_rel_long atomic_store_rel_64 565 566 #define atomic_add_rel_ptr atomic_add_rel_64 567 #define atomic_clear_rel_ptr atomic_add_rel_64 568 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 569 #define atomic_set_rel_ptr atomic_set_rel_64 570 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 571 #define atomic_store_rel_ptr atomic_store_rel_64 572 573 #endif /* _MACHINE_ATOMIC_H_ */ 574 575