1 /*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #ifndef _MACHINE_ATOMIC_H_ 30 #define _MACHINE_ATOMIC_H_ 31 32 #define isb() __asm __volatile("isb" : : : "memory") 33 34 /* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49 #define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50 #define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52 #define mb() dmb(sy) /* Full system memory barrier all */ 53 #define wmb() dmb(st) /* Full system memory barrier store */ 54 #define rmb() dmb(ld) /* Full system memory barrier load */ 55 56 static __inline void 57 atomic_add_32(volatile uint32_t *p, uint32_t val) 58 { 59 uint32_t tmp; 60 int res; 61 62 __asm __volatile( 63 "1: ldxr %w0, [%2] \n" 64 " add %w0, %w0, %w3 \n" 65 " stxr %w1, %w0, [%2] \n" 66 " cbnz %w1, 1b \n" 67 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 68 ); 69 } 70 71 static __inline void 72 atomic_clear_32(volatile uint32_t *p, uint32_t val) 73 { 74 uint32_t tmp; 75 int res; 76 77 __asm __volatile( 78 "1: ldxr %w0, [%2] \n" 79 " bic %w0, %w0, %w3 \n" 80 " stxr %w1, %w0, [%2] \n" 81 " cbnz %w1, 1b \n" 82 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 83 ); 84 } 85 86 static __inline int 87 atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 88 { 89 uint32_t tmp; 90 int res; 91 92 __asm __volatile( 93 "1: mov %w1, #1 \n" 94 " ldxr %w0, [%2] \n" 95 " cmp %w0, %w3 \n" 96 " b.ne 2f \n" 97 " stxr %w1, %w4, [%2] \n" 98 " cbnz %w1, 1b \n" 99 "2:" 100 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 101 : : "cc" 102 ); 103 104 return (!res); 105 } 106 107 static __inline uint32_t 108 atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 109 { 110 uint32_t tmp, ret; 111 int res; 112 113 __asm __volatile( 114 "1: ldxr %w4, [%2] \n" 115 " add %w0, %w4, %w3 \n" 116 " stxr %w1, %w0, [%2] \n" 117 " cbnz %w1, 1b \n" 118 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" 119 ); 120 121 return (ret); 122 } 123 124 static __inline uint32_t 125 atomic_readandclear_32(volatile uint32_t *p) 126 { 127 uint32_t tmp, ret; 128 int res; 129 130 __asm __volatile( 131 " mov %w0, #0 \n" 132 "1: ldxr %w3, [%2] \n" 133 " stxr %w1, %w0, [%2] \n" 134 " cbnz %w1, 1b \n" 135 : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" 136 ); 137 138 return (ret); 139 } 140 141 static __inline void 142 atomic_set_32(volatile uint32_t *p, uint32_t val) 143 { 144 uint32_t tmp; 145 int res; 146 147 __asm __volatile( 148 "1: ldxr %w0, [%2] \n" 149 " orr %w0, %w0, %w3 \n" 150 " stxr %w1, %w0, [%2] \n" 151 " cbnz %w1, 1b \n" 152 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 153 ); 154 } 155 156 static __inline void 157 atomic_subtract_32(volatile uint32_t *p, uint32_t val) 158 { 159 uint32_t tmp; 160 int res; 161 162 __asm __volatile( 163 "1: ldxr %w0, [%2] \n" 164 " sub %w0, %w0, %w3 \n" 165 " stxr %w1, %w0, [%2] \n" 166 " cbnz %w1, 1b \n" 167 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 168 ); 169 } 170 171 #define atomic_add_int atomic_add_32 172 #define atomic_clear_int atomic_clear_32 173 #define atomic_cmpset_int atomic_cmpset_32 174 #define atomic_fetchadd_int atomic_fetchadd_32 175 #define atomic_readandclear_int atomic_readandclear_32 176 #define atomic_set_int atomic_set_32 177 #define atomic_subtract_int atomic_subtract_32 178 179 static __inline void 180 atomic_add_acq_32(volatile uint32_t *p, uint32_t val) 181 { 182 uint32_t tmp; 183 int res; 184 185 __asm __volatile( 186 "1: ldaxr %w0, [%2] \n" 187 " add %w0, %w0, %w3 \n" 188 " stxr %w1, %w0, [%2] \n" 189 " cbnz %w1, 1b \n" 190 "2:" 191 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 192 ); 193 } 194 195 static __inline void 196 atomic_clear_acq_32(volatile uint32_t *p, uint32_t val) 197 { 198 uint32_t tmp; 199 int res; 200 201 __asm __volatile( 202 "1: ldaxr %w0, [%2] \n" 203 " bic %w0, %w0, %w3 \n" 204 " stxr %w1, %w0, [%2] \n" 205 " cbnz %w1, 1b \n" 206 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 207 ); 208 } 209 210 static __inline int 211 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 212 { 213 uint32_t tmp; 214 int res; 215 216 __asm __volatile( 217 "1: mov %w1, #1 \n" 218 " ldaxr %w0, [%2] \n" 219 " cmp %w0, %w3 \n" 220 " b.ne 2f \n" 221 " stxr %w1, %w4, [%2] \n" 222 " cbnz %w1, 1b \n" 223 "2:" 224 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 225 : : "cc", "memory" 226 ); 227 228 return (!res); 229 } 230 231 static __inline uint32_t 232 atomic_load_acq_32(volatile uint32_t *p) 233 { 234 uint32_t ret; 235 236 __asm __volatile( 237 "ldar %w0, [%1] \n" 238 : "=&r" (ret) : "r" (p) : "memory"); 239 240 return (ret); 241 } 242 243 static __inline void 244 atomic_set_acq_32(volatile uint32_t *p, uint32_t val) 245 { 246 uint32_t tmp; 247 int res; 248 249 __asm __volatile( 250 "1: ldaxr %w0, [%2] \n" 251 " orr %w0, %w0, %w3 \n" 252 " stxr %w1, %w0, [%2] \n" 253 " cbnz %w1, 1b \n" 254 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 255 ); 256 } 257 258 static __inline void 259 atomic_subtract_acq_32(volatile uint32_t *p, uint32_t val) 260 { 261 uint32_t tmp; 262 int res; 263 264 __asm __volatile( 265 "1: ldaxr %w0, [%2] \n" 266 " sub %w0, %w0, %w3 \n" 267 " stxr %w1, %w0, [%2] \n" 268 " cbnz %w1, 1b \n" 269 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 270 ); 271 } 272 273 #define atomic_add_acq_int atomic_add_acq_32 274 #define atomic_clear_acq_int atomic_clear_acq_32 275 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 276 #define atomic_load_acq_int atomic_load_acq_32 277 #define atomic_set_acq_int atomic_set_acq_32 278 #define atomic_subtract_acq_int atomic_subtract_acq_32 279 280 /* The atomic functions currently are both acq and rel, we should fix this. */ 281 282 static __inline void 283 atomic_add_rel_32(volatile uint32_t *p, uint32_t val) 284 { 285 uint32_t tmp; 286 int res; 287 288 __asm __volatile( 289 "1: ldxr %w0, [%2] \n" 290 " add %w0, %w0, %w3 \n" 291 " stlxr %w1, %w0, [%2] \n" 292 " cbnz %w1, 1b \n" 293 "2:" 294 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 295 ); 296 } 297 298 static __inline void 299 atomic_clear_rel_32(volatile uint32_t *p, uint32_t val) 300 { 301 uint32_t tmp; 302 int res; 303 304 __asm __volatile( 305 "1: ldxr %w0, [%2] \n" 306 " bic %w0, %w0, %w3 \n" 307 " stlxr %w1, %w0, [%2] \n" 308 " cbnz %w1, 1b \n" 309 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 310 ); 311 } 312 313 static __inline int 314 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 315 { 316 uint32_t tmp; 317 int res; 318 319 __asm __volatile( 320 "1: mov %w1, #1 \n" 321 " ldxr %w0, [%2] \n" 322 " cmp %w0, %w3 \n" 323 " b.ne 2f \n" 324 " stlxr %w1, %w4, [%2] \n" 325 " cbnz %w1, 1b \n" 326 "2:" 327 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 328 : : "cc", "memory" 329 ); 330 331 return (!res); 332 } 333 334 static __inline void 335 atomic_set_rel_32(volatile uint32_t *p, uint32_t val) 336 { 337 uint32_t tmp; 338 int res; 339 340 __asm __volatile( 341 "1: ldxr %w0, [%2] \n" 342 " orr %w0, %w0, %w3 \n" 343 " stlxr %w1, %w0, [%2] \n" 344 " cbnz %w1, 1b \n" 345 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 346 ); 347 } 348 349 static __inline void 350 atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 351 { 352 353 __asm __volatile( 354 "stlr %w0, [%1] \n" 355 : : "r" (val), "r" (p) : "memory"); 356 } 357 358 static __inline void 359 atomic_subtract_rel_32(volatile uint32_t *p, uint32_t val) 360 { 361 uint32_t tmp; 362 int res; 363 364 __asm __volatile( 365 "1: ldxr %w0, [%2] \n" 366 " sub %w0, %w0, %w3 \n" 367 " stlxr %w1, %w0, [%2] \n" 368 " cbnz %w1, 1b \n" 369 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 370 ); 371 } 372 373 #define atomic_add_rel_int atomic_add_rel_32 374 #define atomic_clear_rel_int atomic_add_rel_32 375 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 376 #define atomic_set_rel_int atomic_set_rel_32 377 #define atomic_subtract_rel_int atomic_subtract_rel_32 378 #define atomic_store_rel_int atomic_store_rel_32 379 380 381 static __inline void 382 atomic_add_64(volatile uint64_t *p, uint64_t val) 383 { 384 uint64_t tmp; 385 int res; 386 387 __asm __volatile( 388 "1: ldxr %0, [%2] \n" 389 " add %0, %0, %3 \n" 390 " stxr %w1, %0, [%2] \n" 391 " cbnz %w1, 1b \n" 392 : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (val) : : "cc" 393 ); 394 } 395 396 static __inline void 397 atomic_clear_64(volatile uint64_t *p, uint64_t val) 398 { 399 uint64_t tmp; 400 int res; 401 402 __asm __volatile( 403 "1: ldxr %0, [%2] \n" 404 " bic %0, %0, %3 \n" 405 " stxr %w1, %0, [%2] \n" 406 " cbnz %w1, 1b \n" 407 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 408 ); 409 } 410 411 static __inline int 412 atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 413 { 414 uint64_t tmp; 415 int res; 416 417 __asm __volatile( 418 "1: mov %w1, #1 \n" 419 " ldxr %0, [%2] \n" 420 " cmp %0, %3 \n" 421 " b.ne 2f \n" 422 " stxr %w1, %4, [%2] \n" 423 " cbnz %w1, 1b \n" 424 "2:" 425 : "=&r" (tmp), "=&r"(res), "+r" (p), "+r" (cmpval), "+r" (newval) 426 : : "cc", "memory" 427 ); 428 429 return (!res); 430 } 431 432 static __inline uint64_t 433 atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 434 { 435 uint64_t tmp, ret; 436 int res; 437 438 __asm __volatile( 439 "1: ldxr %4, [%2] \n" 440 " add %0, %4, %3 \n" 441 " stxr %w1, %0, [%2] \n" 442 " cbnz %w1, 1b \n" 443 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val), "=&r"(ret) : : "cc" 444 ); 445 446 return (ret); 447 } 448 449 static __inline uint64_t 450 atomic_readandclear_64(volatile uint64_t *p) 451 { 452 uint64_t tmp, ret; 453 int res; 454 455 __asm __volatile( 456 " mov %0, #0 \n" 457 "1: ldxr %3, [%2] \n" 458 " stxr %w1, %0, [%2] \n" 459 " cbnz %w1, 1b \n" 460 : "=&r"(tmp), "=&r"(res), "+r" (p), "=&r"(ret) : : "cc" 461 ); 462 463 return (ret); 464 } 465 466 static __inline void 467 atomic_set_64(volatile uint64_t *p, uint64_t val) 468 { 469 uint64_t tmp; 470 int res; 471 472 __asm __volatile( 473 "1: ldxr %0, [%2] \n" 474 " orr %0, %0, %3 \n" 475 " stxr %w1, %0, [%2] \n" 476 " cbnz %w1, 1b \n" 477 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 478 ); 479 } 480 481 static __inline void 482 atomic_subtract_64(volatile uint64_t *p, uint64_t val) 483 { 484 uint64_t tmp; 485 int res; 486 487 __asm __volatile( 488 "1: ldxr %0, [%2] \n" 489 " sub %0, %0, %3 \n" 490 " stxr %w1, %0, [%2] \n" 491 " cbnz %w1, 1b \n" 492 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc" 493 ); 494 } 495 496 static __inline uint64_t 497 atomic_swap_64(volatile uint64_t *p, uint64_t val) 498 { 499 uint64_t old; 500 int res; 501 502 __asm __volatile( 503 "1: ldxr %0, [%2] \n" 504 " stxr %w1, %3, [%2] \n" 505 " cbnz %w1, 1b \n" 506 : "=&r"(old), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 507 ); 508 509 return (old); 510 } 511 512 #define atomic_add_long atomic_add_64 513 #define atomic_clear_long atomic_clear_64 514 #define atomic_cmpset_long atomic_cmpset_64 515 #define atomic_fetchadd_long atomic_fetchadd_64 516 #define atomic_readandclear_long atomic_readandclear_64 517 #define atomic_set_long atomic_set_64 518 #define atomic_subtract_long atomic_subtract_64 519 520 #define atomic_add_ptr atomic_add_64 521 #define atomic_clear_ptr atomic_clear_64 522 #define atomic_cmpset_ptr atomic_cmpset_64 523 #define atomic_fetchadd_ptr atomic_fetchadd_64 524 #define atomic_readandclear_ptr atomic_readandclear_64 525 #define atomic_set_ptr atomic_set_64 526 #define atomic_subtract_ptr atomic_subtract_64 527 528 static __inline void 529 atomic_add_acq_64(volatile uint64_t *p, uint64_t val) 530 { 531 uint64_t tmp; 532 int res; 533 534 __asm __volatile( 535 "1: ldaxr %0, [%2] \n" 536 " add %0, %0, %3 \n" 537 " stxr %w1, %0, [%2] \n" 538 " cbnz %w1, 1b \n" 539 "2:" 540 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 541 ); 542 } 543 544 static __inline void 545 atomic_clear_acq_64(volatile uint64_t *p, uint64_t val) 546 { 547 uint64_t tmp; 548 int res; 549 550 __asm __volatile( 551 "1: ldaxr %0, [%2] \n" 552 " bic %0, %0, %3 \n" 553 " stxr %w1, %0, [%2] \n" 554 " cbnz %w1, 1b \n" 555 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 556 ); 557 } 558 559 static __inline int 560 atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 561 { 562 uint64_t tmp; 563 int res; 564 565 __asm __volatile( 566 "1: mov %w1, #1 \n" 567 " ldaxr %0, [%2] \n" 568 " cmp %0, %3 \n" 569 " b.ne 2f \n" 570 " stxr %w1, %4, [%2] \n" 571 " cbnz %w1, 1b \n" 572 "2:" 573 : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval) 574 : : "cc", "memory" 575 ); 576 577 return (!res); 578 } 579 580 static __inline uint64_t 581 atomic_load_acq_64(volatile uint64_t *p) 582 { 583 uint64_t ret; 584 585 __asm __volatile( 586 "ldar %0, [%1] \n" 587 : "=&r" (ret) : "r" (p) : "memory"); 588 589 return (ret); 590 } 591 592 static __inline void 593 atomic_set_acq_64(volatile uint64_t *p, uint64_t val) 594 { 595 uint64_t tmp; 596 int res; 597 598 __asm __volatile( 599 "1: ldaxr %0, [%2] \n" 600 " orr %0, %0, %3 \n" 601 " stxr %w1, %0, [%2] \n" 602 " cbnz %w1, 1b \n" 603 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 604 ); 605 } 606 607 static __inline void 608 atomic_subtract_acq_64(volatile uint64_t *p, uint64_t val) 609 { 610 uint64_t tmp; 611 int res; 612 613 __asm __volatile( 614 "1: ldaxr %0, [%2] \n" 615 " sub %0, %0, %3 \n" 616 " stxr %w1, %0, [%2] \n" 617 " cbnz %w1, 1b \n" 618 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 619 ); 620 } 621 622 #define atomic_add_acq_long atomic_add_acq_64 623 #define atomic_clear_acq_long atomic_add_acq_64 624 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 625 #define atomic_load_acq_long atomic_load_acq_64 626 #define atomic_set_acq_long atomic_set_acq_64 627 #define atomic_subtract_acq_long atomic_subtract_acq_64 628 629 #define atomic_add_acq_ptr atomic_add_acq_64 630 #define atomic_clear_acq_ptr atomic_add_acq_64 631 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 632 #define atomic_load_acq_ptr atomic_load_acq_64 633 #define atomic_set_acq_ptr atomic_set_acq_64 634 #define atomic_subtract_acq_ptr atomic_subtract_acq_64 635 636 /* 637 * TODO: The atomic functions currently are both acq and rel, we should fix 638 * this. 639 */ 640 static __inline void 641 atomic_add_rel_64(volatile uint64_t *p, uint64_t val) 642 { 643 uint64_t tmp; 644 int res; 645 646 __asm __volatile( 647 "1: ldxr %0, [%2] \n" 648 " add %0, %0, %3 \n" 649 " stlxr %w1, %0, [%2] \n" 650 " cbnz %w1, 1b \n" 651 "2:" 652 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 653 ); 654 } 655 656 static __inline void 657 atomic_clear_rel_64(volatile uint64_t *p, uint64_t val) 658 { 659 uint64_t tmp; 660 int res; 661 662 __asm __volatile( 663 "1: ldxr %0, [%2] \n" 664 " bic %0, %0, %3 \n" 665 " stlxr %w1, %0, [%2] \n" 666 " cbnz %w1, 1b \n" 667 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 668 ); 669 } 670 671 static __inline int 672 atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 673 { 674 uint64_t tmp; 675 int res; 676 677 __asm __volatile( 678 "1: mov %w1, #1 \n" 679 " ldxr %0, [%2] \n" 680 " cmp %0, %3 \n" 681 " b.ne 2f \n" 682 " stlxr %w1, %4, [%2] \n" 683 " cbnz %w1, 1b \n" 684 "2:" 685 : "=&r" (tmp), "=&r" (res), "+r" (p), "+r" (cmpval), "+r" (newval) 686 : : "cc", "memory" 687 ); 688 689 return (!res); 690 } 691 692 static __inline void 693 atomic_set_rel_64(volatile uint64_t *p, uint64_t val) 694 { 695 uint64_t tmp; 696 int res; 697 698 __asm __volatile( 699 "1: ldxr %0, [%2] \n" 700 " orr %0, %0, %3 \n" 701 " stlxr %w1, %0, [%2] \n" 702 " cbnz %w1, 1b \n" 703 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 704 ); 705 } 706 707 static __inline void 708 atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 709 { 710 711 __asm __volatile( 712 "stlr %0, [%1] \n" 713 : : "r" (val), "r" (p) : "memory"); 714 } 715 716 static __inline void 717 atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val) 718 { 719 uint64_t tmp; 720 int res; 721 722 __asm __volatile( 723 "1: ldxr %0, [%2] \n" 724 " sub %0, %0, %3 \n" 725 " stlxr %w1, %0, [%2] \n" 726 " cbnz %w1, 1b \n" 727 : "=&r"(tmp), "=&r"(res), "+r" (p), "+r" (val) : : "cc", "memory" 728 ); 729 } 730 731 #define atomic_add_rel_long atomic_add_rel_64 732 #define atomic_clear_rel_long atomic_clear_rel_64 733 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 734 #define atomic_set_rel_long atomic_set_rel_64 735 #define atomic_subtract_rel_long atomic_subtract_rel_64 736 #define atomic_store_rel_long atomic_store_rel_64 737 738 #define atomic_add_rel_ptr atomic_add_rel_64 739 #define atomic_clear_rel_ptr atomic_clear_rel_64 740 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 741 #define atomic_set_rel_ptr atomic_set_rel_64 742 #define atomic_subtract_rel_ptr atomic_subtract_rel_64 743 #define atomic_store_rel_ptr atomic_store_rel_64 744 745 #endif /* _MACHINE_ATOMIC_H_ */ 746 747