1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
46  * Note that not all Book-E processors accept the light-weight sync variant.
48  * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
49  * to use the heavier-weight sync.
84 	"1:	lwarx	%0, 0, %2\n"				\
85 	"	add	%0, %3, %0\n"				\
86 	"	stwcx.	%0, 0, %2\n"				\
87 	"	bne-	1b\n"					\
96 	"1:	ldarx	%0, 0, %2\n"				\
97 	"	add	%0, %3, %0\n"				\
98 	"	stdcx.	%0, 0, %2\n"				\
99 	"	bne-	1b\n"					\
107 	"1:	lwarx	%0, 0, %2\n"				\
108 	"	add	%0, %3, %0\n"				\
109 	"	stwcx.	%0, 0, %2\n"				\
110 	"	bne-	1b\n"					\
170 	"1:	lwarx	%0, 0, %2\n"				\  in _ATOMIC_ADD()
171 	"	andc	%0, %0, %3\n"				\  in _ATOMIC_ADD()
172 	"	stwcx.	%0, 0, %2\n"				\  in _ATOMIC_ADD()
173 	"	bne-	1b\n"					\  in _ATOMIC_ADD()
182 	"1:	ldarx	%0, 0, %2\n"				\  in _ATOMIC_ADD()
183 	"	andc	%0, %0, %3\n"				\  in _ATOMIC_ADD()
184 	"	stdcx.	%0, 0, %2\n"				\  in _ATOMIC_ADD()
185 	"	bne-	1b\n"					\  in _ATOMIC_ADD()
193 	"1:	lwarx	%0, 0, %2\n"				\  in _ATOMIC_ADD()
194 	"	andc	%0, %0, %3\n"				\  in _ATOMIC_ADD()
195 	"	stwcx.	%0, 0, %2\n"				\  in _ATOMIC_ADD()
196 	"	bne-	1b\n"					\  in _ATOMIC_ADD()
250  * atomic_cmpset(p, o, n)
252 /* TODO -- see below */
257 /* TODO -- see below */
262 /* TODO -- see below */
271 	"1:	lwarx	%0, 0, %2\n"				\
272 	"	or	%0, %3, %0\n"				\
273 	"	stwcx.	%0, 0, %2\n"				\
274 	"	bne-	1b\n"					\
283 	"1:	ldarx	%0, 0, %2\n"				\
284 	"	or	%0, %3, %0\n"				\
285 	"	stdcx.	%0, 0, %2\n"				\
286 	"	bne-	1b\n"					\
294 	"1:	lwarx	%0, 0, %2\n"				\
295 	"	or	%0, %3, %0\n"				\
296 	"	stwcx.	%0, 0, %2\n"				\
297 	"	bne-	1b\n"					\
352  * { *p -= v; }
357 	"1:	lwarx	%0, 0, %2\n"				\
358 	"	subf	%0, %3, %0\n"				\
359 	"	stwcx.	%0, 0, %2\n"				\
360 	"	bne-	1b\n"					\
369 	"1:	ldarx	%0, 0, %2\n"				\
370 	"	subf	%0, %3, %0\n"				\
371 	"	stdcx.	%0, 0, %2\n"				\
372 	"	bne-	1b\n"					\
380 	"1:	lwarx	%0, 0, %2\n"				\
381 	"	subf	%0, %3, %0\n"				\
382 	"	stwcx.	%0, 0, %2\n"				\
383 	"	bne-	1b\n"					\
439 /* TODO -- see below */
451 		"\tsync\n"			/* drain writes */
452 		"1:\tlwarx %0, 0, %3\n\t"	/* load old value */
453 		"li %1, 0\n\t"			/* load new value */
454 		"stwcx. %1, 0, %3\n\t"      	/* attempt to store */
455 		"bne- 1b\n\t"			/* spin if failed */
470 		"\tsync\n"			/* drain writes */  in atomic_readandclear_long()
471 		"1:\tldarx %0, 0, %3\n\t"	/* load old value */  in atomic_readandclear_long()
472 		"li %1, 0\n\t"			/* load new value */  in atomic_readandclear_long()
473 		"stdcx. %1, 0, %3\n\t"      	/* attempt to store */  in atomic_readandclear_long()
474 		"bne- 1b\n\t"			/* spin if failed */  in atomic_readandclear_long()
567 		"1:\tlbarx %0, 0, %2\n\t"	/* load old value */
568 		"cmplw %3, %0\n\t"		/* compare */
569 		"bne- 2f\n\t"			/* exit if not equal */
570 		"stbcx. %4, 0, %2\n\t"      	/* attempt to store */
571 		"bne- 1b\n\t"			/* spin if failed */
572 		"li %0, 1\n\t"			/* success - retval = 1 */
573 		"b 3f\n\t"			/* we've succeeded */
574 		"2:\n\t"
575 		"stbcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */
576 		"li %0, 0\n\t"			/* failure - retval = 0 */
577 		"3:\n\t"
591 		"1:\tlharx %0, 0, %2\n\t"	/* load old value */  in atomic_cmpset_short()
592 		"cmplw %3, %0\n\t"		/* compare */  in atomic_cmpset_short()
593 		"bne- 2f\n\t"			/* exit if not equal */  in atomic_cmpset_short()
594 		"sthcx. %4, 0, %2\n\t"      	/* attempt to store */  in atomic_cmpset_short()
595 		"bne- 1b\n\t"			/* spin if failed */  in atomic_cmpset_short()
596 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_cmpset_short()
597 		"b 3f\n\t"			/* we've succeeded */  in atomic_cmpset_short()
598 		"2:\n\t"  in atomic_cmpset_short()
599 		"sthcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */  in atomic_cmpset_short()
600 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_cmpset_short()
601 		"3:\n\t"  in atomic_cmpset_short()
617 		"1:\tlwarx %2, 0, %3\n\t"	/* load old value */
618 		"and %0, %2, %7\n\t"
619 		"cmplw %4, %0\n\t"		/* compare */
620 		"bne- 2f\n\t"			/* exit if not equal */
621 		"andc %2, %2, %7\n\t"
622 		"or %2, %2, %5\n\t"
623 		"stwcx. %2, 0, %3\n\t"      	/* attempt to store */
624 		"bne- 1b\n\t"			/* spin if failed */
625 		"li %0, 1\n\t"			/* success - retval = 1 */
626 		"b 3f\n\t"			/* we've succeeded */
627 		"2:\n\t"
628 		"stwcx. %2, 0, %3\n\t"       	/* clear reservation (74xx) */
629 		"li %0, 0\n\t"			/* failure - retval = 0 */
630 		"3:\n\t"
648 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */  in atomic_cmpset_int()
649 		"cmplw %3, %0\n\t"		/* compare */  in atomic_cmpset_int()
650 		"bne- 2f\n\t"			/* exit if not equal */  in atomic_cmpset_int()
651 		"stwcx. %4, 0, %2\n\t"      	/* attempt to store */  in atomic_cmpset_int()
652 		"bne- 1b\n\t"			/* spin if failed */  in atomic_cmpset_int()
653 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_cmpset_int()
654 		"b 3f\n\t"			/* we've succeeded */  in atomic_cmpset_int()
655 		"2:\n\t"  in atomic_cmpset_int()
656 		"stwcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */  in atomic_cmpset_int()
657 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_cmpset_int()
658 		"3:\n\t"  in atomic_cmpset_int()
672 		"1:\tldarx %0, 0, %2\n\t"	/* load old value */  in atomic_cmpset_long()
673 		"cmpld %3, %0\n\t"		/* compare */  in atomic_cmpset_long()
674 		"bne- 2f\n\t"			/* exit if not equal */  in atomic_cmpset_long()
675 		"stdcx. %4, 0, %2\n\t"		/* attempt to store */  in atomic_cmpset_long()
677 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */  in atomic_cmpset_long()
678 		"cmplw %3, %0\n\t"		/* compare */  in atomic_cmpset_long()
679 		"bne- 2f\n\t"			/* exit if not equal */  in atomic_cmpset_long()
680 		"stwcx. %4, 0, %2\n\t"		/* attempt to store */  in atomic_cmpset_long()
682 		"bne- 1b\n\t"			/* spin if failed */  in atomic_cmpset_long()
683 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_cmpset_long()
684 		"b 3f\n\t"			/* we've succeeded */  in atomic_cmpset_long()
685 		"2:\n\t"  in atomic_cmpset_long()
687 		"stdcx. %0, 0, %2\n\t"		/* clear reservation (74xx) */  in atomic_cmpset_long()
689 		"stwcx. %0, 0, %2\n\t"		/* clear reservation (74xx) */  in atomic_cmpset_long()
691 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_cmpset_long()
692 		"3:\n\t"  in atomic_cmpset_long()
765 		"lbarx %0, 0, %3\n\t"		/* load old value */  in atomic_fcmpset_char()
766 		"cmplw %4, %0\n\t"		/* compare */  in atomic_fcmpset_char()
767 		"bne- 1f\n\t"			/* exit if not equal */  in atomic_fcmpset_char()
768 		"stbcx. %5, 0, %3\n\t"      	/* attempt to store */  in atomic_fcmpset_char()
769 		"bne- 1f\n\t"			/* exit if failed */  in atomic_fcmpset_char()
770 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_fcmpset_char()
771 		"b 2f\n\t"			/* we've succeeded */  in atomic_fcmpset_char()
772 		"1:\n\t"  in atomic_fcmpset_char()
773 		"stbcx. %0, 0, %3\n\t"       	/* clear reservation (74xx) */  in atomic_fcmpset_char()
774 		"stbx %0, 0, %7\n\t"  in atomic_fcmpset_char()
775 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_fcmpset_char()
776 		"2:\n\t"  in atomic_fcmpset_char()
790 		"lharx %0, 0, %3\n\t"		/* load old value */  in atomic_fcmpset_short()
791 		"cmplw %4, %0\n\t"		/* compare */  in atomic_fcmpset_short()
792 		"bne- 1f\n\t"			/* exit if not equal */  in atomic_fcmpset_short()
793 		"sthcx. %5, 0, %3\n\t"      	/* attempt to store */  in atomic_fcmpset_short()
794 		"bne- 1f\n\t"			/* exit if failed */  in atomic_fcmpset_short()
795 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_fcmpset_short()
796 		"b 2f\n\t"			/* we've succeeded */  in atomic_fcmpset_short()
797 		"1:\n\t"  in atomic_fcmpset_short()
798 		"sthcx. %0, 0, %3\n\t"       	/* clear reservation (74xx) */  in atomic_fcmpset_short()
799 		"sthx %0, 0, %7\n\t"  in atomic_fcmpset_short()
800 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_fcmpset_short()
801 		"2:\n\t"  in atomic_fcmpset_short()
816 		"lwarx %0, 0, %3\n\t"		/* load old value */  in atomic_fcmpset_int()
817 		"cmplw %4, %0\n\t"		/* compare */  in atomic_fcmpset_int()
818 		"bne- 1f\n\t"			/* exit if not equal */  in atomic_fcmpset_int()
819 		"stwcx. %5, 0, %3\n\t"      	/* attempt to store */  in atomic_fcmpset_int()
820 		"bne- 1f\n\t"			/* exit if failed */  in atomic_fcmpset_int()
821 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_fcmpset_int()
822 		"b 2f\n\t"			/* we've succeeded */  in atomic_fcmpset_int()
823 		"1:\n\t"  in atomic_fcmpset_int()
824 		"stwcx. %0, 0, %3\n\t"       	/* clear reservation (74xx) */  in atomic_fcmpset_int()
825 		"stwx %0, 0, %7\n\t"  in atomic_fcmpset_int()
826 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_fcmpset_int()
827 		"2:\n\t"  in atomic_fcmpset_int()
841 		"ldarx %0, 0, %3\n\t"		/* load old value */  in atomic_fcmpset_long()
842 		"cmpld %4, %0\n\t"		/* compare */  in atomic_fcmpset_long()
843 		"bne- 1f\n\t"			/* exit if not equal */  in atomic_fcmpset_long()
844 		"stdcx. %5, 0, %3\n\t"		/* attempt to store */  in atomic_fcmpset_long()
846 		"lwarx %0, 0, %3\n\t"		/* load old value */  in atomic_fcmpset_long()
847 		"cmplw %4, %0\n\t"		/* compare */  in atomic_fcmpset_long()
848 		"bne- 1f\n\t"			/* exit if not equal */  in atomic_fcmpset_long()
849 		"stwcx. %5, 0, %3\n\t"		/* attempt to store */  in atomic_fcmpset_long()
851 		"bne- 1f\n\t"			/* exit if failed */  in atomic_fcmpset_long()
852 		"li %0, 1\n\t"			/* success - retval = 1 */  in atomic_fcmpset_long()
853 		"b 2f\n\t"			/* we've succeeded */  in atomic_fcmpset_long()
854 		"1:\n\t"  in atomic_fcmpset_long()
856 		"stdcx. %0, 0, %3\n\t"		/* clear reservation (74xx) */  in atomic_fcmpset_long()
857 		"stdx %0, 0, %7\n\t"  in atomic_fcmpset_long()
859 		"stwcx. %0, 0, %3\n\t"		/* clear reservation (74xx) */  in atomic_fcmpset_long()
860 		"stwx %0, 0, %7\n\t"  in atomic_fcmpset_long()
862 		"li %0, 0\n\t"			/* failure - retval = 0 */  in atomic_fcmpset_long()
863 		"2:\n\t"  in atomic_fcmpset_long()
951 	"1:	lwarx	%0,0,%2\n"  in atomic_swap_32()
952 	"	stwcx.	%3,0,%2\n"  in atomic_swap_32()
953 	"	bne-	1b\n"  in atomic_swap_32()
968 	"1:	ldarx	%0,0,%2\n"  in atomic_swap_64()
969 	"	stdcx.	%3,0,%2\n"  in atomic_swap_64()
970 	"	bne-	1b\n"  in atomic_swap_64()
999 	"1:	lwarx	%0,0,%3\n"  in atomic_testandset_int()
1000 	"	and	%1,%0,%4\n"  in atomic_testandset_int()
1001 	"	or	%0,%0,%4\n"  in atomic_testandset_int()
1002 	"	stwcx.	%0,0,%3\n"  in atomic_testandset_int()
1003 	"	bne-	1b\n"  in atomic_testandset_int()
1019 	"1:	lwarx	%0,0,%3\n"  in atomic_testandclear_int()
1020 	"	and	%1,%0,%4\n"  in atomic_testandclear_int()
1021 	"	andc	%0,%0,%4\n"  in atomic_testandclear_int()
1022 	"	stwcx.	%0,0,%3\n"  in atomic_testandclear_int()
1023 	"	bne-	1b\n"  in atomic_testandclear_int()
1040 	"1:	ldarx	%0,0,%3\n"  in atomic_testandset_long()
1041 	"	and	%1,%0,%4\n"  in atomic_testandset_long()
1042 	"	or	%0,%0,%4\n"  in atomic_testandset_long()
1043 	"	stdcx.	%0,0,%3\n"  in atomic_testandset_long()
1044 	"	bne-	1b\n"  in atomic_testandset_long()
1060 	"1:	ldarx	%0,0,%3\n"  in atomic_testandclear_long()
1061 	"	and	%1,%0,%4\n"  in atomic_testandclear_long()
1062 	"	andc	%0,%0,%4\n"  in atomic_testandclear_long()
1063 	"	stdcx.	%0,0,%3\n"  in atomic_testandclear_long()
1064 	"	bne-	1b\n"  in atomic_testandclear_long()
1173 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */