xref: /titanic_52/usr/src/common/atomic/sparc/atomic.s (revision 55f5292c612446ce6f93ddd248c0019b5974618b) 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27	.file	"atomic.s"
28
29#include <sys/asm_linkage.h>
30
31#if defined(_KERNEL)
32	/*
33	 * Legacy kernel interfaces; they will go away (eventually).
34	 */
35	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
36	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
37	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
38	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
39	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
40	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
41	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
42	ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
43#endif
44
45	/*
46	 * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
47	 * separated, you need to also edit the libc sparc platform
48	 * specific mapfile and remove the NODYNSORT attribute
49	 * from atomic_inc_8_nv.
50	 */
51	ENTRY(atomic_inc_8)
52	ALTENTRY(atomic_inc_8_nv)
53	ALTENTRY(atomic_inc_uchar)
54	ALTENTRY(atomic_inc_uchar_nv)
55	ba	add_8
56	  add	%g0, 1, %o1
57	SET_SIZE(atomic_inc_uchar_nv)
58	SET_SIZE(atomic_inc_uchar)
59	SET_SIZE(atomic_inc_8_nv)
60	SET_SIZE(atomic_inc_8)
61
62	/*
63	 * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
64	 * separated, you need to also edit the libc sparc platform
65	 * specific mapfile and remove the NODYNSORT attribute
66	 * from atomic_dec_8_nv.
67	 */
68	ENTRY(atomic_dec_8)
69	ALTENTRY(atomic_dec_8_nv)
70	ALTENTRY(atomic_dec_uchar)
71	ALTENTRY(atomic_dec_uchar_nv)
72	ba	add_8
73	  sub	%g0, 1, %o1
74	SET_SIZE(atomic_dec_uchar_nv)
75	SET_SIZE(atomic_dec_uchar)
76	SET_SIZE(atomic_dec_8_nv)
77	SET_SIZE(atomic_dec_8)
78
79	/*
80	 * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
81	 * separated, you need to also edit the libc sparc platform
82	 * specific mapfile and remove the NODYNSORT attribute
83	 * from atomic_add_8_nv.
84	 */
85	ENTRY(atomic_add_8)
86	ALTENTRY(atomic_add_8_nv)
87	ALTENTRY(atomic_add_char)
88	ALTENTRY(atomic_add_char_nv)
89add_8:
90	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
91	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
92	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
93	set	0xff, %o3		! %o3 = mask
94	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
95	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
96	and	%o1, %o3, %o1		! %o1 = single byte value
97	andn	%o0, 0x3, %o0		! %o0 = word address
98	ld	[%o0], %o2		! read old value
991:
100	add	%o2, %o1, %o5		! add value to the old value
101	and	%o5, %o3, %o5		! clear other bits
102	andn	%o2, %o3, %o4		! clear target bits
103	or	%o4, %o5, %o5		! insert the new value
104	cas	[%o0], %o2, %o5
105	cmp	%o2, %o5
106	bne,a,pn %icc, 1b
107	  mov	%o5, %o2		! %o2 = old value
108	add	%o2, %o1, %o5
109	and	%o5, %o3, %o5
110	retl
111	srl	%o5, %g1, %o0		! %o0 = new value
112	SET_SIZE(atomic_add_char_nv)
113	SET_SIZE(atomic_add_char)
114	SET_SIZE(atomic_add_8_nv)
115	SET_SIZE(atomic_add_8)
116
117	/*
118	 * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
119	 * separated, you need to also edit the libc sparc platform
120	 * specific mapfile and remove the NODYNSORT attribute
121	 * from atomic_inc_16_nv.
122	 */
123	ENTRY(atomic_inc_16)
124	ALTENTRY(atomic_inc_16_nv)
125	ALTENTRY(atomic_inc_ushort)
126	ALTENTRY(atomic_inc_ushort_nv)
127	ba	add_16
128	  add	%g0, 1, %o1
129	SET_SIZE(atomic_inc_ushort_nv)
130	SET_SIZE(atomic_inc_ushort)
131	SET_SIZE(atomic_inc_16_nv)
132	SET_SIZE(atomic_inc_16)
133
134	/*
135	 * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
136	 * separated, you need to also edit the libc sparc platform
137	 * specific mapfile and remove the NODYNSORT attribute
138	 * from atomic_dec_16_nv.
139	 */
140	ENTRY(atomic_dec_16)
141	ALTENTRY(atomic_dec_16_nv)
142	ALTENTRY(atomic_dec_ushort)
143	ALTENTRY(atomic_dec_ushort_nv)
144	ba	add_16
145	  sub	%g0, 1, %o1
146	SET_SIZE(atomic_dec_ushort_nv)
147	SET_SIZE(atomic_dec_ushort)
148	SET_SIZE(atomic_dec_16_nv)
149	SET_SIZE(atomic_dec_16)
150
151	/*
152	 * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
153	 * separated, you need to also edit the libc sparc platform
154	 * specific mapfile and remove the NODYNSORT attribute
155	 * from atomic_add_16_nv.
156	 */
157	ENTRY(atomic_add_16)
158	ALTENTRY(atomic_add_16_nv)
159	ALTENTRY(atomic_add_short)
160	ALTENTRY(atomic_add_short_nv)
161add_16:
162	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
163	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
164	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
165	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
166	sethi	%hi(0xffff0000), %o3	! %o3 = mask
167	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
168	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
169	and	%o1, %o3, %o1		! %o1 = single short value
170	andn	%o0, 0x2, %o0		! %o0 = word address
171	! if low-order bit is 1, we will properly get an alignment fault here
172	ld	[%o0], %o2		! read old value
1731:
174	add	%o1, %o2, %o5		! add value to the old value
175	and	%o5, %o3, %o5		! clear other bits
176	andn	%o2, %o3, %o4		! clear target bits
177	or	%o4, %o5, %o5		! insert the new value
178	cas	[%o0], %o2, %o5
179	cmp	%o2, %o5
180	bne,a,pn %icc, 1b
181	  mov	%o5, %o2		! %o2 = old value
182	add	%o1, %o2, %o5
183	and	%o5, %o3, %o5
184	retl
185	srl	%o5, %g1, %o0		! %o0 = new value
186	SET_SIZE(atomic_add_short_nv)
187	SET_SIZE(atomic_add_short)
188	SET_SIZE(atomic_add_16_nv)
189	SET_SIZE(atomic_add_16)
190
191	/*
192	 * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
193	 * separated, you need to also edit the libc sparc platform
194	 * specific mapfile and remove the NODYNSORT attribute
195	 * from atomic_inc_32_nv.
196	 */
197	ENTRY(atomic_inc_32)
198	ALTENTRY(atomic_inc_32_nv)
199	ALTENTRY(atomic_inc_uint)
200	ALTENTRY(atomic_inc_uint_nv)
201	ALTENTRY(atomic_inc_ulong)
202	ALTENTRY(atomic_inc_ulong_nv)
203	ba	add_32
204	  add	%g0, 1, %o1
205	SET_SIZE(atomic_inc_ulong_nv)
206	SET_SIZE(atomic_inc_ulong)
207	SET_SIZE(atomic_inc_uint_nv)
208	SET_SIZE(atomic_inc_uint)
209	SET_SIZE(atomic_inc_32_nv)
210	SET_SIZE(atomic_inc_32)
211
212	/*
213	 * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
214	 * separated, you need to also edit the libc sparc platform
215	 * specific mapfile and remove the NODYNSORT attribute
216	 * from atomic_dec_32_nv.
217	 */
218	ENTRY(atomic_dec_32)
219	ALTENTRY(atomic_dec_32_nv)
220	ALTENTRY(atomic_dec_uint)
221	ALTENTRY(atomic_dec_uint_nv)
222	ALTENTRY(atomic_dec_ulong)
223	ALTENTRY(atomic_dec_ulong_nv)
224	ba	add_32
225	  sub	%g0, 1, %o1
226	SET_SIZE(atomic_dec_ulong_nv)
227	SET_SIZE(atomic_dec_ulong)
228	SET_SIZE(atomic_dec_uint_nv)
229	SET_SIZE(atomic_dec_uint)
230	SET_SIZE(atomic_dec_32_nv)
231	SET_SIZE(atomic_dec_32)
232
233	/*
234	 * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
235	 * separated, you need to also edit the libc sparc platform
236	 * specific mapfile and remove the NODYNSORT attribute
237	 * from atomic_add_32_nv.
238	 */
239	ENTRY(atomic_add_32)
240	ALTENTRY(atomic_add_32_nv)
241	ALTENTRY(atomic_add_int)
242	ALTENTRY(atomic_add_int_nv)
243	ALTENTRY(atomic_add_ptr)
244	ALTENTRY(atomic_add_ptr_nv)
245	ALTENTRY(atomic_add_long)
246	ALTENTRY(atomic_add_long_nv)
247add_32:
248	ld	[%o0], %o2
2491:
250	add	%o2, %o1, %o3
251	cas	[%o0], %o2, %o3
252	cmp	%o2, %o3
253	bne,a,pn %icc, 1b
254	  mov	%o3, %o2
255	retl
256	add	%o2, %o1, %o0		! return new value
257	SET_SIZE(atomic_add_long_nv)
258	SET_SIZE(atomic_add_long)
259	SET_SIZE(atomic_add_ptr_nv)
260	SET_SIZE(atomic_add_ptr)
261	SET_SIZE(atomic_add_int_nv)
262	SET_SIZE(atomic_add_int)
263	SET_SIZE(atomic_add_32_nv)
264	SET_SIZE(atomic_add_32)
265
266	/*
267	 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
268	 * separated, you need to also edit the libc sparc platform
269	 * specific mapfile and remove the NODYNSORT attribute
270	 * from atomic_inc_64_nv.
271	 */
272	ENTRY(atomic_inc_64)
273	ALTENTRY(atomic_inc_64_nv)
274	ba	add_64
275	  add	%g0, 1, %o1
276	SET_SIZE(atomic_inc_64_nv)
277	SET_SIZE(atomic_inc_64)
278
279	/*
280	 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
281	 * separated, you need to also edit the libc sparc platform
282	 * specific mapfile and remove the NODYNSORT attribute
283	 * from atomic_dec_64_nv.
284	 */
285	ENTRY(atomic_dec_64)
286	ALTENTRY(atomic_dec_64_nv)
287	ba	add_64
288	  sub	%g0, 1, %o1
289	SET_SIZE(atomic_dec_64_nv)
290	SET_SIZE(atomic_dec_64)
291
292	/*
293	 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
294	 * separated, you need to also edit the libc sparc platform
295	 * specific mapfile and remove the NODYNSORT attribute
296	 * from atomic_add_64_nv.
297	 */
298	ENTRY(atomic_add_64)
299	ALTENTRY(atomic_add_64_nv)
300	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
301	srl	%o2, 0, %o2
302	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
303add_64:
304	ldx	[%o0], %o2
3051:
306	add	%o2, %o1, %o3
307	casx	[%o0], %o2, %o3
308	cmp	%o2, %o3
309	bne,a,pn %xcc, 1b
310	  mov	%o3, %o2
311	add	%o2, %o1, %o1		! return lower 32-bits in %o1
312	retl
313	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
314	SET_SIZE(atomic_add_64_nv)
315	SET_SIZE(atomic_add_64)
316
317	/*
318	 * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
319	 * separated, you need to also edit the libc sparc platform
320	 * specific mapfile and remove the NODYNSORT attribute
321	 * from atomic_or_8_nv.
322	 */
323	ENTRY(atomic_or_8)
324	ALTENTRY(atomic_or_8_nv)
325	ALTENTRY(atomic_or_uchar)
326	ALTENTRY(atomic_or_uchar_nv)
327	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
328	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
329	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
330	set	0xff, %o3		! %o3 = mask
331	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
332	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
333	and	%o1, %o3, %o1		! %o1 = single byte value
334	andn	%o0, 0x3, %o0		! %o0 = word address
335	ld	[%o0], %o2		! read old value
3361:
337	or	%o2, %o1, %o5		! or in the new value
338	cas	[%o0], %o2, %o5
339	cmp	%o2, %o5
340	bne,a,pn %icc, 1b
341	  mov	%o5, %o2		! %o2 = old value
342	or	%o2, %o1, %o5
343	and	%o5, %o3, %o5
344	retl
345	srl	%o5, %g1, %o0		! %o0 = new value
346	SET_SIZE(atomic_or_uchar_nv)
347	SET_SIZE(atomic_or_uchar)
348	SET_SIZE(atomic_or_8_nv)
349	SET_SIZE(atomic_or_8)
350
351	/*
352	 * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
353	 * separated, you need to also edit the libc sparc platform
354	 * specific mapfile and remove the NODYNSORT attribute
355	 * from atomic_or_16_nv.
356	 */
357	ENTRY(atomic_or_16)
358	ALTENTRY(atomic_or_16_nv)
359	ALTENTRY(atomic_or_ushort)
360	ALTENTRY(atomic_or_ushort_nv)
361	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
362	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
363	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
364	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
365	sethi	%hi(0xffff0000), %o3	! %o3 = mask
366	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
367	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
368	and	%o1, %o3, %o1		! %o1 = single short value
369	andn	%o0, 0x2, %o0		! %o0 = word address
370	! if low-order bit is 1, we will properly get an alignment fault here
371	ld	[%o0], %o2		! read old value
3721:
373	or	%o2, %o1, %o5		! or in the new value
374	cas	[%o0], %o2, %o5
375	cmp	%o2, %o5
376	bne,a,pn %icc, 1b
377	  mov	%o5, %o2		! %o2 = old value
378	or	%o2, %o1, %o5		! or in the new value
379	and	%o5, %o3, %o5
380	retl
381	srl	%o5, %g1, %o0		! %o0 = new value
382	SET_SIZE(atomic_or_ushort_nv)
383	SET_SIZE(atomic_or_ushort)
384	SET_SIZE(atomic_or_16_nv)
385	SET_SIZE(atomic_or_16)
386
387	/*
388	 * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
389	 * separated, you need to also edit the libc sparc platform
390	 * specific mapfile and remove the NODYNSORT attribute
391	 * from atomic_or_32_nv.
392	 */
393	ENTRY(atomic_or_32)
394	ALTENTRY(atomic_or_32_nv)
395	ALTENTRY(atomic_or_uint)
396	ALTENTRY(atomic_or_uint_nv)
397	ALTENTRY(atomic_or_ulong)
398	ALTENTRY(atomic_or_ulong_nv)
399	ld	[%o0], %o2
4001:
401	or	%o2, %o1, %o3
402	cas	[%o0], %o2, %o3
403	cmp	%o2, %o3
404	bne,a,pn %icc, 1b
405	  mov	%o3, %o2
406	retl
407	or	%o2, %o1, %o0		! return new value
408	SET_SIZE(atomic_or_ulong_nv)
409	SET_SIZE(atomic_or_ulong)
410	SET_SIZE(atomic_or_uint_nv)
411	SET_SIZE(atomic_or_uint)
412	SET_SIZE(atomic_or_32_nv)
413	SET_SIZE(atomic_or_32)
414
415	/*
416	 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
417	 * separated, you need to also edit the libc sparc platform
418	 * specific mapfile and remove the NODYNSORT attribute
419	 * from atomic_or_64_nv.
420	 */
421	ENTRY(atomic_or_64)
422	ALTENTRY(atomic_or_64_nv)
423	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
424	srl	%o2, 0, %o2
425	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
426	ldx	[%o0], %o2
4271:
428	or	%o2, %o1, %o3
429	casx	[%o0], %o2, %o3
430	cmp	%o2, %o3
431	bne,a,pn %xcc, 1b
432	  mov	%o3, %o2
433	or	%o2, %o1, %o1		! return lower 32-bits in %o1
434	retl
435	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
436	SET_SIZE(atomic_or_64_nv)
437	SET_SIZE(atomic_or_64)
438
439	/*
440	 * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
441	 * separated, you need to also edit the libc sparc platform
442	 * specific mapfile and remove the NODYNSORT attribute
443	 * from atomic_and_8_nv.
444	 */
445	ENTRY(atomic_and_8)
446	ALTENTRY(atomic_and_8_nv)
447	ALTENTRY(atomic_and_uchar)
448	ALTENTRY(atomic_and_uchar_nv)
449	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
450	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
451	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
452	set	0xff, %o3		! %o3 = mask
453	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
454	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
455	orn	%o1, %o3, %o1		! all ones in other bytes
456	andn	%o0, 0x3, %o0		! %o0 = word address
457	ld	[%o0], %o2		! read old value
4581:
459	and	%o2, %o1, %o5		! and in the new value
460	cas	[%o0], %o2, %o5
461	cmp	%o2, %o5
462	bne,a,pn %icc, 1b
463	  mov	%o5, %o2		! %o2 = old value
464	and	%o2, %o1, %o5
465	and	%o5, %o3, %o5
466	retl
467	srl	%o5, %g1, %o0		! %o0 = new value
468	SET_SIZE(atomic_and_uchar_nv)
469	SET_SIZE(atomic_and_uchar)
470	SET_SIZE(atomic_and_8_nv)
471	SET_SIZE(atomic_and_8)
472
473	/*
474	 * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
475	 * separated, you need to also edit the libc sparc platform
476	 * specific mapfile and remove the NODYNSORT attribute
477	 * from atomic_and_16_nv.
478	 */
479	ENTRY(atomic_and_16)
480	ALTENTRY(atomic_and_16_nv)
481	ALTENTRY(atomic_and_ushort)
482	ALTENTRY(atomic_and_ushort_nv)
483	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
484	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
485	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
486	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
487	sethi	%hi(0xffff0000), %o3	! %o3 = mask
488	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
489	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
490	orn	%o1, %o3, %o1		! all ones in the other half
491	andn	%o0, 0x2, %o0		! %o0 = word address
492	! if low-order bit is 1, we will properly get an alignment fault here
493	ld	[%o0], %o2		! read old value
4941:
495	and	%o2, %o1, %o5		! and in the new value
496	cas	[%o0], %o2, %o5
497	cmp	%o2, %o5
498	bne,a,pn %icc, 1b
499	  mov	%o5, %o2		! %o2 = old value
500	and	%o2, %o1, %o5
501	and	%o5, %o3, %o5
502	retl
503	srl	%o5, %g1, %o0		! %o0 = new value
504	SET_SIZE(atomic_and_ushort_nv)
505	SET_SIZE(atomic_and_ushort)
506	SET_SIZE(atomic_and_16_nv)
507	SET_SIZE(atomic_and_16)
508
509	/*
510	 * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
511	 * separated, you need to also edit the libc sparc platform
512	 * specific mapfile and remove the NODYNSORT attribute
513	 * from atomic_and_32_nv.
514	 */
515	ENTRY(atomic_and_32)
516	ALTENTRY(atomic_and_32_nv)
517	ALTENTRY(atomic_and_uint)
518	ALTENTRY(atomic_and_uint_nv)
519	ALTENTRY(atomic_and_ulong)
520	ALTENTRY(atomic_and_ulong_nv)
521	ld	[%o0], %o2
5221:
523	and	%o2, %o1, %o3
524	cas	[%o0], %o2, %o3
525	cmp	%o2, %o3
526	bne,a,pn %icc, 1b
527	  mov	%o3, %o2
528	retl
529	and	%o2, %o1, %o0		! return new value
530	SET_SIZE(atomic_and_ulong_nv)
531	SET_SIZE(atomic_and_ulong)
532	SET_SIZE(atomic_and_uint_nv)
533	SET_SIZE(atomic_and_uint)
534	SET_SIZE(atomic_and_32_nv)
535	SET_SIZE(atomic_and_32)
536
537	/*
538	 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
539	 * separated, you need to also edit the libc sparc platform
540	 * specific mapfile and remove the NODYNSORT attribute
541	 * from atomic_and_64_nv.
542	 */
543	ENTRY(atomic_and_64)
544	ALTENTRY(atomic_and_64_nv)
545	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
546	srl	%o2, 0, %o2
547	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
548	ldx	[%o0], %o2
5491:
550	and	%o2, %o1, %o3
551	casx	[%o0], %o2, %o3
552	cmp	%o2, %o3
553	bne,a,pn %xcc, 1b
554	  mov	%o3, %o2
555	and	%o2, %o1, %o1		! return lower 32-bits in %o1
556	retl
557	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
558	SET_SIZE(atomic_and_64_nv)
559	SET_SIZE(atomic_and_64)
560
561	ENTRY(atomic_cas_8)
562	ALTENTRY(atomic_cas_uchar)
563	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
564	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
565	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
566	set	0xff, %o3		! %o3 = mask
567	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
568	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
569	and	%o1, %o3, %o1		! %o1 = single byte value
570	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
571	and	%o2, %o3, %o2		! %o2 = single byte value
572	andn	%o0, 0x3, %o0		! %o0 = word address
573	ld	[%o0], %o4		! read old value
5741:
575	andn	%o4, %o3, %o4		! clear target bits
576	or	%o4, %o2, %o5		! insert the new value
577	or	%o4, %o1, %o4		! insert the comparison value
578	cas	[%o0], %o4, %o5
579	cmp	%o4, %o5		! did we succeed?
580	be,pt	%icc, 2f
581	  and	%o5, %o3, %o4		! isolate the old value
582	cmp	%o1, %o4		! should we have succeeded?
583	be,a,pt	%icc, 1b		! yes, try again
584	  mov	%o5, %o4		! %o4 = old value
5852:
586	retl
587	srl	%o4, %g1, %o0		! %o0 = old value
588	SET_SIZE(atomic_cas_uchar)
589	SET_SIZE(atomic_cas_8)
590
591	ENTRY(atomic_cas_16)
592	ALTENTRY(atomic_cas_ushort)
593	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
594	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
595	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
596	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
597	sethi	%hi(0xffff0000), %o3	! %o3 = mask
598	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
599	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
600	and	%o1, %o3, %o1		! %o1 = single short value
601	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
602	and	%o2, %o3, %o2		! %o2 = single short value
603	andn	%o0, 0x2, %o0		! %o0 = word address
604	! if low-order bit is 1, we will properly get an alignment fault here
605	ld	[%o0], %o4		! read old value
6061:
607	andn	%o4, %o3, %o4		! clear target bits
608	or	%o4, %o2, %o5		! insert the new value
609	or	%o4, %o1, %o4		! insert the comparison value
610	cas	[%o0], %o4, %o5
611	cmp	%o4, %o5		! did we succeed?
612	be,pt	%icc, 2f
613	  and	%o5, %o3, %o4		! isolate the old value
614	cmp	%o1, %o4		! should we have succeeded?
615	be,a,pt	%icc, 1b		! yes, try again
616	  mov	%o5, %o4		! %o4 = old value
6172:
618	retl
619	srl	%o4, %g1, %o0		! %o0 = old value
620	SET_SIZE(atomic_cas_ushort)
621	SET_SIZE(atomic_cas_16)
622
623	ENTRY(atomic_cas_32)
624	ALTENTRY(atomic_cas_uint)
625	ALTENTRY(atomic_cas_ptr)
626	ALTENTRY(atomic_cas_ulong)
627	cas	[%o0], %o1, %o2
628	retl
629	mov	%o2, %o0
630	SET_SIZE(atomic_cas_ulong)
631	SET_SIZE(atomic_cas_ptr)
632	SET_SIZE(atomic_cas_uint)
633	SET_SIZE(atomic_cas_32)
634
635	ENTRY(atomic_cas_64)
636	sllx	%o1, 32, %o1		! cmp's upper 32 in %o1, lower in %o2
637	srl	%o2, 0, %o2		! convert 2 32-bit args into 1 64-bit
638	add	%o1, %o2, %o1
639	sllx	%o3, 32, %o2		! newval upper 32 in %o3, lower in %o4
640	srl	%o4, 0, %o4		! setup %o2 to have newval
641	add	%o2, %o4, %o2
642	casx	[%o0], %o1, %o2
643	srl	%o2, 0, %o1		! return lower 32-bits in %o1
644	retl
645	srlx	%o2, 32, %o0		! return upper 32-bits in %o0
646	SET_SIZE(atomic_cas_64)
647
648	ENTRY(atomic_swap_8)
649	ALTENTRY(atomic_swap_uchar)
650	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
651	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
652	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
653	set	0xff, %o3		! %o3 = mask
654	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
655	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
656	and	%o1, %o3, %o1		! %o1 = single byte value
657	andn	%o0, 0x3, %o0		! %o0 = word address
658	ld	[%o0], %o2		! read old value
6591:
660	andn	%o2, %o3, %o5		! clear target bits
661	or	%o5, %o1, %o5		! insert the new value
662	cas	[%o0], %o2, %o5
663	cmp	%o2, %o5
664	bne,a,pn %icc, 1b
665	  mov	%o5, %o2		! %o2 = old value
666	and	%o5, %o3, %o5
667	retl
668	srl	%o5, %g1, %o0		! %o0 = old value
669	SET_SIZE(atomic_swap_uchar)
670	SET_SIZE(atomic_swap_8)
671
672	ENTRY(atomic_swap_16)
673	ALTENTRY(atomic_swap_ushort)
674	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
675	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
676	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
677	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
678	sethi	%hi(0xffff0000), %o3	! %o3 = mask
679	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
680	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
681	and	%o1, %o3, %o1		! %o1 = single short value
682	andn	%o0, 0x2, %o0		! %o0 = word address
683	! if low-order bit is 1, we will properly get an alignment fault here
684	ld	[%o0], %o2		! read old value
6851:
686	andn	%o2, %o3, %o5		! clear target bits
687	or	%o5, %o1, %o5		! insert the new value
688	cas	[%o0], %o2, %o5
689	cmp	%o2, %o5
690	bne,a,pn %icc, 1b
691	  mov	%o5, %o2		! %o2 = old value
692	and	%o5, %o3, %o5
693	retl
694	srl	%o5, %g1, %o0		! %o0 = old value
695	SET_SIZE(atomic_swap_ushort)
696	SET_SIZE(atomic_swap_16)
697
698	ENTRY(atomic_swap_32)
699	ALTENTRY(atomic_swap_uint)
700	ALTENTRY(atomic_swap_ptr)
701	ALTENTRY(atomic_swap_ulong)
702	ld	[%o0], %o2
7031:
704	mov	%o1, %o3
705	cas	[%o0], %o2, %o3
706	cmp	%o2, %o3
707	bne,a,pn %icc, 1b
708	  mov	%o3, %o2
709	retl
710	mov	%o3, %o0
711	SET_SIZE(atomic_swap_ulong)
712	SET_SIZE(atomic_swap_ptr)
713	SET_SIZE(atomic_swap_uint)
714	SET_SIZE(atomic_swap_32)
715
716	ENTRY(atomic_swap_64)
717	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
718	srl	%o2, 0, %o2
719	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
720	ldx	[%o0], %o2
7211:
722	mov	%o1, %o3
723	casx	[%o0], %o2, %o3
724	cmp	%o2, %o3
725	bne,a,pn %xcc, 1b
726	  mov	%o3, %o2
727	srl	%o3, 0, %o1		! return lower 32-bits in %o1
728	retl
729	srlx	%o3, 32, %o0		! return upper 32-bits in %o0
730	SET_SIZE(atomic_swap_64)
731
732	ENTRY(atomic_set_long_excl)
733	mov	1, %o3
734	slln	%o3, %o1, %o3
735	ldn	[%o0], %o2
7361:
737	andcc	%o2, %o3, %g0		! test if the bit is set
738	bnz,a,pn %ncc, 2f		! if so, then fail out
739	  mov	-1, %o0
740	or	%o2, %o3, %o4		! set the bit, and try to commit it
741	casn	[%o0], %o2, %o4
742	cmp	%o2, %o4
743	bne,a,pn %ncc, 1b		! failed to commit, try again
744	  mov	%o4, %o2
745	mov	%g0, %o0
7462:
747	retl
748	nop
749	SET_SIZE(atomic_set_long_excl)
750
751	ENTRY(atomic_clear_long_excl)
752	mov	1, %o3
753	slln	%o3, %o1, %o3
754	ldn	[%o0], %o2
7551:
756	andncc	%o3, %o2, %g0		! test if the bit is clear
757	bnz,a,pn %ncc, 2f		! if so, then fail out
758	  mov	-1, %o0
759	andn	%o2, %o3, %o4		! clear the bit, and try to commit it
760	casn	[%o0], %o2, %o4
761	cmp	%o2, %o4
762	bne,a,pn %ncc, 1b		! failed to commit, try again
763	  mov	%o4, %o2
764	mov	%g0, %o0
7652:
766	retl
767	nop
768	SET_SIZE(atomic_clear_long_excl)
769
770#if !defined(_KERNEL)
771
772	/*
773	 * Spitfires and Blackbirds have a problem with membars in the
774	 * delay slot (SF_ERRATA_51).  For safety's sake, we assume
775	 * that the whole world needs the workaround.
776	 */
777	ENTRY(membar_enter)
778	membar	#StoreLoad|#StoreStore
779	retl
780	nop
781	SET_SIZE(membar_enter)
782
783	ENTRY(membar_exit)
784	membar	#LoadStore|#StoreStore
785	retl
786	nop
787	SET_SIZE(membar_exit)
788
789	ENTRY(membar_producer)
790	membar	#StoreStore
791	retl
792	nop
793	SET_SIZE(membar_producer)
794
795	ENTRY(membar_consumer)
796	membar	#LoadLoad
797	retl
798	nop
799	SET_SIZE(membar_consumer)
800
801#endif	/* !_KERNEL */
802