xref: /titanic_52/usr/src/common/atomic/sparc/atomic.s (revision a29e56d91db891741f1af9f6bbd3e3c3cac5f19b) 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27	.file	"atomic.s"
28
29#include <sys/asm_linkage.h>
30
31#if defined(_KERNEL)
32	/*
33	 * Legacy kernel interfaces; they will go away the moment our closed
34	 * bins no longer require them.
35	 */
36	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
37	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
38	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
39	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
40	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
41	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
42	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
43	ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
44#endif
45
46	/*
47	 * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
48	 * separated, you need to also edit the libc sparc platform
49	 * specific mapfile and remove the NODYNSORT attribute
50	 * from atomic_inc_8_nv.
51	 */
52	ENTRY(atomic_inc_8)
53	ALTENTRY(atomic_inc_8_nv)
54	ALTENTRY(atomic_inc_uchar)
55	ALTENTRY(atomic_inc_uchar_nv)
56	ba	add_8
57	  add	%g0, 1, %o1
58	SET_SIZE(atomic_inc_uchar_nv)
59	SET_SIZE(atomic_inc_uchar)
60	SET_SIZE(atomic_inc_8_nv)
61	SET_SIZE(atomic_inc_8)
62
63	/*
64	 * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
65	 * separated, you need to also edit the libc sparc platform
66	 * specific mapfile and remove the NODYNSORT attribute
67	 * from atomic_dec_8_nv.
68	 */
69	ENTRY(atomic_dec_8)
70	ALTENTRY(atomic_dec_8_nv)
71	ALTENTRY(atomic_dec_uchar)
72	ALTENTRY(atomic_dec_uchar_nv)
73	ba	add_8
74	  sub	%g0, 1, %o1
75	SET_SIZE(atomic_dec_uchar_nv)
76	SET_SIZE(atomic_dec_uchar)
77	SET_SIZE(atomic_dec_8_nv)
78	SET_SIZE(atomic_dec_8)
79
80	/*
81	 * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
82	 * separated, you need to also edit the libc sparc platform
83	 * specific mapfile and remove the NODYNSORT attribute
84	 * from atomic_add_8_nv.
85	 */
86	ENTRY(atomic_add_8)
87	ALTENTRY(atomic_add_8_nv)
88	ALTENTRY(atomic_add_char)
89	ALTENTRY(atomic_add_char_nv)
90add_8:
91	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
92	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
93	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
94	set	0xff, %o3		! %o3 = mask
95	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
96	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
97	and	%o1, %o3, %o1		! %o1 = single byte value
98	andn	%o0, 0x3, %o0		! %o0 = word address
99	ld	[%o0], %o2		! read old value
1001:
101	add	%o2, %o1, %o5		! add value to the old value
102	and	%o5, %o3, %o5		! clear other bits
103	andn	%o2, %o3, %o4		! clear target bits
104	or	%o4, %o5, %o5		! insert the new value
105	cas	[%o0], %o2, %o5
106	cmp	%o2, %o5
107	bne,a,pn %icc, 1b
108	  mov	%o5, %o2		! %o2 = old value
109	add	%o2, %o1, %o5
110	and	%o5, %o3, %o5
111	retl
112	srl	%o5, %g1, %o0		! %o0 = new value
113	SET_SIZE(atomic_add_char_nv)
114	SET_SIZE(atomic_add_char)
115	SET_SIZE(atomic_add_8_nv)
116	SET_SIZE(atomic_add_8)
117
118	/*
119	 * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
120	 * separated, you need to also edit the libc sparc platform
121	 * specific mapfile and remove the NODYNSORT attribute
122	 * from atomic_inc_16_nv.
123	 */
124	ENTRY(atomic_inc_16)
125	ALTENTRY(atomic_inc_16_nv)
126	ALTENTRY(atomic_inc_ushort)
127	ALTENTRY(atomic_inc_ushort_nv)
128	ba	add_16
129	  add	%g0, 1, %o1
130	SET_SIZE(atomic_inc_ushort_nv)
131	SET_SIZE(atomic_inc_ushort)
132	SET_SIZE(atomic_inc_16_nv)
133	SET_SIZE(atomic_inc_16)
134
135	/*
136	 * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
137	 * separated, you need to also edit the libc sparc platform
138	 * specific mapfile and remove the NODYNSORT attribute
139	 * from atomic_dec_16_nv.
140	 */
141	ENTRY(atomic_dec_16)
142	ALTENTRY(atomic_dec_16_nv)
143	ALTENTRY(atomic_dec_ushort)
144	ALTENTRY(atomic_dec_ushort_nv)
145	ba	add_16
146	  sub	%g0, 1, %o1
147	SET_SIZE(atomic_dec_ushort_nv)
148	SET_SIZE(atomic_dec_ushort)
149	SET_SIZE(atomic_dec_16_nv)
150	SET_SIZE(atomic_dec_16)
151
152	/*
153	 * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
154	 * separated, you need to also edit the libc sparc platform
155	 * specific mapfile and remove the NODYNSORT attribute
156	 * from atomic_add_16_nv.
157	 */
158	ENTRY(atomic_add_16)
159	ALTENTRY(atomic_add_16_nv)
160	ALTENTRY(atomic_add_short)
161	ALTENTRY(atomic_add_short_nv)
162add_16:
163	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
164	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
165	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
166	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
167	sethi	%hi(0xffff0000), %o3	! %o3 = mask
168	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
169	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
170	and	%o1, %o3, %o1		! %o1 = single short value
171	andn	%o0, 0x2, %o0		! %o0 = word address
172	! if low-order bit is 1, we will properly get an alignment fault here
173	ld	[%o0], %o2		! read old value
1741:
175	add	%o1, %o2, %o5		! add value to the old value
176	and	%o5, %o3, %o5		! clear other bits
177	andn	%o2, %o3, %o4		! clear target bits
178	or	%o4, %o5, %o5		! insert the new value
179	cas	[%o0], %o2, %o5
180	cmp	%o2, %o5
181	bne,a,pn %icc, 1b
182	  mov	%o5, %o2		! %o2 = old value
183	add	%o1, %o2, %o5
184	and	%o5, %o3, %o5
185	retl
186	srl	%o5, %g1, %o0		! %o0 = new value
187	SET_SIZE(atomic_add_short_nv)
188	SET_SIZE(atomic_add_short)
189	SET_SIZE(atomic_add_16_nv)
190	SET_SIZE(atomic_add_16)
191
192	/*
193	 * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
194	 * separated, you need to also edit the libc sparc platform
195	 * specific mapfile and remove the NODYNSORT attribute
196	 * from atomic_inc_32_nv.
197	 */
198	ENTRY(atomic_inc_32)
199	ALTENTRY(atomic_inc_32_nv)
200	ALTENTRY(atomic_inc_uint)
201	ALTENTRY(atomic_inc_uint_nv)
202	ALTENTRY(atomic_inc_ulong)
203	ALTENTRY(atomic_inc_ulong_nv)
204	ba	add_32
205	  add	%g0, 1, %o1
206	SET_SIZE(atomic_inc_ulong_nv)
207	SET_SIZE(atomic_inc_ulong)
208	SET_SIZE(atomic_inc_uint_nv)
209	SET_SIZE(atomic_inc_uint)
210	SET_SIZE(atomic_inc_32_nv)
211	SET_SIZE(atomic_inc_32)
212
213	/*
214	 * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
215	 * separated, you need to also edit the libc sparc platform
216	 * specific mapfile and remove the NODYNSORT attribute
217	 * from atomic_dec_32_nv.
218	 */
219	ENTRY(atomic_dec_32)
220	ALTENTRY(atomic_dec_32_nv)
221	ALTENTRY(atomic_dec_uint)
222	ALTENTRY(atomic_dec_uint_nv)
223	ALTENTRY(atomic_dec_ulong)
224	ALTENTRY(atomic_dec_ulong_nv)
225	ba	add_32
226	  sub	%g0, 1, %o1
227	SET_SIZE(atomic_dec_ulong_nv)
228	SET_SIZE(atomic_dec_ulong)
229	SET_SIZE(atomic_dec_uint_nv)
230	SET_SIZE(atomic_dec_uint)
231	SET_SIZE(atomic_dec_32_nv)
232	SET_SIZE(atomic_dec_32)
233
234	/*
235	 * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
236	 * separated, you need to also edit the libc sparc platform
237	 * specific mapfile and remove the NODYNSORT attribute
238	 * from atomic_add_32_nv.
239	 */
240	ENTRY(atomic_add_32)
241	ALTENTRY(atomic_add_32_nv)
242	ALTENTRY(atomic_add_int)
243	ALTENTRY(atomic_add_int_nv)
244	ALTENTRY(atomic_add_ptr)
245	ALTENTRY(atomic_add_ptr_nv)
246	ALTENTRY(atomic_add_long)
247	ALTENTRY(atomic_add_long_nv)
248add_32:
249	ld	[%o0], %o2
2501:
251	add	%o2, %o1, %o3
252	cas	[%o0], %o2, %o3
253	cmp	%o2, %o3
254	bne,a,pn %icc, 1b
255	  mov	%o3, %o2
256	retl
257	add	%o2, %o1, %o0		! return new value
258	SET_SIZE(atomic_add_long_nv)
259	SET_SIZE(atomic_add_long)
260	SET_SIZE(atomic_add_ptr_nv)
261	SET_SIZE(atomic_add_ptr)
262	SET_SIZE(atomic_add_int_nv)
263	SET_SIZE(atomic_add_int)
264	SET_SIZE(atomic_add_32_nv)
265	SET_SIZE(atomic_add_32)
266
267	/*
268	 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
269	 * separated, you need to also edit the libc sparc platform
270	 * specific mapfile and remove the NODYNSORT attribute
271	 * from atomic_inc_64_nv.
272	 */
273	ENTRY(atomic_inc_64)
274	ALTENTRY(atomic_inc_64_nv)
275	ba	add_64
276	  add	%g0, 1, %o1
277	SET_SIZE(atomic_inc_64_nv)
278	SET_SIZE(atomic_inc_64)
279
280	/*
281	 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
282	 * separated, you need to also edit the libc sparc platform
283	 * specific mapfile and remove the NODYNSORT attribute
284	 * from atomic_dec_64_nv.
285	 */
286	ENTRY(atomic_dec_64)
287	ALTENTRY(atomic_dec_64_nv)
288	ba	add_64
289	  sub	%g0, 1, %o1
290	SET_SIZE(atomic_dec_64_nv)
291	SET_SIZE(atomic_dec_64)
292
293	/*
294	 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
295	 * separated, you need to also edit the libc sparc platform
296	 * specific mapfile and remove the NODYNSORT attribute
297	 * from atomic_add_64_nv.
298	 */
299	ENTRY(atomic_add_64)
300	ALTENTRY(atomic_add_64_nv)
301	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
302	srl	%o2, 0, %o2
303	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
304add_64:
305	ldx	[%o0], %o2
3061:
307	add	%o2, %o1, %o3
308	casx	[%o0], %o2, %o3
309	cmp	%o2, %o3
310	bne,a,pn %xcc, 1b
311	  mov	%o3, %o2
312	add	%o2, %o1, %o1		! return lower 32-bits in %o1
313	retl
314	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
315	SET_SIZE(atomic_add_64_nv)
316	SET_SIZE(atomic_add_64)
317
318	/*
319	 * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
320	 * separated, you need to also edit the libc sparc platform
321	 * specific mapfile and remove the NODYNSORT attribute
322	 * from atomic_or_8_nv.
323	 */
324	ENTRY(atomic_or_8)
325	ALTENTRY(atomic_or_8_nv)
326	ALTENTRY(atomic_or_uchar)
327	ALTENTRY(atomic_or_uchar_nv)
328	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
329	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
330	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
331	set	0xff, %o3		! %o3 = mask
332	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
333	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
334	and	%o1, %o3, %o1		! %o1 = single byte value
335	andn	%o0, 0x3, %o0		! %o0 = word address
336	ld	[%o0], %o2		! read old value
3371:
338	or	%o2, %o1, %o5		! or in the new value
339	cas	[%o0], %o2, %o5
340	cmp	%o2, %o5
341	bne,a,pn %icc, 1b
342	  mov	%o5, %o2		! %o2 = old value
343	or	%o2, %o1, %o5
344	and	%o5, %o3, %o5
345	retl
346	srl	%o5, %g1, %o0		! %o0 = new value
347	SET_SIZE(atomic_or_uchar_nv)
348	SET_SIZE(atomic_or_uchar)
349	SET_SIZE(atomic_or_8_nv)
350	SET_SIZE(atomic_or_8)
351
352	/*
353	 * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
354	 * separated, you need to also edit the libc sparc platform
355	 * specific mapfile and remove the NODYNSORT attribute
356	 * from atomic_or_16_nv.
357	 */
358	ENTRY(atomic_or_16)
359	ALTENTRY(atomic_or_16_nv)
360	ALTENTRY(atomic_or_ushort)
361	ALTENTRY(atomic_or_ushort_nv)
362	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
363	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
364	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
365	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
366	sethi	%hi(0xffff0000), %o3	! %o3 = mask
367	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
368	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
369	and	%o1, %o3, %o1		! %o1 = single short value
370	andn	%o0, 0x2, %o0		! %o0 = word address
371	! if low-order bit is 1, we will properly get an alignment fault here
372	ld	[%o0], %o2		! read old value
3731:
374	or	%o2, %o1, %o5		! or in the new value
375	cas	[%o0], %o2, %o5
376	cmp	%o2, %o5
377	bne,a,pn %icc, 1b
378	  mov	%o5, %o2		! %o2 = old value
379	or	%o2, %o1, %o5		! or in the new value
380	and	%o5, %o3, %o5
381	retl
382	srl	%o5, %g1, %o0		! %o0 = new value
383	SET_SIZE(atomic_or_ushort_nv)
384	SET_SIZE(atomic_or_ushort)
385	SET_SIZE(atomic_or_16_nv)
386	SET_SIZE(atomic_or_16)
387
388	/*
389	 * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
390	 * separated, you need to also edit the libc sparc platform
391	 * specific mapfile and remove the NODYNSORT attribute
392	 * from atomic_or_32_nv.
393	 */
394	ENTRY(atomic_or_32)
395	ALTENTRY(atomic_or_32_nv)
396	ALTENTRY(atomic_or_uint)
397	ALTENTRY(atomic_or_uint_nv)
398	ALTENTRY(atomic_or_ulong)
399	ALTENTRY(atomic_or_ulong_nv)
400	ld	[%o0], %o2
4011:
402	or	%o2, %o1, %o3
403	cas	[%o0], %o2, %o3
404	cmp	%o2, %o3
405	bne,a,pn %icc, 1b
406	  mov	%o3, %o2
407	retl
408	or	%o2, %o1, %o0		! return new value
409	SET_SIZE(atomic_or_ulong_nv)
410	SET_SIZE(atomic_or_ulong)
411	SET_SIZE(atomic_or_uint_nv)
412	SET_SIZE(atomic_or_uint)
413	SET_SIZE(atomic_or_32_nv)
414	SET_SIZE(atomic_or_32)
415
416	/*
417	 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
418	 * separated, you need to also edit the libc sparc platform
419	 * specific mapfile and remove the NODYNSORT attribute
420	 * from atomic_or_64_nv.
421	 */
422	ENTRY(atomic_or_64)
423	ALTENTRY(atomic_or_64_nv)
424	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
425	srl	%o2, 0, %o2
426	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
427	ldx	[%o0], %o2
4281:
429	or	%o2, %o1, %o3
430	casx	[%o0], %o2, %o3
431	cmp	%o2, %o3
432	bne,a,pn %xcc, 1b
433	  mov	%o3, %o2
434	or	%o2, %o1, %o1		! return lower 32-bits in %o1
435	retl
436	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
437	SET_SIZE(atomic_or_64_nv)
438	SET_SIZE(atomic_or_64)
439
440	/*
441	 * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
442	 * separated, you need to also edit the libc sparc platform
443	 * specific mapfile and remove the NODYNSORT attribute
444	 * from atomic_and_8_nv.
445	 */
446	ENTRY(atomic_and_8)
447	ALTENTRY(atomic_and_8_nv)
448	ALTENTRY(atomic_and_uchar)
449	ALTENTRY(atomic_and_uchar_nv)
450	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
451	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
452	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
453	set	0xff, %o3		! %o3 = mask
454	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
455	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
456	orn	%o1, %o3, %o1		! all ones in other bytes
457	andn	%o0, 0x3, %o0		! %o0 = word address
458	ld	[%o0], %o2		! read old value
4591:
460	and	%o2, %o1, %o5		! and in the new value
461	cas	[%o0], %o2, %o5
462	cmp	%o2, %o5
463	bne,a,pn %icc, 1b
464	  mov	%o5, %o2		! %o2 = old value
465	and	%o2, %o1, %o5
466	and	%o5, %o3, %o5
467	retl
468	srl	%o5, %g1, %o0		! %o0 = new value
469	SET_SIZE(atomic_and_uchar_nv)
470	SET_SIZE(atomic_and_uchar)
471	SET_SIZE(atomic_and_8_nv)
472	SET_SIZE(atomic_and_8)
473
474	/*
475	 * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
476	 * separated, you need to also edit the libc sparc platform
477	 * specific mapfile and remove the NODYNSORT attribute
478	 * from atomic_and_16_nv.
479	 */
480	ENTRY(atomic_and_16)
481	ALTENTRY(atomic_and_16_nv)
482	ALTENTRY(atomic_and_ushort)
483	ALTENTRY(atomic_and_ushort_nv)
484	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
485	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
486	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
487	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
488	sethi	%hi(0xffff0000), %o3	! %o3 = mask
489	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
490	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
491	orn	%o1, %o3, %o1		! all ones in the other half
492	andn	%o0, 0x2, %o0		! %o0 = word address
493	! if low-order bit is 1, we will properly get an alignment fault here
494	ld	[%o0], %o2		! read old value
4951:
496	and	%o2, %o1, %o5		! and in the new value
497	cas	[%o0], %o2, %o5
498	cmp	%o2, %o5
499	bne,a,pn %icc, 1b
500	  mov	%o5, %o2		! %o2 = old value
501	and	%o2, %o1, %o5
502	and	%o5, %o3, %o5
503	retl
504	srl	%o5, %g1, %o0		! %o0 = new value
505	SET_SIZE(atomic_and_ushort_nv)
506	SET_SIZE(atomic_and_ushort)
507	SET_SIZE(atomic_and_16_nv)
508	SET_SIZE(atomic_and_16)
509
510	/*
511	 * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
512	 * separated, you need to also edit the libc sparc platform
513	 * specific mapfile and remove the NODYNSORT attribute
514	 * from atomic_and_32_nv.
515	 */
516	ENTRY(atomic_and_32)
517	ALTENTRY(atomic_and_32_nv)
518	ALTENTRY(atomic_and_uint)
519	ALTENTRY(atomic_and_uint_nv)
520	ALTENTRY(atomic_and_ulong)
521	ALTENTRY(atomic_and_ulong_nv)
522	ld	[%o0], %o2
5231:
524	and	%o2, %o1, %o3
525	cas	[%o0], %o2, %o3
526	cmp	%o2, %o3
527	bne,a,pn %icc, 1b
528	  mov	%o3, %o2
529	retl
530	and	%o2, %o1, %o0		! return new value
531	SET_SIZE(atomic_and_ulong_nv)
532	SET_SIZE(atomic_and_ulong)
533	SET_SIZE(atomic_and_uint_nv)
534	SET_SIZE(atomic_and_uint)
535	SET_SIZE(atomic_and_32_nv)
536	SET_SIZE(atomic_and_32)
537
538	/*
539	 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
540	 * separated, you need to also edit the libc sparc platform
541	 * specific mapfile and remove the NODYNSORT attribute
542	 * from atomic_and_64_nv.
543	 */
544	ENTRY(atomic_and_64)
545	ALTENTRY(atomic_and_64_nv)
546	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
547	srl	%o2, 0, %o2
548	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
549	ldx	[%o0], %o2
5501:
551	and	%o2, %o1, %o3
552	casx	[%o0], %o2, %o3
553	cmp	%o2, %o3
554	bne,a,pn %xcc, 1b
555	  mov	%o3, %o2
556	and	%o2, %o1, %o1		! return lower 32-bits in %o1
557	retl
558	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
559	SET_SIZE(atomic_and_64_nv)
560	SET_SIZE(atomic_and_64)
561
562	ENTRY(atomic_cas_8)
563	ALTENTRY(atomic_cas_uchar)
564	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
565	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
566	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
567	set	0xff, %o3		! %o3 = mask
568	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
569	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
570	and	%o1, %o3, %o1		! %o1 = single byte value
571	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
572	and	%o2, %o3, %o2		! %o2 = single byte value
573	andn	%o0, 0x3, %o0		! %o0 = word address
574	ld	[%o0], %o4		! read old value
5751:
576	andn	%o4, %o3, %o4		! clear target bits
577	or	%o4, %o2, %o5		! insert the new value
578	or	%o4, %o1, %o4		! insert the comparison value
579	cas	[%o0], %o4, %o5
580	cmp	%o4, %o5		! did we succeed?
581	be,pt	%icc, 2f
582	  and	%o5, %o3, %o4		! isolate the old value
583	cmp	%o1, %o4		! should we have succeeded?
584	be,a,pt	%icc, 1b		! yes, try again
585	  mov	%o5, %o4		! %o4 = old value
5862:
587	retl
588	srl	%o4, %g1, %o0		! %o0 = old value
589	SET_SIZE(atomic_cas_uchar)
590	SET_SIZE(atomic_cas_8)
591
592	ENTRY(atomic_cas_16)
593	ALTENTRY(atomic_cas_ushort)
594	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
595	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
596	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
597	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
598	sethi	%hi(0xffff0000), %o3	! %o3 = mask
599	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
600	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
601	and	%o1, %o3, %o1		! %o1 = single short value
602	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
603	and	%o2, %o3, %o2		! %o2 = single short value
604	andn	%o0, 0x2, %o0		! %o0 = word address
605	! if low-order bit is 1, we will properly get an alignment fault here
606	ld	[%o0], %o4		! read old value
6071:
608	andn	%o4, %o3, %o4		! clear target bits
609	or	%o4, %o2, %o5		! insert the new value
610	or	%o4, %o1, %o4		! insert the comparison value
611	cas	[%o0], %o4, %o5
612	cmp	%o4, %o5		! did we succeed?
613	be,pt	%icc, 2f
614	  and	%o5, %o3, %o4		! isolate the old value
615	cmp	%o1, %o4		! should we have succeeded?
616	be,a,pt	%icc, 1b		! yes, try again
617	  mov	%o5, %o4		! %o4 = old value
6182:
619	retl
620	srl	%o4, %g1, %o0		! %o0 = old value
621	SET_SIZE(atomic_cas_ushort)
622	SET_SIZE(atomic_cas_16)
623
624	ENTRY(atomic_cas_32)
625	ALTENTRY(atomic_cas_uint)
626	ALTENTRY(atomic_cas_ptr)
627	ALTENTRY(atomic_cas_ulong)
628	cas	[%o0], %o1, %o2
629	retl
630	mov	%o2, %o0
631	SET_SIZE(atomic_cas_ulong)
632	SET_SIZE(atomic_cas_ptr)
633	SET_SIZE(atomic_cas_uint)
634	SET_SIZE(atomic_cas_32)
635
636	ENTRY(atomic_cas_64)
637	sllx	%o1, 32, %o1		! cmp's upper 32 in %o1, lower in %o2
638	srl	%o2, 0, %o2		! convert 2 32-bit args into 1 64-bit
639	add	%o1, %o2, %o1
640	sllx	%o3, 32, %o2		! newval upper 32 in %o3, lower in %o4
641	srl	%o4, 0, %o4		! setup %o2 to have newval
642	add	%o2, %o4, %o2
643	casx	[%o0], %o1, %o2
644	srl	%o2, 0, %o1		! return lower 32-bits in %o1
645	retl
646	srlx	%o2, 32, %o0		! return upper 32-bits in %o0
647	SET_SIZE(atomic_cas_64)
648
649	ENTRY(atomic_swap_8)
650	ALTENTRY(atomic_swap_uchar)
651	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
652	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
653	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
654	set	0xff, %o3		! %o3 = mask
655	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
656	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
657	and	%o1, %o3, %o1		! %o1 = single byte value
658	andn	%o0, 0x3, %o0		! %o0 = word address
659	ld	[%o0], %o2		! read old value
6601:
661	andn	%o2, %o3, %o5		! clear target bits
662	or	%o5, %o1, %o5		! insert the new value
663	cas	[%o0], %o2, %o5
664	cmp	%o2, %o5
665	bne,a,pn %icc, 1b
666	  mov	%o5, %o2		! %o2 = old value
667	and	%o5, %o3, %o5
668	retl
669	srl	%o5, %g1, %o0		! %o0 = old value
670	SET_SIZE(atomic_swap_uchar)
671	SET_SIZE(atomic_swap_8)
672
673	ENTRY(atomic_swap_16)
674	ALTENTRY(atomic_swap_ushort)
675	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
676	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
677	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
678	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
679	sethi	%hi(0xffff0000), %o3	! %o3 = mask
680	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
681	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
682	and	%o1, %o3, %o1		! %o1 = single short value
683	andn	%o0, 0x2, %o0		! %o0 = word address
684	! if low-order bit is 1, we will properly get an alignment fault here
685	ld	[%o0], %o2		! read old value
6861:
687	andn	%o2, %o3, %o5		! clear target bits
688	or	%o5, %o1, %o5		! insert the new value
689	cas	[%o0], %o2, %o5
690	cmp	%o2, %o5
691	bne,a,pn %icc, 1b
692	  mov	%o5, %o2		! %o2 = old value
693	and	%o5, %o3, %o5
694	retl
695	srl	%o5, %g1, %o0		! %o0 = old value
696	SET_SIZE(atomic_swap_ushort)
697	SET_SIZE(atomic_swap_16)
698
699	ENTRY(atomic_swap_32)
700	ALTENTRY(atomic_swap_uint)
701	ALTENTRY(atomic_swap_ptr)
702	ALTENTRY(atomic_swap_ulong)
703	ld	[%o0], %o2
7041:
705	mov	%o1, %o3
706	cas	[%o0], %o2, %o3
707	cmp	%o2, %o3
708	bne,a,pn %icc, 1b
709	  mov	%o3, %o2
710	retl
711	mov	%o3, %o0
712	SET_SIZE(atomic_swap_ulong)
713	SET_SIZE(atomic_swap_ptr)
714	SET_SIZE(atomic_swap_uint)
715	SET_SIZE(atomic_swap_32)
716
717	ENTRY(atomic_swap_64)
718	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
719	srl	%o2, 0, %o2
720	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
721	ldx	[%o0], %o2
7221:
723	mov	%o1, %o3
724	casx	[%o0], %o2, %o3
725	cmp	%o2, %o3
726	bne,a,pn %xcc, 1b
727	  mov	%o3, %o2
728	srl	%o3, 0, %o1		! return lower 32-bits in %o1
729	retl
730	srlx	%o3, 32, %o0		! return upper 32-bits in %o0
731	SET_SIZE(atomic_swap_64)
732
733	ENTRY(atomic_set_long_excl)
734	mov	1, %o3
735	slln	%o3, %o1, %o3
736	ldn	[%o0], %o2
7371:
738	andcc	%o2, %o3, %g0		! test if the bit is set
739	bnz,a,pn %ncc, 2f		! if so, then fail out
740	  mov	-1, %o0
741	or	%o2, %o3, %o4		! set the bit, and try to commit it
742	casn	[%o0], %o2, %o4
743	cmp	%o2, %o4
744	bne,a,pn %ncc, 1b		! failed to commit, try again
745	  mov	%o4, %o2
746	mov	%g0, %o0
7472:
748	retl
749	nop
750	SET_SIZE(atomic_set_long_excl)
751
752	ENTRY(atomic_clear_long_excl)
753	mov	1, %o3
754	slln	%o3, %o1, %o3
755	ldn	[%o0], %o2
7561:
757	andncc	%o3, %o2, %g0		! test if the bit is clear
758	bnz,a,pn %ncc, 2f		! if so, then fail out
759	  mov	-1, %o0
760	andn	%o2, %o3, %o4		! clear the bit, and try to commit it
761	casn	[%o0], %o2, %o4
762	cmp	%o2, %o4
763	bne,a,pn %ncc, 1b		! failed to commit, try again
764	  mov	%o4, %o2
765	mov	%g0, %o0
7662:
767	retl
768	nop
769	SET_SIZE(atomic_clear_long_excl)
770
771#if !defined(_KERNEL)
772
773	/*
774	 * Spitfires and Blackbirds have a problem with membars in the
775	 * delay slot (SF_ERRATA_51).  For safety's sake, we assume
776	 * that the whole world needs the workaround.
777	 */
778	ENTRY(membar_enter)
779	membar	#StoreLoad|#StoreStore
780	retl
781	nop
782	SET_SIZE(membar_enter)
783
784	ENTRY(membar_exit)
785	membar	#LoadStore|#StoreStore
786	retl
787	nop
788	SET_SIZE(membar_exit)
789
790	ENTRY(membar_producer)
791	membar	#StoreStore
792	retl
793	nop
794	SET_SIZE(membar_producer)
795
796	ENTRY(membar_consumer)
797	membar	#LoadLoad
798	retl
799	nop
800	SET_SIZE(membar_consumer)
801
802#endif	/* !_KERNEL */
803