1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 .file "atomic.s" 28 29#include <sys/asm_linkage.h> 30 31#if defined(_KERNEL) 32 /* 33 * Legacy kernel interfaces; they will go away the moment our closed 34 * bins no longer require them. 35 */ 36 ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function) 37 ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function) 38 ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function) 39 ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function) 40 ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function) 41 ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function) 42 ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function) 43 ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function) 44#endif 45 46 /* 47 * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever 48 * separated, you need to also edit the libc sparc platform 49 * specific mapfile and remove the NODYNSORT attribute 50 * from atomic_inc_8_nv. 51 */ 52 ENTRY(atomic_inc_8) 53 ALTENTRY(atomic_inc_8_nv) 54 ALTENTRY(atomic_inc_uchar) 55 ALTENTRY(atomic_inc_uchar_nv) 56 ba add_8 57 add %g0, 1, %o1 58 SET_SIZE(atomic_inc_uchar_nv) 59 SET_SIZE(atomic_inc_uchar) 60 SET_SIZE(atomic_inc_8_nv) 61 SET_SIZE(atomic_inc_8) 62 63 /* 64 * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever 65 * separated, you need to also edit the libc sparc platform 66 * specific mapfile and remove the NODYNSORT attribute 67 * from atomic_dec_8_nv. 68 */ 69 ENTRY(atomic_dec_8) 70 ALTENTRY(atomic_dec_8_nv) 71 ALTENTRY(atomic_dec_uchar) 72 ALTENTRY(atomic_dec_uchar_nv) 73 ba add_8 74 sub %g0, 1, %o1 75 SET_SIZE(atomic_dec_uchar_nv) 76 SET_SIZE(atomic_dec_uchar) 77 SET_SIZE(atomic_dec_8_nv) 78 SET_SIZE(atomic_dec_8) 79 80 /* 81 * NOTE: If atomic_add_8 and atomic_add_8_nv are ever 82 * separated, you need to also edit the libc sparc platform 83 * specific mapfile and remove the NODYNSORT attribute 84 * from atomic_add_8_nv. 85 */ 86 ENTRY(atomic_add_8) 87 ALTENTRY(atomic_add_8_nv) 88 ALTENTRY(atomic_add_char) 89 ALTENTRY(atomic_add_char_nv) 90add_8: 91 and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right 92 xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left 93 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 94 set 0xff, %o3 ! %o3 = mask 95 sll %o3, %g1, %o3 ! %o3 = shifted to bit offset 96 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 97 and %o1, %o3, %o1 ! %o1 = single byte value 98 andn %o0, 0x3, %o0 ! %o0 = word address 99 ld [%o0], %o2 ! read old value 1001: 101 add %o2, %o1, %o5 ! add value to the old value 102 and %o5, %o3, %o5 ! clear other bits 103 andn %o2, %o3, %o4 ! clear target bits 104 or %o4, %o5, %o5 ! insert the new value 105 cas [%o0], %o2, %o5 106 cmp %o2, %o5 107 bne,a,pn %icc, 1b 108 mov %o5, %o2 ! %o2 = old value 109 add %o2, %o1, %o5 110 and %o5, %o3, %o5 111 retl 112 srl %o5, %g1, %o0 ! %o0 = new value 113 SET_SIZE(atomic_add_char_nv) 114 SET_SIZE(atomic_add_char) 115 SET_SIZE(atomic_add_8_nv) 116 SET_SIZE(atomic_add_8) 117 118 /* 119 * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever 120 * separated, you need to also edit the libc sparc platform 121 * specific mapfile and remove the NODYNSORT attribute 122 * from atomic_inc_16_nv. 123 */ 124 ENTRY(atomic_inc_16) 125 ALTENTRY(atomic_inc_16_nv) 126 ALTENTRY(atomic_inc_ushort) 127 ALTENTRY(atomic_inc_ushort_nv) 128 ba add_16 129 add %g0, 1, %o1 130 SET_SIZE(atomic_inc_ushort_nv) 131 SET_SIZE(atomic_inc_ushort) 132 SET_SIZE(atomic_inc_16_nv) 133 SET_SIZE(atomic_inc_16) 134 135 /* 136 * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever 137 * separated, you need to also edit the libc sparc platform 138 * specific mapfile and remove the NODYNSORT attribute 139 * from atomic_dec_16_nv. 140 */ 141 ENTRY(atomic_dec_16) 142 ALTENTRY(atomic_dec_16_nv) 143 ALTENTRY(atomic_dec_ushort) 144 ALTENTRY(atomic_dec_ushort_nv) 145 ba add_16 146 sub %g0, 1, %o1 147 SET_SIZE(atomic_dec_ushort_nv) 148 SET_SIZE(atomic_dec_ushort) 149 SET_SIZE(atomic_dec_16_nv) 150 SET_SIZE(atomic_dec_16) 151 152 /* 153 * NOTE: If atomic_add_16 and atomic_add_16_nv are ever 154 * separated, you need to also edit the libc sparc platform 155 * specific mapfile and remove the NODYNSORT attribute 156 * from atomic_add_16_nv. 157 */ 158 ENTRY(atomic_add_16) 159 ALTENTRY(atomic_add_16_nv) 160 ALTENTRY(atomic_add_short) 161 ALTENTRY(atomic_add_short_nv) 162add_16: 163 and %o0, 0x2, %o4 ! %o4 = byte offset, left-to-right 164 xor %o4, 0x2, %g1 ! %g1 = byte offset, right-to-left 165 sll %o4, 3, %o4 ! %o4 = bit offset, left-to-right 166 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 167 sethi %hi(0xffff0000), %o3 ! %o3 = mask 168 srl %o3, %o4, %o3 ! %o3 = shifted to bit offset 169 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 170 and %o1, %o3, %o1 ! %o1 = single short value 171 andn %o0, 0x2, %o0 ! %o0 = word address 172 ! if low-order bit is 1, we will properly get an alignment fault here 173 ld [%o0], %o2 ! read old value 1741: 175 add %o1, %o2, %o5 ! add value to the old value 176 and %o5, %o3, %o5 ! clear other bits 177 andn %o2, %o3, %o4 ! clear target bits 178 or %o4, %o5, %o5 ! insert the new value 179 cas [%o0], %o2, %o5 180 cmp %o2, %o5 181 bne,a,pn %icc, 1b 182 mov %o5, %o2 ! %o2 = old value 183 add %o1, %o2, %o5 184 and %o5, %o3, %o5 185 retl 186 srl %o5, %g1, %o0 ! %o0 = new value 187 SET_SIZE(atomic_add_short_nv) 188 SET_SIZE(atomic_add_short) 189 SET_SIZE(atomic_add_16_nv) 190 SET_SIZE(atomic_add_16) 191 192 /* 193 * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever 194 * separated, you need to also edit the libc sparc platform 195 * specific mapfile and remove the NODYNSORT attribute 196 * from atomic_inc_32_nv. 197 */ 198 ENTRY(atomic_inc_32) 199 ALTENTRY(atomic_inc_32_nv) 200 ALTENTRY(atomic_inc_uint) 201 ALTENTRY(atomic_inc_uint_nv) 202 ALTENTRY(atomic_inc_ulong) 203 ALTENTRY(atomic_inc_ulong_nv) 204 ba add_32 205 add %g0, 1, %o1 206 SET_SIZE(atomic_inc_ulong_nv) 207 SET_SIZE(atomic_inc_ulong) 208 SET_SIZE(atomic_inc_uint_nv) 209 SET_SIZE(atomic_inc_uint) 210 SET_SIZE(atomic_inc_32_nv) 211 SET_SIZE(atomic_inc_32) 212 213 /* 214 * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever 215 * separated, you need to also edit the libc sparc platform 216 * specific mapfile and remove the NODYNSORT attribute 217 * from atomic_dec_32_nv. 218 */ 219 ENTRY(atomic_dec_32) 220 ALTENTRY(atomic_dec_32_nv) 221 ALTENTRY(atomic_dec_uint) 222 ALTENTRY(atomic_dec_uint_nv) 223 ALTENTRY(atomic_dec_ulong) 224 ALTENTRY(atomic_dec_ulong_nv) 225 ba add_32 226 sub %g0, 1, %o1 227 SET_SIZE(atomic_dec_ulong_nv) 228 SET_SIZE(atomic_dec_ulong) 229 SET_SIZE(atomic_dec_uint_nv) 230 SET_SIZE(atomic_dec_uint) 231 SET_SIZE(atomic_dec_32_nv) 232 SET_SIZE(atomic_dec_32) 233 234 /* 235 * NOTE: If atomic_add_32 and atomic_add_32_nv are ever 236 * separated, you need to also edit the libc sparc platform 237 * specific mapfile and remove the NODYNSORT attribute 238 * from atomic_add_32_nv. 239 */ 240 ENTRY(atomic_add_32) 241 ALTENTRY(atomic_add_32_nv) 242 ALTENTRY(atomic_add_int) 243 ALTENTRY(atomic_add_int_nv) 244 ALTENTRY(atomic_add_ptr) 245 ALTENTRY(atomic_add_ptr_nv) 246 ALTENTRY(atomic_add_long) 247 ALTENTRY(atomic_add_long_nv) 248add_32: 249 ld [%o0], %o2 2501: 251 add %o2, %o1, %o3 252 cas [%o0], %o2, %o3 253 cmp %o2, %o3 254 bne,a,pn %icc, 1b 255 mov %o3, %o2 256 retl 257 add %o2, %o1, %o0 ! return new value 258 SET_SIZE(atomic_add_long_nv) 259 SET_SIZE(atomic_add_long) 260 SET_SIZE(atomic_add_ptr_nv) 261 SET_SIZE(atomic_add_ptr) 262 SET_SIZE(atomic_add_int_nv) 263 SET_SIZE(atomic_add_int) 264 SET_SIZE(atomic_add_32_nv) 265 SET_SIZE(atomic_add_32) 266 267 /* 268 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever 269 * separated, you need to also edit the libc sparc platform 270 * specific mapfile and remove the NODYNSORT attribute 271 * from atomic_inc_64_nv. 272 */ 273 ENTRY(atomic_inc_64) 274 ALTENTRY(atomic_inc_64_nv) 275 ba add_64 276 add %g0, 1, %o1 277 SET_SIZE(atomic_inc_64_nv) 278 SET_SIZE(atomic_inc_64) 279 280 /* 281 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever 282 * separated, you need to also edit the libc sparc platform 283 * specific mapfile and remove the NODYNSORT attribute 284 * from atomic_dec_64_nv. 285 */ 286 ENTRY(atomic_dec_64) 287 ALTENTRY(atomic_dec_64_nv) 288 ba add_64 289 sub %g0, 1, %o1 290 SET_SIZE(atomic_dec_64_nv) 291 SET_SIZE(atomic_dec_64) 292 293 /* 294 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever 295 * separated, you need to also edit the libc sparc platform 296 * specific mapfile and remove the NODYNSORT attribute 297 * from atomic_add_64_nv. 298 */ 299 ENTRY(atomic_add_64) 300 ALTENTRY(atomic_add_64_nv) 301 sllx %o1, 32, %o1 ! upper 32 in %o1, lower in %o2 302 srl %o2, 0, %o2 303 add %o1, %o2, %o1 ! convert 2 32-bit args into 1 64-bit 304add_64: 305 ldx [%o0], %o2 3061: 307 add %o2, %o1, %o3 308 casx [%o0], %o2, %o3 309 cmp %o2, %o3 310 bne,a,pn %xcc, 1b 311 mov %o3, %o2 312 add %o2, %o1, %o1 ! return lower 32-bits in %o1 313 retl 314 srlx %o1, 32, %o0 ! return upper 32-bits in %o0 315 SET_SIZE(atomic_add_64_nv) 316 SET_SIZE(atomic_add_64) 317 318 /* 319 * NOTE: If atomic_or_8 and atomic_or_8_nv are ever 320 * separated, you need to also edit the libc sparc platform 321 * specific mapfile and remove the NODYNSORT attribute 322 * from atomic_or_8_nv. 323 */ 324 ENTRY(atomic_or_8) 325 ALTENTRY(atomic_or_8_nv) 326 ALTENTRY(atomic_or_uchar) 327 ALTENTRY(atomic_or_uchar_nv) 328 and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right 329 xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left 330 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 331 set 0xff, %o3 ! %o3 = mask 332 sll %o3, %g1, %o3 ! %o3 = shifted to bit offset 333 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 334 and %o1, %o3, %o1 ! %o1 = single byte value 335 andn %o0, 0x3, %o0 ! %o0 = word address 336 ld [%o0], %o2 ! read old value 3371: 338 or %o2, %o1, %o5 ! or in the new value 339 cas [%o0], %o2, %o5 340 cmp %o2, %o5 341 bne,a,pn %icc, 1b 342 mov %o5, %o2 ! %o2 = old value 343 or %o2, %o1, %o5 344 and %o5, %o3, %o5 345 retl 346 srl %o5, %g1, %o0 ! %o0 = new value 347 SET_SIZE(atomic_or_uchar_nv) 348 SET_SIZE(atomic_or_uchar) 349 SET_SIZE(atomic_or_8_nv) 350 SET_SIZE(atomic_or_8) 351 352 /* 353 * NOTE: If atomic_or_16 and atomic_or_16_nv are ever 354 * separated, you need to also edit the libc sparc platform 355 * specific mapfile and remove the NODYNSORT attribute 356 * from atomic_or_16_nv. 357 */ 358 ENTRY(atomic_or_16) 359 ALTENTRY(atomic_or_16_nv) 360 ALTENTRY(atomic_or_ushort) 361 ALTENTRY(atomic_or_ushort_nv) 362 and %o0, 0x2, %o4 ! %o4 = byte offset, left-to-right 363 xor %o4, 0x2, %g1 ! %g1 = byte offset, right-to-left 364 sll %o4, 3, %o4 ! %o4 = bit offset, left-to-right 365 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 366 sethi %hi(0xffff0000), %o3 ! %o3 = mask 367 srl %o3, %o4, %o3 ! %o3 = shifted to bit offset 368 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 369 and %o1, %o3, %o1 ! %o1 = single short value 370 andn %o0, 0x2, %o0 ! %o0 = word address 371 ! if low-order bit is 1, we will properly get an alignment fault here 372 ld [%o0], %o2 ! read old value 3731: 374 or %o2, %o1, %o5 ! or in the new value 375 cas [%o0], %o2, %o5 376 cmp %o2, %o5 377 bne,a,pn %icc, 1b 378 mov %o5, %o2 ! %o2 = old value 379 or %o2, %o1, %o5 ! or in the new value 380 and %o5, %o3, %o5 381 retl 382 srl %o5, %g1, %o0 ! %o0 = new value 383 SET_SIZE(atomic_or_ushort_nv) 384 SET_SIZE(atomic_or_ushort) 385 SET_SIZE(atomic_or_16_nv) 386 SET_SIZE(atomic_or_16) 387 388 /* 389 * NOTE: If atomic_or_32 and atomic_or_32_nv are ever 390 * separated, you need to also edit the libc sparc platform 391 * specific mapfile and remove the NODYNSORT attribute 392 * from atomic_or_32_nv. 393 */ 394 ENTRY(atomic_or_32) 395 ALTENTRY(atomic_or_32_nv) 396 ALTENTRY(atomic_or_uint) 397 ALTENTRY(atomic_or_uint_nv) 398 ALTENTRY(atomic_or_ulong) 399 ALTENTRY(atomic_or_ulong_nv) 400 ld [%o0], %o2 4011: 402 or %o2, %o1, %o3 403 cas [%o0], %o2, %o3 404 cmp %o2, %o3 405 bne,a,pn %icc, 1b 406 mov %o3, %o2 407 retl 408 or %o2, %o1, %o0 ! return new value 409 SET_SIZE(atomic_or_ulong_nv) 410 SET_SIZE(atomic_or_ulong) 411 SET_SIZE(atomic_or_uint_nv) 412 SET_SIZE(atomic_or_uint) 413 SET_SIZE(atomic_or_32_nv) 414 SET_SIZE(atomic_or_32) 415 416 /* 417 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever 418 * separated, you need to also edit the libc sparc platform 419 * specific mapfile and remove the NODYNSORT attribute 420 * from atomic_or_64_nv. 421 */ 422 ENTRY(atomic_or_64) 423 ALTENTRY(atomic_or_64_nv) 424 sllx %o1, 32, %o1 ! upper 32 in %o1, lower in %o2 425 srl %o2, 0, %o2 426 add %o1, %o2, %o1 ! convert 2 32-bit args into 1 64-bit 427 ldx [%o0], %o2 4281: 429 or %o2, %o1, %o3 430 casx [%o0], %o2, %o3 431 cmp %o2, %o3 432 bne,a,pn %xcc, 1b 433 mov %o3, %o2 434 or %o2, %o1, %o1 ! return lower 32-bits in %o1 435 retl 436 srlx %o1, 32, %o0 ! return upper 32-bits in %o0 437 SET_SIZE(atomic_or_64_nv) 438 SET_SIZE(atomic_or_64) 439 440 /* 441 * NOTE: If atomic_and_8 and atomic_and_8_nv are ever 442 * separated, you need to also edit the libc sparc platform 443 * specific mapfile and remove the NODYNSORT attribute 444 * from atomic_and_8_nv. 445 */ 446 ENTRY(atomic_and_8) 447 ALTENTRY(atomic_and_8_nv) 448 ALTENTRY(atomic_and_uchar) 449 ALTENTRY(atomic_and_uchar_nv) 450 and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right 451 xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left 452 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 453 set 0xff, %o3 ! %o3 = mask 454 sll %o3, %g1, %o3 ! %o3 = shifted to bit offset 455 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 456 orn %o1, %o3, %o1 ! all ones in other bytes 457 andn %o0, 0x3, %o0 ! %o0 = word address 458 ld [%o0], %o2 ! read old value 4591: 460 and %o2, %o1, %o5 ! and in the new value 461 cas [%o0], %o2, %o5 462 cmp %o2, %o5 463 bne,a,pn %icc, 1b 464 mov %o5, %o2 ! %o2 = old value 465 and %o2, %o1, %o5 466 and %o5, %o3, %o5 467 retl 468 srl %o5, %g1, %o0 ! %o0 = new value 469 SET_SIZE(atomic_and_uchar_nv) 470 SET_SIZE(atomic_and_uchar) 471 SET_SIZE(atomic_and_8_nv) 472 SET_SIZE(atomic_and_8) 473 474 /* 475 * NOTE: If atomic_and_16 and atomic_and_16_nv are ever 476 * separated, you need to also edit the libc sparc platform 477 * specific mapfile and remove the NODYNSORT attribute 478 * from atomic_and_16_nv. 479 */ 480 ENTRY(atomic_and_16) 481 ALTENTRY(atomic_and_16_nv) 482 ALTENTRY(atomic_and_ushort) 483 ALTENTRY(atomic_and_ushort_nv) 484 and %o0, 0x2, %o4 ! %o4 = byte offset, left-to-right 485 xor %o4, 0x2, %g1 ! %g1 = byte offset, right-to-left 486 sll %o4, 3, %o4 ! %o4 = bit offset, left-to-right 487 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 488 sethi %hi(0xffff0000), %o3 ! %o3 = mask 489 srl %o3, %o4, %o3 ! %o3 = shifted to bit offset 490 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 491 orn %o1, %o3, %o1 ! all ones in the other half 492 andn %o0, 0x2, %o0 ! %o0 = word address 493 ! if low-order bit is 1, we will properly get an alignment fault here 494 ld [%o0], %o2 ! read old value 4951: 496 and %o2, %o1, %o5 ! and in the new value 497 cas [%o0], %o2, %o5 498 cmp %o2, %o5 499 bne,a,pn %icc, 1b 500 mov %o5, %o2 ! %o2 = old value 501 and %o2, %o1, %o5 502 and %o5, %o3, %o5 503 retl 504 srl %o5, %g1, %o0 ! %o0 = new value 505 SET_SIZE(atomic_and_ushort_nv) 506 SET_SIZE(atomic_and_ushort) 507 SET_SIZE(atomic_and_16_nv) 508 SET_SIZE(atomic_and_16) 509 510 /* 511 * NOTE: If atomic_and_32 and atomic_and_32_nv are ever 512 * separated, you need to also edit the libc sparc platform 513 * specific mapfile and remove the NODYNSORT attribute 514 * from atomic_and_32_nv. 515 */ 516 ENTRY(atomic_and_32) 517 ALTENTRY(atomic_and_32_nv) 518 ALTENTRY(atomic_and_uint) 519 ALTENTRY(atomic_and_uint_nv) 520 ALTENTRY(atomic_and_ulong) 521 ALTENTRY(atomic_and_ulong_nv) 522 ld [%o0], %o2 5231: 524 and %o2, %o1, %o3 525 cas [%o0], %o2, %o3 526 cmp %o2, %o3 527 bne,a,pn %icc, 1b 528 mov %o3, %o2 529 retl 530 and %o2, %o1, %o0 ! return new value 531 SET_SIZE(atomic_and_ulong_nv) 532 SET_SIZE(atomic_and_ulong) 533 SET_SIZE(atomic_and_uint_nv) 534 SET_SIZE(atomic_and_uint) 535 SET_SIZE(atomic_and_32_nv) 536 SET_SIZE(atomic_and_32) 537 538 /* 539 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever 540 * separated, you need to also edit the libc sparc platform 541 * specific mapfile and remove the NODYNSORT attribute 542 * from atomic_and_64_nv. 543 */ 544 ENTRY(atomic_and_64) 545 ALTENTRY(atomic_and_64_nv) 546 sllx %o1, 32, %o1 ! upper 32 in %o1, lower in %o2 547 srl %o2, 0, %o2 548 add %o1, %o2, %o1 ! convert 2 32-bit args into 1 64-bit 549 ldx [%o0], %o2 5501: 551 and %o2, %o1, %o3 552 casx [%o0], %o2, %o3 553 cmp %o2, %o3 554 bne,a,pn %xcc, 1b 555 mov %o3, %o2 556 and %o2, %o1, %o1 ! return lower 32-bits in %o1 557 retl 558 srlx %o1, 32, %o0 ! return upper 32-bits in %o0 559 SET_SIZE(atomic_and_64_nv) 560 SET_SIZE(atomic_and_64) 561 562 ENTRY(atomic_cas_8) 563 ALTENTRY(atomic_cas_uchar) 564 and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right 565 xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left 566 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 567 set 0xff, %o3 ! %o3 = mask 568 sll %o3, %g1, %o3 ! %o3 = shifted to bit offset 569 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 570 and %o1, %o3, %o1 ! %o1 = single byte value 571 sll %o2, %g1, %o2 ! %o2 = shifted to bit offset 572 and %o2, %o3, %o2 ! %o2 = single byte value 573 andn %o0, 0x3, %o0 ! %o0 = word address 574 ld [%o0], %o4 ! read old value 5751: 576 andn %o4, %o3, %o4 ! clear target bits 577 or %o4, %o2, %o5 ! insert the new value 578 or %o4, %o1, %o4 ! insert the comparison value 579 cas [%o0], %o4, %o5 580 cmp %o4, %o5 ! did we succeed? 581 be,pt %icc, 2f 582 and %o5, %o3, %o4 ! isolate the old value 583 cmp %o1, %o4 ! should we have succeeded? 584 be,a,pt %icc, 1b ! yes, try again 585 mov %o5, %o4 ! %o4 = old value 5862: 587 retl 588 srl %o4, %g1, %o0 ! %o0 = old value 589 SET_SIZE(atomic_cas_uchar) 590 SET_SIZE(atomic_cas_8) 591 592 ENTRY(atomic_cas_16) 593 ALTENTRY(atomic_cas_ushort) 594 and %o0, 0x2, %o4 ! %o4 = byte offset, left-to-right 595 xor %o4, 0x2, %g1 ! %g1 = byte offset, right-to-left 596 sll %o4, 3, %o4 ! %o4 = bit offset, left-to-right 597 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 598 sethi %hi(0xffff0000), %o3 ! %o3 = mask 599 srl %o3, %o4, %o3 ! %o3 = shifted to bit offset 600 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 601 and %o1, %o3, %o1 ! %o1 = single short value 602 sll %o2, %g1, %o2 ! %o2 = shifted to bit offset 603 and %o2, %o3, %o2 ! %o2 = single short value 604 andn %o0, 0x2, %o0 ! %o0 = word address 605 ! if low-order bit is 1, we will properly get an alignment fault here 606 ld [%o0], %o4 ! read old value 6071: 608 andn %o4, %o3, %o4 ! clear target bits 609 or %o4, %o2, %o5 ! insert the new value 610 or %o4, %o1, %o4 ! insert the comparison value 611 cas [%o0], %o4, %o5 612 cmp %o4, %o5 ! did we succeed? 613 be,pt %icc, 2f 614 and %o5, %o3, %o4 ! isolate the old value 615 cmp %o1, %o4 ! should we have succeeded? 616 be,a,pt %icc, 1b ! yes, try again 617 mov %o5, %o4 ! %o4 = old value 6182: 619 retl 620 srl %o4, %g1, %o0 ! %o0 = old value 621 SET_SIZE(atomic_cas_ushort) 622 SET_SIZE(atomic_cas_16) 623 624 ENTRY(atomic_cas_32) 625 ALTENTRY(atomic_cas_uint) 626 ALTENTRY(atomic_cas_ptr) 627 ALTENTRY(atomic_cas_ulong) 628 cas [%o0], %o1, %o2 629 retl 630 mov %o2, %o0 631 SET_SIZE(atomic_cas_ulong) 632 SET_SIZE(atomic_cas_ptr) 633 SET_SIZE(atomic_cas_uint) 634 SET_SIZE(atomic_cas_32) 635 636 ENTRY(atomic_cas_64) 637 sllx %o1, 32, %o1 ! cmp's upper 32 in %o1, lower in %o2 638 srl %o2, 0, %o2 ! convert 2 32-bit args into 1 64-bit 639 add %o1, %o2, %o1 640 sllx %o3, 32, %o2 ! newval upper 32 in %o3, lower in %o4 641 srl %o4, 0, %o4 ! setup %o2 to have newval 642 add %o2, %o4, %o2 643 casx [%o0], %o1, %o2 644 srl %o2, 0, %o1 ! return lower 32-bits in %o1 645 retl 646 srlx %o2, 32, %o0 ! return upper 32-bits in %o0 647 SET_SIZE(atomic_cas_64) 648 649 ENTRY(atomic_swap_8) 650 ALTENTRY(atomic_swap_uchar) 651 and %o0, 0x3, %o4 ! %o4 = byte offset, left-to-right 652 xor %o4, 0x3, %g1 ! %g1 = byte offset, right-to-left 653 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 654 set 0xff, %o3 ! %o3 = mask 655 sll %o3, %g1, %o3 ! %o3 = shifted to bit offset 656 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 657 and %o1, %o3, %o1 ! %o1 = single byte value 658 andn %o0, 0x3, %o0 ! %o0 = word address 659 ld [%o0], %o2 ! read old value 6601: 661 andn %o2, %o3, %o5 ! clear target bits 662 or %o5, %o1, %o5 ! insert the new value 663 cas [%o0], %o2, %o5 664 cmp %o2, %o5 665 bne,a,pn %icc, 1b 666 mov %o5, %o2 ! %o2 = old value 667 and %o5, %o3, %o5 668 retl 669 srl %o5, %g1, %o0 ! %o0 = old value 670 SET_SIZE(atomic_swap_uchar) 671 SET_SIZE(atomic_swap_8) 672 673 ENTRY(atomic_swap_16) 674 ALTENTRY(atomic_swap_ushort) 675 and %o0, 0x2, %o4 ! %o4 = byte offset, left-to-right 676 xor %o4, 0x2, %g1 ! %g1 = byte offset, right-to-left 677 sll %o4, 3, %o4 ! %o4 = bit offset, left-to-right 678 sll %g1, 3, %g1 ! %g1 = bit offset, right-to-left 679 sethi %hi(0xffff0000), %o3 ! %o3 = mask 680 srl %o3, %o4, %o3 ! %o3 = shifted to bit offset 681 sll %o1, %g1, %o1 ! %o1 = shifted to bit offset 682 and %o1, %o3, %o1 ! %o1 = single short value 683 andn %o0, 0x2, %o0 ! %o0 = word address 684 ! if low-order bit is 1, we will properly get an alignment fault here 685 ld [%o0], %o2 ! read old value 6861: 687 andn %o2, %o3, %o5 ! clear target bits 688 or %o5, %o1, %o5 ! insert the new value 689 cas [%o0], %o2, %o5 690 cmp %o2, %o5 691 bne,a,pn %icc, 1b 692 mov %o5, %o2 ! %o2 = old value 693 and %o5, %o3, %o5 694 retl 695 srl %o5, %g1, %o0 ! %o0 = old value 696 SET_SIZE(atomic_swap_ushort) 697 SET_SIZE(atomic_swap_16) 698 699 ENTRY(atomic_swap_32) 700 ALTENTRY(atomic_swap_uint) 701 ALTENTRY(atomic_swap_ptr) 702 ALTENTRY(atomic_swap_ulong) 703 ld [%o0], %o2 7041: 705 mov %o1, %o3 706 cas [%o0], %o2, %o3 707 cmp %o2, %o3 708 bne,a,pn %icc, 1b 709 mov %o3, %o2 710 retl 711 mov %o3, %o0 712 SET_SIZE(atomic_swap_ulong) 713 SET_SIZE(atomic_swap_ptr) 714 SET_SIZE(atomic_swap_uint) 715 SET_SIZE(atomic_swap_32) 716 717 ENTRY(atomic_swap_64) 718 sllx %o1, 32, %o1 ! upper 32 in %o1, lower in %o2 719 srl %o2, 0, %o2 720 add %o1, %o2, %o1 ! convert 2 32-bit args into 1 64-bit 721 ldx [%o0], %o2 7221: 723 mov %o1, %o3 724 casx [%o0], %o2, %o3 725 cmp %o2, %o3 726 bne,a,pn %xcc, 1b 727 mov %o3, %o2 728 srl %o3, 0, %o1 ! return lower 32-bits in %o1 729 retl 730 srlx %o3, 32, %o0 ! return upper 32-bits in %o0 731 SET_SIZE(atomic_swap_64) 732 733 ENTRY(atomic_set_long_excl) 734 mov 1, %o3 735 slln %o3, %o1, %o3 736 ldn [%o0], %o2 7371: 738 andcc %o2, %o3, %g0 ! test if the bit is set 739 bnz,a,pn %ncc, 2f ! if so, then fail out 740 mov -1, %o0 741 or %o2, %o3, %o4 ! set the bit, and try to commit it 742 casn [%o0], %o2, %o4 743 cmp %o2, %o4 744 bne,a,pn %ncc, 1b ! failed to commit, try again 745 mov %o4, %o2 746 mov %g0, %o0 7472: 748 retl 749 nop 750 SET_SIZE(atomic_set_long_excl) 751 752 ENTRY(atomic_clear_long_excl) 753 mov 1, %o3 754 slln %o3, %o1, %o3 755 ldn [%o0], %o2 7561: 757 andncc %o3, %o2, %g0 ! test if the bit is clear 758 bnz,a,pn %ncc, 2f ! if so, then fail out 759 mov -1, %o0 760 andn %o2, %o3, %o4 ! clear the bit, and try to commit it 761 casn [%o0], %o2, %o4 762 cmp %o2, %o4 763 bne,a,pn %ncc, 1b ! failed to commit, try again 764 mov %o4, %o2 765 mov %g0, %o0 7662: 767 retl 768 nop 769 SET_SIZE(atomic_clear_long_excl) 770 771#if !defined(_KERNEL) 772 773 /* 774 * Spitfires and Blackbirds have a problem with membars in the 775 * delay slot (SF_ERRATA_51). For safety's sake, we assume 776 * that the whole world needs the workaround. 777 */ 778 ENTRY(membar_enter) 779 membar #StoreLoad|#StoreStore 780 retl 781 nop 782 SET_SIZE(membar_enter) 783 784 ENTRY(membar_exit) 785 membar #LoadStore|#StoreStore 786 retl 787 nop 788 SET_SIZE(membar_exit) 789 790 ENTRY(membar_producer) 791 membar #StoreStore 792 retl 793 nop 794 SET_SIZE(membar_producer) 795 796 ENTRY(membar_consumer) 797 membar #LoadLoad 798 retl 799 nop 800 SET_SIZE(membar_consumer) 801 802#endif /* !_KERNEL */ 803