1d265f300SJohn Baldwin.\" Copyright (c) 2000-2001 John H. Baldwin <jhb@FreeBSD.org> 2eaca6183SJohn Baldwin.\" All rights reserved. 3eaca6183SJohn Baldwin.\" 4eaca6183SJohn Baldwin.\" Redistribution and use in source and binary forms, with or without 5eaca6183SJohn Baldwin.\" modification, are permitted provided that the following conditions 6eaca6183SJohn Baldwin.\" are met: 7eaca6183SJohn Baldwin.\" 1. Redistributions of source code must retain the above copyright 8eaca6183SJohn Baldwin.\" notice, this list of conditions and the following disclaimer. 9eaca6183SJohn Baldwin.\" 2. Redistributions in binary form must reproduce the above copyright 10eaca6183SJohn Baldwin.\" notice, this list of conditions and the following disclaimer in the 11eaca6183SJohn Baldwin.\" documentation and/or other materials provided with the distribution. 12eaca6183SJohn Baldwin.\" 13eaca6183SJohn Baldwin.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR 14eaca6183SJohn Baldwin.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 15eaca6183SJohn Baldwin.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 16eaca6183SJohn Baldwin.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT, 17eaca6183SJohn Baldwin.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 18eaca6183SJohn Baldwin.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19eaca6183SJohn Baldwin.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 20eaca6183SJohn Baldwin.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21eaca6183SJohn Baldwin.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 22eaca6183SJohn Baldwin.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23eaca6183SJohn Baldwin.\" 24eaca6183SJohn Baldwin.\" $FreeBSD$ 25eaca6183SJohn Baldwin.\" 26eaca6183SJohn Baldwin.Dd October 27, 2000 27eaca6183SJohn Baldwin.Os 28eaca6183SJohn Baldwin.Dt ATOMIC 9 29eaca6183SJohn Baldwin.Sh NAME 30eaca6183SJohn Baldwin.Nm atomic_add , 31eaca6183SJohn Baldwin.Nm atomic_clear , 32eaca6183SJohn Baldwin.Nm atomic_cmpset , 334ea211a4SJohn Baldwin.Nm atomic_fetchadd , 34eaca6183SJohn Baldwin.Nm atomic_load , 35eaca6183SJohn Baldwin.Nm atomic_readandclear , 36eaca6183SJohn Baldwin.Nm atomic_set , 37eaca6183SJohn Baldwin.Nm atomic_subtract , 38eaca6183SJohn Baldwin.Nm atomic_store 39eaca6183SJohn Baldwin.Nd atomic operations 40eaca6183SJohn Baldwin.Sh SYNOPSIS 4132eef9aeSRuslan Ermilov.In sys/types.h 4232eef9aeSRuslan Ermilov.In machine/atomic.h 43eaca6183SJohn Baldwin.Ft void 44c6a51f1cSRuslan Ermilov.Fn atomic_add_[acq_|rel_]<type> "volatile <type> *p" "<type> v" 45eaca6183SJohn Baldwin.Ft void 46c6a51f1cSRuslan Ermilov.Fn atomic_clear_[acq_|rel_]<type> "volatile <type> *p" "<type> v" 47eaca6183SJohn Baldwin.Ft int 48c6a51f1cSRuslan Ermilov.Fo atomic_cmpset_[acq_|rel_]<type> 49c6a51f1cSRuslan Ermilov.Fa "volatile <type> *dst" 50c6a51f1cSRuslan Ermilov.Fa "<type> old" 51c6a51f1cSRuslan Ermilov.Fa "<type> new" 52eaca6183SJohn Baldwin.Fc 53c6a51f1cSRuslan Ermilov.Ft <type> 544ea211a4SJohn Baldwin.Fn atomic_fetchadd_<type> "volatile <type> *p" "<type> v" 554ea211a4SJohn Baldwin.Ft <type> 56c6a51f1cSRuslan Ermilov.Fn atomic_load_acq_<type> "volatile <type> *p" 57c6a51f1cSRuslan Ermilov.Ft <type> 58c6a51f1cSRuslan Ermilov.Fn atomic_readandclear_<type> "volatile <type> *p" 59eaca6183SJohn Baldwin.Ft void 60c6a51f1cSRuslan Ermilov.Fn atomic_set_[acq_|rel_]<type> "volatile <type> *p" "<type> v" 61eaca6183SJohn Baldwin.Ft void 62c6a51f1cSRuslan Ermilov.Fn atomic_subtract_[acq_|rel_]<type> "volatile <type> *p" "<type> v" 63eaca6183SJohn Baldwin.Ft void 64c6a51f1cSRuslan Ermilov.Fn atomic_store_rel_<type> "volatile <type> *p" "<type> v" 652be6c09fSRuslan Ermilov.rm LB RB La Ra 66eaca6183SJohn Baldwin.Sh DESCRIPTION 67eaca6183SJohn BaldwinEach of the atomic operations is guaranteed to be atomic in the presence of 68eaca6183SJohn Baldwininterrupts. 69eaca6183SJohn BaldwinThey can be used to implement reference counts or as building blocks for more 70eaca6183SJohn Baldwinadvanced synchronization primitives such as mutexes. 71eaca6183SJohn Baldwin.Ss Types 722be6c09fSRuslan ErmilovEach atomic operation operates on a specific 730640e9e0SHiten Pandya.Fa type . 74eaca6183SJohn BaldwinThe type to use is indicated in the function name. 75eaca6183SJohn BaldwinThe available types that can be used are: 762be6c09fSRuslan Ermilov.Pp 772be6c09fSRuslan Ermilov.Bl -tag -offset indent -width short -compact 782be6c09fSRuslan Ermilov.It Li int 79eaca6183SJohn Baldwinunsigned integer 802be6c09fSRuslan Ermilov.It Li long 81eaca6183SJohn Baldwinunsigned long integer 822be6c09fSRuslan Ermilov.It Li ptr 83eaca6183SJohn Baldwinunsigned integer the size of a pointer 842be6c09fSRuslan Ermilov.It Li 32 85eaca6183SJohn Baldwinunsigned 32-bit integer 862be6c09fSRuslan Ermilov.It Li 64 87eaca6183SJohn Baldwinunsigned 64-bit integer 88eaca6183SJohn Baldwin.El 89eaca6183SJohn Baldwin.Pp 90eaca6183SJohn BaldwinFor example, the function to atomically add two integers is called 91eaca6183SJohn Baldwin.Fn atomic_add_int . 92c645e17aSJake Burkholder.Pp 932be6c09fSRuslan ErmilovCertain architectures also provide operations for types smaller than 942be6c09fSRuslan Ermilov.Dq Li int . 952be6c09fSRuslan Ermilov.Pp 962be6c09fSRuslan Ermilov.Bl -tag -offset indent -width short -compact 972be6c09fSRuslan Ermilov.It Li char 98c645e17aSJake Burkholderunsigned character 992be6c09fSRuslan Ermilov.It Li short 100c645e17aSJake Burkholderunsigned short integer 1012be6c09fSRuslan Ermilov.It Li 8 102c645e17aSJake Burkholderunsigned 8-bit integer 1032be6c09fSRuslan Ermilov.It Li 16 104c645e17aSJake Burkholderunsigned 16-bit integer 105c645e17aSJake Burkholder.El 106c645e17aSJake Burkholder.Pp 107c645e17aSJake BurkholderThese must not be used in MI code because the instructions to implement them 108c645e17aSJake Burkholderefficiently may not be available. 109eaca6183SJohn Baldwin.Ss Memory Barriers 1102be6c09fSRuslan ErmilovMemory barriers are used to guarantee the order of data accesses in 111eaca6183SJohn Baldwintwo ways. 112eaca6183SJohn BaldwinFirst, they specify hints to the compiler to not re-order or optimize the 113eaca6183SJohn Baldwinoperations. 1142be6c09fSRuslan ErmilovSecond, on architectures that do not guarantee ordered data accesses, 115eaca6183SJohn Baldwinspecial instructions or special variants of instructions are used to indicate 116eaca6183SJohn Baldwinto the processor that data accesses need to occur in a certain order. 117eaca6183SJohn BaldwinAs a result, most of the atomic operations have three variants in order to 118eaca6183SJohn Baldwininclude optional memory barriers. 119eaca6183SJohn BaldwinThe first form just performs the operation without any explicit barriers. 1202be6c09fSRuslan ErmilovThe second form uses a read memory barrier, and the third variant uses a write 121eaca6183SJohn Baldwinmemory barrier. 122eaca6183SJohn Baldwin.Pp 123eaca6183SJohn BaldwinThe second variant of each operation includes a read memory barrier. 124eaca6183SJohn BaldwinThis barrier ensures that the effects of this operation are completed before the 125eaca6183SJohn Baldwineffects of any later data accesses. 126eaca6183SJohn BaldwinAs a result, the operation is said to have acquire semantics as it acquires a 127eaca6183SJohn Baldwinpseudo-lock requiring further operations to wait until it has completed. 128eaca6183SJohn BaldwinTo denote this, the suffix 1292be6c09fSRuslan Ermilov.Dq Li _acq 130eaca6183SJohn Baldwinis inserted into the function name immediately prior to the 1310640e9e0SHiten Pandya.Dq Li _ Ns Aq Fa type 132eaca6183SJohn Baldwinsuffix. 133eaca6183SJohn BaldwinFor example, to subtract two integers ensuring that any later writes will 134eaca6183SJohn Baldwinhappen after the subtraction is performed, use 135eaca6183SJohn Baldwin.Fn atomic_subtract_acq_int . 136eaca6183SJohn Baldwin.Pp 137eaca6183SJohn BaldwinThe third variant of each operation includes a write memory barrier. 138eaca6183SJohn BaldwinThis ensures that all effects of all previous data accesses are completed 139eaca6183SJohn Baldwinbefore this operation takes place. 140eaca6183SJohn BaldwinAs a result, the operation is said to have release semantics as it releases 141eaca6183SJohn Baldwinany pending data accesses to be completed before its operation is performed. 142eaca6183SJohn BaldwinTo denote this, the suffix 1432be6c09fSRuslan Ermilov.Dq Li _rel 144eaca6183SJohn Baldwinis inserted into the function name immediately prior to the 1450640e9e0SHiten Pandya.Dq Li _ Ns Aq Fa type 146eaca6183SJohn Baldwinsuffix. 147eaca6183SJohn BaldwinFor example, to add two long integers ensuring that all previous 148eaca6183SJohn Baldwinwrites will happen first, use 149eaca6183SJohn Baldwin.Fn atomic_add_rel_long . 150eaca6183SJohn Baldwin.Pp 151eaca6183SJohn BaldwinA practical example of using memory barriers is to ensure that data accesses 152eaca6183SJohn Baldwinthat are protected by a lock are all performed while the lock is held. 153eaca6183SJohn BaldwinTo achieve this, one would use a read barrier when acquiring the lock to 154eaca6183SJohn Baldwinguarantee that the lock is held before any protected operations are performed. 155eaca6183SJohn BaldwinFinally, one would use a write barrier when releasing the lock to ensure that 156eaca6183SJohn Baldwinall of the protected operations are completed before the lock is released. 157eaca6183SJohn Baldwin.Ss Multiple Processors 158eaca6183SJohn BaldwinThe current set of atomic operations do not necessarily guarantee atomicity 159eaca6183SJohn Baldwinacross multiple processors. 160eaca6183SJohn BaldwinTo guarantee atomicity across processors, not only does the individual 161d1ed27b6SJens Schweikhardtoperation need to be atomic on the processor performing the operation, but 162eaca6183SJohn Baldwinthe result of the operation needs to be pushed out to stable storage and the 163eaca6183SJohn Baldwincaches of all other processors on the system need to invalidate any cache 164eaca6183SJohn Baldwinlines that include the affected memory region. 165eaca6183SJohn BaldwinOn the 166eaca6183SJohn Baldwin.Tn i386 167eaca6183SJohn Baldwinarchitecture, the cache coherency model requires that the hardware perform 168eaca6183SJohn Baldwinthis task, thus the atomic operations are atomic across multiple processors. 169eaca6183SJohn BaldwinOn the 170eaca6183SJohn Baldwin.Tn ia64 171eaca6183SJohn Baldwinarchitecture, coherency is only guaranteed for pages that are configured to 172eaca6183SJohn Baldwinusing a caching policy of either uncached or write back. 173eaca6183SJohn Baldwin.Ss Semantics 174eaca6183SJohn BaldwinThis section describes the semantics of each operation using a C like notation. 175eaca6183SJohn Baldwin.Bl -hang 1762be6c09fSRuslan Ermilov.It Fn atomic_add p v 1772be6c09fSRuslan Ermilov.Bd -literal -compact 178eaca6183SJohn Baldwin*p += v; 179eaca6183SJohn Baldwin.Ed 1802be6c09fSRuslan Ermilov.It Fn atomic_clear p v 1812be6c09fSRuslan Ermilov.Bd -literal -compact 182eaca6183SJohn Baldwin*p &= ~v; 183eaca6183SJohn Baldwin.Ed 1842be6c09fSRuslan Ermilov.It Fn atomic_cmpset dst old new 1852be6c09fSRuslan Ermilov.Bd -literal -compact 186eaca6183SJohn Baldwinif (*dst == old) { 187eaca6183SJohn Baldwin *dst = new; 188eaca6183SJohn Baldwin return 1; 189eaca6183SJohn Baldwin} else 190eaca6183SJohn Baldwin return 0; 191eaca6183SJohn Baldwin.Ed 192eaca6183SJohn Baldwin.El 193eaca6183SJohn Baldwin.Pp 194eaca6183SJohn BaldwinThe 195eaca6183SJohn Baldwin.Fn atomic_cmpset 1962be6c09fSRuslan Ermilovfunctions are not implemented for the types 1972be6c09fSRuslan Ermilov.Dq Li char , 1982be6c09fSRuslan Ermilov.Dq Li short , 1992be6c09fSRuslan Ermilov.Dq Li 8 , 2002be6c09fSRuslan Ermilovand 2012be6c09fSRuslan Ermilov.Dq Li 16 . 202eaca6183SJohn Baldwin.Bl -hang 2034ea211a4SJohn Baldwin.It Fn atomic_fetchadd p v 2044ea211a4SJohn Baldwin.Bd -literal -compact 2054ea211a4SJohn Baldwintmp = *p; 2064ea211a4SJohn Baldwin*p += v; 2074ea211a4SJohn Baldwinreturn tmp; 2084ea211a4SJohn Baldwin.Ed 2094ea211a4SJohn Baldwin.El 2104ea211a4SJohn Baldwin.Pp 2114ea211a4SJohn BaldwinThe 2124ea211a4SJohn Baldwin.Fn atomic_fetchadd 2134ea211a4SJohn Baldwinfunctions are only implemented for the types 2144ea211a4SJohn Baldwin.Dq Li int 2154ea211a4SJohn Baldwinand 2164ea211a4SJohn Baldwin.Dq Li 32 2174ea211a4SJohn Baldwinand do not have any variants with memory barriers at this time. 2184ea211a4SJohn Baldwin.Bl -hang 2192be6c09fSRuslan Ermilov.It Fn atomic_load addr 2202be6c09fSRuslan Ermilov.Bd -literal -compact 221eaca6183SJohn Baldwinreturn (*addr) 222eaca6183SJohn Baldwin.Ed 223eaca6183SJohn Baldwin.El 224eaca6183SJohn Baldwin.Pp 225eaca6183SJohn BaldwinThe 226eaca6183SJohn Baldwin.Fn atomic_load 227eaca6183SJohn Baldwinfunctions always have acquire semantics. 228eaca6183SJohn Baldwin.Bl -hang 2292be6c09fSRuslan Ermilov.It Fn atomic_readandclear addr 2302be6c09fSRuslan Ermilov.Bd -literal -compact 231eaca6183SJohn Baldwintemp = *addr; 232eaca6183SJohn Baldwin*addr = 0; 233eaca6183SJohn Baldwinreturn (temp); 234eaca6183SJohn Baldwin.Ed 235eaca6183SJohn Baldwin.El 236eaca6183SJohn Baldwin.Pp 237eaca6183SJohn BaldwinThe 238eaca6183SJohn Baldwin.Fn atomic_readandclear 2392be6c09fSRuslan Ermilovfunctions are not implemented for the types 2402be6c09fSRuslan Ermilov.Dq Li char , 2412be6c09fSRuslan Ermilov.Dq Li short , 2422be6c09fSRuslan Ermilov.Dq Li ptr , 2432be6c09fSRuslan Ermilov.Dq Li 8 , 2442be6c09fSRuslan Ermilovand 2452be6c09fSRuslan Ermilov.Dq Li 16 2462be6c09fSRuslan Ermilovand do 247eaca6183SJohn Baldwinnot have any variants with memory barriers at this time. 248eaca6183SJohn Baldwin.Bl -hang 2492be6c09fSRuslan Ermilov.It Fn atomic_set p v 2502be6c09fSRuslan Ermilov.Bd -literal -compact 251eaca6183SJohn Baldwin*p |= v; 252eaca6183SJohn Baldwin.Ed 2532be6c09fSRuslan Ermilov.It Fn atomic_subtract p v 2542be6c09fSRuslan Ermilov.Bd -literal -compact 255eaca6183SJohn Baldwin*p -= v; 256eaca6183SJohn Baldwin.Ed 2572be6c09fSRuslan Ermilov.It Fn atomic_store p v 2582be6c09fSRuslan Ermilov.Bd -literal -compact 259eaca6183SJohn Baldwin*p = v; 260eaca6183SJohn Baldwin.Ed 261eaca6183SJohn Baldwin.El 262eaca6183SJohn Baldwin.Pp 263eaca6183SJohn BaldwinThe 264eaca6183SJohn Baldwin.Fn atomic_store 265eaca6183SJohn Baldwinfunctions always have release semantics. 266eaca6183SJohn Baldwin.Pp 267eaca6183SJohn BaldwinThe type 2682be6c09fSRuslan Ermilov.Dq Li 64 269eaca6183SJohn Baldwinis currently not implemented for any of the atomic operations on the 2704ea211a4SJohn Baldwin.Tn arm , 2714ea211a4SJohn Baldwin.Tn i386 , 2724ea211a4SJohn Baldwinand 2734ea211a4SJohn Baldwin.Tn powerpc 2744ea211a4SJohn Baldwinarchitectures. 275eaca6183SJohn Baldwin.Sh RETURN VALUES 2762be6c09fSRuslan ErmilovThe 277eaca6183SJohn Baldwin.Fn atomic_cmpset 2782be6c09fSRuslan Ermilovfunction 279eaca6183SJohn Baldwinreturns the result of the compare operation. 2802be6c09fSRuslan ErmilovThe 2814ea211a4SJohn Baldwin.Fn atomic_fetchadd , 2824ea211a4SJohn Baldwin.Fn atomic_load , 283eaca6183SJohn Baldwinand 284eaca6183SJohn Baldwin.Fn atomic_readandclear 2852be6c09fSRuslan Ermilovfunctions 286eaca6183SJohn Baldwinreturn the value at the specified address. 287eaca6183SJohn Baldwin.Sh EXAMPLES 288eaca6183SJohn BaldwinThis example uses the 289eaca6183SJohn Baldwin.Fn atomic_cmpset_acq_ptr 290eaca6183SJohn Baldwinand 291eaca6183SJohn Baldwin.Fn atomic_set_ptr 292eaca6183SJohn Baldwinfunctions to obtain a sleep mutex and handle recursion. 293eaca6183SJohn BaldwinSince the 294eaca6183SJohn Baldwin.Va mtx_lock 295eaca6183SJohn Baldwinmember of a 2962be6c09fSRuslan Ermilov.Vt "struct mtx" 297eaca6183SJohn Baldwinis a pointer, the 2982be6c09fSRuslan Ermilov.Dq Li ptr 299eaca6183SJohn Baldwintype is used. 300eaca6183SJohn Baldwin.Bd -literal 3014ea211a4SJohn Baldwin/* Try to obtain mtx_lock once. */ 302eaca6183SJohn Baldwin#define _obtain_lock(mp, tid) \\ 3034ea211a4SJohn Baldwin atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) 304eaca6183SJohn Baldwin 305eaca6183SJohn Baldwin/* Get a sleep lock, deal with recursion inline. */ 3064ea211a4SJohn Baldwin#define _get_sleep_lock(mp, tid, opts, file, line) do { \\ 3074ea211a4SJohn Baldwin uintptr_t _tid = (uintptr_t)(tid); \\ 3084ea211a4SJohn Baldwin \\ 309eaca6183SJohn Baldwin if (!_obtain_lock(mp, tid)) { \\ 3104ea211a4SJohn Baldwin if (((mp)->mtx_lock & MTX_FLAGMASK) != _tid) \\ 3114ea211a4SJohn Baldwin _mtx_lock_sleep((mp), _tid, (opts), (file), (line));\\ 312eaca6183SJohn Baldwin else { \\ 313eaca6183SJohn Baldwin atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSE); \\ 314eaca6183SJohn Baldwin (mp)->mtx_recurse++; \\ 315eaca6183SJohn Baldwin } \\ 316eaca6183SJohn Baldwin } \\ 317eaca6183SJohn Baldwin} while (0) 318eaca6183SJohn Baldwin.Ed 319eaca6183SJohn Baldwin.Sh HISTORY 320eaca6183SJohn BaldwinThe 321eaca6183SJohn Baldwin.Fn atomic_add , 322eaca6183SJohn Baldwin.Fn atomic_clear , 323eaca6183SJohn Baldwin.Fn atomic_set , 324eaca6183SJohn Baldwinand 325eaca6183SJohn Baldwin.Fn atomic_subtract 326eaca6183SJohn Baldwinoperations were first introduced in 327eaca6183SJohn Baldwin.Fx 3.0 . 3282be6c09fSRuslan ErmilovThis first set only supported the types 3292be6c09fSRuslan Ermilov.Dq Li char , 3302be6c09fSRuslan Ermilov.Dq Li short , 3312be6c09fSRuslan Ermilov.Dq Li int , 3322be6c09fSRuslan Ermilovand 3332be6c09fSRuslan Ermilov.Dq Li long . 334eaca6183SJohn BaldwinThe 335eaca6183SJohn Baldwin.Fn atomic_cmpset , 336eaca6183SJohn Baldwin.Fn atomic_load , 337eaca6183SJohn Baldwin.Fn atomic_readandclear , 338eaca6183SJohn Baldwinand 339eaca6183SJohn Baldwin.Fn atomic_store 340eaca6183SJohn Baldwinoperations were added in 341eaca6183SJohn Baldwin.Fx 5.0 . 3422be6c09fSRuslan ErmilovThe types 3432be6c09fSRuslan Ermilov.Dq Li 8 , 3442be6c09fSRuslan Ermilov.Dq Li 16 , 3452be6c09fSRuslan Ermilov.Dq Li 32 , 3462be6c09fSRuslan Ermilov.Dq Li 64 , 3472be6c09fSRuslan Ermilovand 3484ea211a4SJohn Baldwin.Dq Li ptr 3492be6c09fSRuslan Ermilovand all of the acquire and release variants 350eaca6183SJohn Baldwinwere added in 351eaca6183SJohn Baldwin.Fx 5.0 352eaca6183SJohn Baldwinas well. 3534ea211a4SJohn BaldwinThe 3544ea211a4SJohn Baldwin.Fn atomic_fetchadd 3554ea211a4SJohn Baldwinoperations were added in 3564ea211a4SJohn Baldwin.Fx 6.0 . 357