1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Atomic operand code-gen constructs. 10// 11//===----------------------------------------------------------------------===// 12 13//===---------------------------------- 14// Atomic fences 15//===---------------------------------- 16let AddedComplexity = 15, Size = 0 in 17def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering), 18 [(atomic_fence timm:$ordering, 0)]>, Sched<[]>; 19def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>; 20def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>; 21 22//===---------------------------------- 23// Atomic loads 24//===---------------------------------- 25 26// When they're actually atomic, only one addressing mode (GPR64sp) is 27// supported, but when they're relaxed and anything can be used, all the 28// standard modes would be valid and may give efficiency gains. 29 30// An atomic load operation that does not need either acquire or release 31// semantics. 32class relaxed_load<PatFrags base> 33 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 34 let IsAtomic = 1; 35 let IsAtomicOrderingAcquireOrStronger = 0; 36} 37 38// A atomic load operation that actually needs acquire semantics. 39class acquiring_load<PatFrags base> 40 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 41 let IsAtomic = 1; 42 let IsAtomicOrderingAcquire = 1; 43} 44 45// An atomic load operation that needs sequential consistency. 46class seq_cst_load<PatFrags base> 47 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 48 let IsAtomic = 1; 49 let IsAtomicOrderingSequentiallyConsistent = 1; 50} 51 52// RCPC extension, currently opt-in under a separate feature. 53let Predicates = [HasLDAPR] in { 54 // v8.3 Release Consistent Processor Consistent support, optional in v8.2. 55 // 8-bit loads 56 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDAPRB GPR64sp:$ptr)>; 57 // 16-bit loads 58 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDAPRH GPR64sp:$ptr)>; 59 // 32-bit loads 60 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>; 61 // 64-bit loads 62 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>; 63} 64 65// 8-bit loads 66def : Pat<(seq_cst_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 67def : Pat<(acquiring_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 68def : Pat<(relaxed_load<atomic_load_az_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, 69 ro_Wextend8:$offset)), 70 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>; 71def : Pat<(relaxed_load<atomic_load_az_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, 72 ro_Xextend8:$offset)), 73 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>; 74def : Pat<(relaxed_load<atomic_load_az_8> (am_indexed8 GPR64sp:$Rn, 75 uimm12s1:$offset)), 76 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>; 77def : Pat<(relaxed_load<atomic_load_az_8> 78 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 79 (LDURBBi GPR64sp:$Rn, simm9:$offset)>; 80 81// 16-bit loads 82def : Pat<(seq_cst_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 83def : Pat<(acquiring_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 84def : Pat<(relaxed_load<atomic_load_az_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 85 ro_Wextend16:$extend)), 86 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 87def : Pat<(relaxed_load<atomic_load_az_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 88 ro_Xextend16:$extend)), 89 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 90def : Pat<(relaxed_load<atomic_load_az_16> (am_indexed16 GPR64sp:$Rn, 91 uimm12s2:$offset)), 92 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>; 93def : Pat<(relaxed_load<atomic_load_az_16> 94 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 95 (LDURHHi GPR64sp:$Rn, simm9:$offset)>; 96 97// 32-bit loads 98def : Pat<(seq_cst_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 99def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 100def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 101 ro_Wextend32:$extend)), 102 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 103def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 104 ro_Xextend32:$extend)), 105 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 106def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 107 uimm12s4:$offset)), 108 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; 109def : Pat<(relaxed_load<atomic_load_32> 110 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 111 (LDURWi GPR64sp:$Rn, simm9:$offset)>; 112 113// 64-bit loads 114def : Pat<(seq_cst_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 115def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 116def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 117 ro_Wextend64:$extend)), 118 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 119def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 120 ro_Xextend64:$extend)), 121 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 122def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 123 uimm12s8:$offset)), 124 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; 125def : Pat<(relaxed_load<atomic_load_64> 126 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 127 (LDURXi GPR64sp:$Rn, simm9:$offset)>; 128 129// FP 32-bit loads 130def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 131 ro_Wextend32:$extend))))), 132 (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 133def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 134 ro_Xextend32:$extend))))), 135 (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 136def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 137 uimm12s8:$offset))))), 138 (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>; 139def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> 140 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))), 141 (LDURSi GPR64sp:$Rn, simm9:$offset)>; 142 143// FP 64-bit loads 144def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 145 ro_Wextend64:$extend))))), 146 (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 147def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 148 ro_Xextend64:$extend))))), 149 (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 150def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 151 uimm12s8:$offset))))), 152 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>; 153def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> 154 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))), 155 (LDURDi GPR64sp:$Rn, simm9:$offset)>; 156 157//===---------------------------------- 158// Atomic stores 159//===---------------------------------- 160 161// When they're actually atomic, only one addressing mode (GPR64sp) is 162// supported, but when they're relaxed and anything can be used, all the 163// standard modes would be valid and may give efficiency gains. 164 165// A store operation that actually needs release semantics. 166class releasing_store<PatFrag base> 167 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 168 let IsAtomic = 1; 169 let IsAtomicOrderingReleaseOrStronger = 1; 170} 171 172// An atomic store operation that doesn't actually need to be atomic on AArch64. 173class relaxed_store<PatFrag base> 174 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 175 let IsAtomic = 1; 176 let IsAtomicOrderingReleaseOrStronger = 0; 177} 178 179// 8-bit stores 180def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val), 181 (STLRB GPR32:$val, GPR64sp:$ptr)>; 182def : Pat<(relaxed_store<atomic_store_8> 183 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend), 184 GPR32:$val), 185 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>; 186def : Pat<(relaxed_store<atomic_store_8> 187 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend), 188 GPR32:$val), 189 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>; 190def : Pat<(relaxed_store<atomic_store_8> 191 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val), 192 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>; 193def : Pat<(relaxed_store<atomic_store_8> 194 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 195 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 196 197// 16-bit stores 198def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val), 199 (STLRH GPR32:$val, GPR64sp:$ptr)>; 200def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 201 ro_Wextend16:$extend), 202 GPR32:$val), 203 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 204def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 205 ro_Xextend16:$extend), 206 GPR32:$val), 207 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 208def : Pat<(relaxed_store<atomic_store_16> 209 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val), 210 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>; 211def : Pat<(relaxed_store<atomic_store_16> 212 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 213 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 214 215// 32-bit stores 216def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val), 217 (STLRW GPR32:$val, GPR64sp:$ptr)>; 218def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 219 ro_Wextend32:$extend), 220 GPR32:$val), 221 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 222def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 223 ro_Xextend32:$extend), 224 GPR32:$val), 225 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 226def : Pat<(relaxed_store<atomic_store_32> 227 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val), 228 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 229def : Pat<(relaxed_store<atomic_store_32> 230 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 231 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 232 233// 64-bit stores 234def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val), 235 (STLRX GPR64:$val, GPR64sp:$ptr)>; 236def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 237 ro_Wextend16:$extend), 238 GPR64:$val), 239 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 240def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 241 ro_Xextend16:$extend), 242 GPR64:$val), 243 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 244def : Pat<(relaxed_store<atomic_store_64> 245 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val), 246 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>; 247def : Pat<(relaxed_store<atomic_store_64> 248 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 249 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 250 251// FP 32-bit stores 252def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 253 ro_Wextend32:$extend), 254 (i32 (bitconvert (f32 FPR32Op:$val)))), 255 (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 256def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 257 ro_Xextend32:$extend), 258 (i32 (bitconvert (f32 FPR32Op:$val)))), 259 (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 260def : Pat<(relaxed_store<atomic_store_32> 261 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 262 (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 263def : Pat<(relaxed_store<atomic_store_32> 264 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 265 (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>; 266 267// FP 64-bit stores 268def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 269 ro_Wextend64:$extend), 270 (i64 (bitconvert (f64 FPR64Op:$val)))), 271 (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 272def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 273 ro_Xextend64:$extend), 274 (i64 (bitconvert (f64 FPR64Op:$val)))), 275 (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 276def : Pat<(relaxed_store<atomic_store_64> 277 (am_indexed64 GPR64sp:$Rn, uimm12s4:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 278 (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 279def : Pat<(relaxed_store<atomic_store_64> 280 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 281 (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; 282 283//===---------------------------------- 284// Low-level exclusive operations 285//===---------------------------------- 286 287// Load-exclusives. 288 289def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 290 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 291}]> { 292 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 293} 294 295def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 296 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 297}]> { 298 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 299} 300 301def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 302 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 303}]> { 304 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 305} 306 307def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 308 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 309}]> { 310 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 311} 312 313def : Pat<(ldxr_1 GPR64sp:$addr), 314 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 315def : Pat<(ldxr_2 GPR64sp:$addr), 316 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 317def : Pat<(ldxr_4 GPR64sp:$addr), 318 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 319def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>; 320 321def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff), 322 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 323def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff), 324 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 325def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff), 326 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 327 328// Load-exclusives. 329 330def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 331 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 332}]> { 333 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 334} 335 336def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 337 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 338}]> { 339 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 340} 341 342def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 343 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 344}]> { 345 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 346} 347 348def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 349 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 350}]> { 351 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 352} 353 354def : Pat<(ldaxr_1 GPR64sp:$addr), 355 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 356def : Pat<(ldaxr_2 GPR64sp:$addr), 357 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 358def : Pat<(ldaxr_4 GPR64sp:$addr), 359 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 360def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>; 361 362def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff), 363 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 364def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff), 365 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 366def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff), 367 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 368 369// Store-exclusives. 370 371def stxr_1 : PatFrag<(ops node:$val, node:$ptr), 372 (int_aarch64_stxr node:$val, node:$ptr), [{ 373 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 374}]> { 375 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 376} 377 378def stxr_2 : PatFrag<(ops node:$val, node:$ptr), 379 (int_aarch64_stxr node:$val, node:$ptr), [{ 380 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 381}]> { 382 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 383} 384 385def stxr_4 : PatFrag<(ops node:$val, node:$ptr), 386 (int_aarch64_stxr node:$val, node:$ptr), [{ 387 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 388}]> { 389 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 390} 391 392def stxr_8 : PatFrag<(ops node:$val, node:$ptr), 393 (int_aarch64_stxr node:$val, node:$ptr), [{ 394 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 395}]> { 396 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 397} 398 399 400def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr), 401 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 402def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr), 403 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 404def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr), 405 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 406def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr), 407 (STXRX GPR64:$val, GPR64sp:$addr)>; 408 409def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 410 (STXRB GPR32:$val, GPR64sp:$addr)>; 411def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 412 (STXRH GPR32:$val, GPR64sp:$addr)>; 413def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr), 414 (STXRW GPR32:$val, GPR64sp:$addr)>; 415 416def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 417 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 418def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 419 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 420def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 421 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 422 423// Store-release-exclusives. 424 425def stlxr_1 : PatFrag<(ops node:$val, node:$ptr), 426 (int_aarch64_stlxr node:$val, node:$ptr), [{ 427 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 428}]> { 429 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 430} 431 432def stlxr_2 : PatFrag<(ops node:$val, node:$ptr), 433 (int_aarch64_stlxr node:$val, node:$ptr), [{ 434 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 435}]> { 436 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 437} 438 439def stlxr_4 : PatFrag<(ops node:$val, node:$ptr), 440 (int_aarch64_stlxr node:$val, node:$ptr), [{ 441 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 442}]> { 443 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 444} 445 446def stlxr_8 : PatFrag<(ops node:$val, node:$ptr), 447 (int_aarch64_stlxr node:$val, node:$ptr), [{ 448 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 449}]> { 450 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 451} 452 453 454def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr), 455 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 456def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr), 457 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 458def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr), 459 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 460def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr), 461 (STLXRX GPR64:$val, GPR64sp:$addr)>; 462 463def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 464 (STLXRB GPR32:$val, GPR64sp:$addr)>; 465def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 466 (STLXRH GPR32:$val, GPR64sp:$addr)>; 467def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr), 468 (STLXRW GPR32:$val, GPR64sp:$addr)>; 469 470def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 471 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 472def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 473 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 474def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 475 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 476 477 478// And clear exclusive. 479 480def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; 481 482//===---------------------------------- 483// Atomic cmpxchg for -O0 484//===---------------------------------- 485 486// The fast register allocator used during -O0 inserts spills to cover any VRegs 487// live across basic block boundaries. When this happens between an LDXR and an 488// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to 489// fail. 490 491// Unfortunately, this means we have to have an alternative (expanded 492// post-regalloc) path for -O0 compilations. Fortunately this path can be 493// significantly more naive than the standard expansion: we conservatively 494// assume seq_cst, strong cmpxchg and omit clrex on failure. 495 496let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch", 497 mayLoad = 1, mayStore = 1 in { 498def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 499 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 500 Sched<[WriteAtomic]>; 501 502def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 503 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 504 Sched<[WriteAtomic]>; 505 506def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 507 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 508 Sched<[WriteAtomic]>; 509 510def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch), 511 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>, 512 Sched<[WriteAtomic]>; 513} 514 515let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch", 516 mayLoad = 1, mayStore = 1 in { 517class cmp_swap_128 : Pseudo<(outs GPR64common:$RdLo, GPR64common:$RdHi, 518 GPR32common:$scratch), 519 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi, 520 GPR64:$newLo, GPR64:$newHi), []>, 521 Sched<[WriteAtomic]>; 522def CMP_SWAP_128 : cmp_swap_128; 523def CMP_SWAP_128_RELEASE : cmp_swap_128; 524def CMP_SWAP_128_ACQUIRE : cmp_swap_128; 525def CMP_SWAP_128_MONOTONIC : cmp_swap_128; 526} 527 528// v8.1 Atomic instructions: 529let Predicates = [HasLSE] in { 530 defm : LDOPregister_patterns<"LDADD", "atomic_load_add">; 531 defm : LDOPregister_patterns<"LDSET", "atomic_load_or">; 532 defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">; 533 defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">; 534 defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">; 535 defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">; 536 defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">; 537 defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">; 538 defm : LDOPregister_patterns<"SWP", "atomic_swap">; 539 defm : CASregister_patterns<"CAS", "atomic_cmp_swap">; 540 541 // These two patterns are only needed for global isel, selection dag isel 542 // converts atomic load-sub into a sub and atomic load-add, and likewise for 543 // and -> clr. 544 defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">; 545 defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">; 546} 547