1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Atomic operand code-gen constructs. 10// 11//===----------------------------------------------------------------------===// 12 13//===---------------------------------- 14// Atomic fences 15//===---------------------------------- 16let AddedComplexity = 15 in 17def : Pat<(atomic_fence (timm), 0), (MEMBARRIER)>; 18def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>; 19def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>; 20 21//===---------------------------------- 22// Atomic loads 23//===---------------------------------- 24 25// When they're actually atomic, only one addressing mode (GPR64sp) is 26// supported, but when they're relaxed and anything can be used, all the 27// standard modes would be valid and may give efficiency gains. 28 29// An atomic load operation that does not need either acquire or release 30// semantics. 31class relaxed_load<PatFrags base> 32 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 33 let IsAtomic = 1; 34 let IsAtomicOrderingAcquireOrStronger = 0; 35} 36 37// A atomic load operation that actually needs acquire semantics. 38class acquiring_load<PatFrags base> 39 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 40 let IsAtomic = 1; 41 let IsAtomicOrderingAcquire = 1; 42} 43 44// An atomic load operation that needs sequential consistency. 45class seq_cst_load<PatFrags base> 46 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 47 let IsAtomic = 1; 48 let IsAtomicOrderingSequentiallyConsistent = 1; 49} 50 51let Predicates = [HasRCPC] in { 52 // v8.3 Release Consistent Processor Consistent support, optional in v8.2. 53 // 8-bit loads 54 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDAPRB GPR64sp:$ptr)>; 55 // 16-bit loads 56 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDAPRH GPR64sp:$ptr)>; 57 // 32-bit loads 58 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>; 59 // 64-bit loads 60 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>; 61} 62 63// 8-bit loads 64def : Pat<(seq_cst_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 65def : Pat<(acquiring_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 66def : Pat<(relaxed_load<atomic_load_az_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, 67 ro_Wextend8:$offset)), 68 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>; 69def : Pat<(relaxed_load<atomic_load_az_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, 70 ro_Xextend8:$offset)), 71 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>; 72def : Pat<(relaxed_load<atomic_load_az_8> (am_indexed8 GPR64sp:$Rn, 73 uimm12s1:$offset)), 74 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>; 75def : Pat<(relaxed_load<atomic_load_az_8> 76 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 77 (LDURBBi GPR64sp:$Rn, simm9:$offset)>; 78 79// 16-bit loads 80def : Pat<(seq_cst_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 81def : Pat<(acquiring_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 82def : Pat<(relaxed_load<atomic_load_az_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 83 ro_Wextend16:$extend)), 84 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 85def : Pat<(relaxed_load<atomic_load_az_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 86 ro_Xextend16:$extend)), 87 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 88def : Pat<(relaxed_load<atomic_load_az_16> (am_indexed16 GPR64sp:$Rn, 89 uimm12s2:$offset)), 90 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>; 91def : Pat<(relaxed_load<atomic_load_az_16> 92 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 93 (LDURHHi GPR64sp:$Rn, simm9:$offset)>; 94 95// 32-bit loads 96def : Pat<(seq_cst_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 97def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 98def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 99 ro_Wextend32:$extend)), 100 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 101def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 102 ro_Xextend32:$extend)), 103 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 104def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 105 uimm12s4:$offset)), 106 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; 107def : Pat<(relaxed_load<atomic_load_32> 108 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 109 (LDURWi GPR64sp:$Rn, simm9:$offset)>; 110 111// 64-bit loads 112def : Pat<(seq_cst_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 113def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 114def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 115 ro_Wextend64:$extend)), 116 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 117def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 118 ro_Xextend64:$extend)), 119 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 120def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 121 uimm12s8:$offset)), 122 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; 123def : Pat<(relaxed_load<atomic_load_64> 124 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 125 (LDURXi GPR64sp:$Rn, simm9:$offset)>; 126 127// FP 32-bit loads 128def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 129 ro_Wextend32:$extend))))), 130 (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 131def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 132 ro_Xextend32:$extend))))), 133 (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 134def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 135 uimm12s8:$offset))))), 136 (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>; 137def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> 138 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))), 139 (LDURSi GPR64sp:$Rn, simm9:$offset)>; 140 141// FP 64-bit loads 142def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 143 ro_Wextend64:$extend))))), 144 (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 145def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 146 ro_Xextend64:$extend))))), 147 (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 148def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 149 uimm12s8:$offset))))), 150 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>; 151def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> 152 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))), 153 (LDURDi GPR64sp:$Rn, simm9:$offset)>; 154 155//===---------------------------------- 156// Atomic stores 157//===---------------------------------- 158 159// When they're actually atomic, only one addressing mode (GPR64sp) is 160// supported, but when they're relaxed and anything can be used, all the 161// standard modes would be valid and may give efficiency gains. 162 163// A store operation that actually needs release semantics. 164class releasing_store<PatFrag base> 165 : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { 166 let IsAtomic = 1; 167 let IsAtomicOrderingReleaseOrStronger = 1; 168} 169 170// An atomic store operation that doesn't actually need to be atomic on AArch64. 171class relaxed_store<PatFrag base> 172 : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> { 173 let IsAtomic = 1; 174 let IsAtomicOrderingReleaseOrStronger = 0; 175} 176 177// 8-bit stores 178def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val), 179 (STLRB GPR32:$val, GPR64sp:$ptr)>; 180def : Pat<(relaxed_store<atomic_store_8> 181 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend), 182 GPR32:$val), 183 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>; 184def : Pat<(relaxed_store<atomic_store_8> 185 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend), 186 GPR32:$val), 187 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>; 188def : Pat<(relaxed_store<atomic_store_8> 189 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val), 190 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>; 191def : Pat<(relaxed_store<atomic_store_8> 192 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 193 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 194 195// 16-bit stores 196def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val), 197 (STLRH GPR32:$val, GPR64sp:$ptr)>; 198def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 199 ro_Wextend16:$extend), 200 GPR32:$val), 201 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 202def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 203 ro_Xextend16:$extend), 204 GPR32:$val), 205 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 206def : Pat<(relaxed_store<atomic_store_16> 207 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val), 208 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>; 209def : Pat<(relaxed_store<atomic_store_16> 210 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 211 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 212 213// 32-bit stores 214def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val), 215 (STLRW GPR32:$val, GPR64sp:$ptr)>; 216def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 217 ro_Wextend32:$extend), 218 GPR32:$val), 219 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 220def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 221 ro_Xextend32:$extend), 222 GPR32:$val), 223 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 224def : Pat<(relaxed_store<atomic_store_32> 225 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val), 226 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 227def : Pat<(relaxed_store<atomic_store_32> 228 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 229 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 230 231// 64-bit stores 232def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val), 233 (STLRX GPR64:$val, GPR64sp:$ptr)>; 234def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 235 ro_Wextend16:$extend), 236 GPR64:$val), 237 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 238def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 239 ro_Xextend16:$extend), 240 GPR64:$val), 241 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 242def : Pat<(relaxed_store<atomic_store_64> 243 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val), 244 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>; 245def : Pat<(relaxed_store<atomic_store_64> 246 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 247 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 248 249// FP 32-bit stores 250def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 251 ro_Wextend32:$extend), 252 (i32 (bitconvert (f32 FPR32Op:$val)))), 253 (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 254def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 255 ro_Xextend32:$extend), 256 (i32 (bitconvert (f32 FPR32Op:$val)))), 257 (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 258def : Pat<(relaxed_store<atomic_store_32> 259 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 260 (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 261def : Pat<(relaxed_store<atomic_store_32> 262 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 263 (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>; 264 265// FP 64-bit stores 266def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 267 ro_Wextend64:$extend), 268 (i64 (bitconvert (f64 FPR64Op:$val)))), 269 (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 270def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 271 ro_Xextend64:$extend), 272 (i64 (bitconvert (f64 FPR64Op:$val)))), 273 (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 274def : Pat<(relaxed_store<atomic_store_64> 275 (am_indexed64 GPR64sp:$Rn, uimm12s4:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 276 (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 277def : Pat<(relaxed_store<atomic_store_64> 278 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 279 (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; 280 281//===---------------------------------- 282// Low-level exclusive operations 283//===---------------------------------- 284 285// Load-exclusives. 286 287def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 288 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 289}]> { 290 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 291} 292 293def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 294 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 295}]> { 296 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 297} 298 299def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 300 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 301}]> { 302 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 303} 304 305def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 306 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 307}]> { 308 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 309} 310 311def : Pat<(ldxr_1 GPR64sp:$addr), 312 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 313def : Pat<(ldxr_2 GPR64sp:$addr), 314 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 315def : Pat<(ldxr_4 GPR64sp:$addr), 316 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 317def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>; 318 319def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff), 320 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 321def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff), 322 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 323def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff), 324 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 325 326// Load-exclusives. 327 328def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 329 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 330}]> { 331 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 332} 333 334def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 335 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 336}]> { 337 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 338} 339 340def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 341 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 342}]> { 343 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 344} 345 346def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 347 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 348}]> { 349 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 350} 351 352def : Pat<(ldaxr_1 GPR64sp:$addr), 353 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 354def : Pat<(ldaxr_2 GPR64sp:$addr), 355 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 356def : Pat<(ldaxr_4 GPR64sp:$addr), 357 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 358def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>; 359 360def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff), 361 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 362def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff), 363 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 364def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff), 365 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 366 367// Store-exclusives. 368 369def stxr_1 : PatFrag<(ops node:$val, node:$ptr), 370 (int_aarch64_stxr node:$val, node:$ptr), [{ 371 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 372}]> { 373 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 374} 375 376def stxr_2 : PatFrag<(ops node:$val, node:$ptr), 377 (int_aarch64_stxr node:$val, node:$ptr), [{ 378 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 379}]> { 380 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 381} 382 383def stxr_4 : PatFrag<(ops node:$val, node:$ptr), 384 (int_aarch64_stxr node:$val, node:$ptr), [{ 385 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 386}]> { 387 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 388} 389 390def stxr_8 : PatFrag<(ops node:$val, node:$ptr), 391 (int_aarch64_stxr node:$val, node:$ptr), [{ 392 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 393}]> { 394 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 395} 396 397 398def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr), 399 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 400def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr), 401 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 402def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr), 403 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 404def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr), 405 (STXRX GPR64:$val, GPR64sp:$addr)>; 406 407def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 408 (STXRB GPR32:$val, GPR64sp:$addr)>; 409def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 410 (STXRH GPR32:$val, GPR64sp:$addr)>; 411def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr), 412 (STXRW GPR32:$val, GPR64sp:$addr)>; 413 414def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 415 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 416def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 417 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 418def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 419 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 420 421// Store-release-exclusives. 422 423def stlxr_1 : PatFrag<(ops node:$val, node:$ptr), 424 (int_aarch64_stlxr node:$val, node:$ptr), [{ 425 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 426}]> { 427 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 428} 429 430def stlxr_2 : PatFrag<(ops node:$val, node:$ptr), 431 (int_aarch64_stlxr node:$val, node:$ptr), [{ 432 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 433}]> { 434 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 435} 436 437def stlxr_4 : PatFrag<(ops node:$val, node:$ptr), 438 (int_aarch64_stlxr node:$val, node:$ptr), [{ 439 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 440}]> { 441 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 442} 443 444def stlxr_8 : PatFrag<(ops node:$val, node:$ptr), 445 (int_aarch64_stlxr node:$val, node:$ptr), [{ 446 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 447}]> { 448 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 449} 450 451 452def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr), 453 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 454def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr), 455 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 456def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr), 457 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 458def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr), 459 (STLXRX GPR64:$val, GPR64sp:$addr)>; 460 461def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 462 (STLXRB GPR32:$val, GPR64sp:$addr)>; 463def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 464 (STLXRH GPR32:$val, GPR64sp:$addr)>; 465def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr), 466 (STLXRW GPR32:$val, GPR64sp:$addr)>; 467 468def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 469 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 470def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 471 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 472def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 473 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 474 475 476// And clear exclusive. 477 478def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; 479 480//===---------------------------------- 481// Atomic cmpxchg for -O0 482//===---------------------------------- 483 484// The fast register allocator used during -O0 inserts spills to cover any VRegs 485// live across basic block boundaries. When this happens between an LDXR and an 486// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to 487// fail. 488 489// Unfortunately, this means we have to have an alternative (expanded 490// post-regalloc) path for -O0 compilations. Fortunately this path can be 491// significantly more naive than the standard expansion: we conservatively 492// assume seq_cst, strong cmpxchg and omit clrex on failure. 493 494let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch", 495 mayLoad = 1, mayStore = 1 in { 496def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 497 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 498 Sched<[WriteAtomic]>; 499 500def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 501 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 502 Sched<[WriteAtomic]>; 503 504def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 505 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 506 Sched<[WriteAtomic]>; 507 508def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch), 509 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>, 510 Sched<[WriteAtomic]>; 511} 512 513let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch", 514 mayLoad = 1, mayStore = 1 in { 515class cmp_swap_128 : Pseudo<(outs GPR64common:$RdLo, GPR64common:$RdHi, 516 GPR32common:$scratch), 517 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi, 518 GPR64:$newLo, GPR64:$newHi), []>, 519 Sched<[WriteAtomic]>; 520def CMP_SWAP_128 : cmp_swap_128; 521def CMP_SWAP_128_RELEASE : cmp_swap_128; 522def CMP_SWAP_128_ACQUIRE : cmp_swap_128; 523def CMP_SWAP_128_MONOTONIC : cmp_swap_128; 524} 525 526// v8.1 Atomic instructions: 527let Predicates = [HasLSE] in { 528 defm : LDOPregister_patterns<"LDADD", "atomic_load_add">; 529 defm : LDOPregister_patterns<"LDSET", "atomic_load_or">; 530 defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">; 531 defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">; 532 defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">; 533 defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">; 534 defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">; 535 defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">; 536 defm : LDOPregister_patterns<"SWP", "atomic_swap">; 537 defm : CASregister_patterns<"CAS", "atomic_cmp_swap">; 538 539 // These two patterns are only needed for global isel, selection dag isel 540 // converts atomic load-sub into a sub and atomic load-add, and likewise for 541 // and -> clr. 542 defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">; 543 defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">; 544} 545 546// v8.9a/v9.4a FEAT_LRCPC patterns 547let Predicates = [HasRCPC3, HasNEON] in { 548 // LDAP1 loads 549 def : Pat<(vector_insert (v2i64 VecListOne128:$Rd), 550 (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)), VectorIndexD:$idx), 551 (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>; 552 def : Pat<(vector_insert (v2f64 VecListOne128:$Rd), 553 (f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), VectorIndexD:$idx), 554 (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>; 555 def : Pat<(v1i64 (scalar_to_vector 556 (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))), 557 (EXTRACT_SUBREG (LDAP1 (v2i64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>; 558 def : Pat<(v1f64 (scalar_to_vector 559 (f64 (bitconvert (i64 (acquiring_load<atomic_load_64> GPR64sp:$Rn)))))), 560 (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>; 561 562 // STL1 stores 563 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn, 564 (i64 (vector_extract (v2i64 VecListOne128:$Vt), VectorIndexD:$idx))), 565 (STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>; 566 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn, 567 (i64 (bitconvert (f64 (vector_extract (v2f64 VecListOne128:$Vt), VectorIndexD:$idx))))), 568 (STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>; 569 // The v1i64 version of the vldap1_lane_* intrinsic is represented as a 570 // vector_insert -> vector_extract -> atomic store sequence, which is captured 571 // by the patterns above. We only need to cover the v1f64 case manually. 572 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn, 573 (i64 (bitconvert (v1f64 VecListOne64:$Vt)))), 574 (STL1 (SUBREG_TO_REG (i64 0), VecListOne64:$Vt, dsub), (i64 0), GPR64sp:$Rn)>; 575} 576 577// v8.4a FEAT_LRCPC2 patterns 578let Predicates = [HasRCPC_IMMO] in { 579 // Load-Acquire RCpc Register unscaled loads 580 def : Pat<(acquiring_load<atomic_load_az_8> 581 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 582 (LDAPURBi GPR64sp:$Rn, simm9:$offset)>; 583 def : Pat<(acquiring_load<atomic_load_az_16> 584 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 585 (LDAPURHi GPR64sp:$Rn, simm9:$offset)>; 586 def : Pat<(acquiring_load<atomic_load_32> 587 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 588 (LDAPURi GPR64sp:$Rn, simm9:$offset)>; 589 def : Pat<(acquiring_load<atomic_load_64> 590 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 591 (LDAPURXi GPR64sp:$Rn, simm9:$offset)>; 592 593 // Store-Release Register unscaled stores 594 def : Pat<(releasing_store<atomic_store_8> 595 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 596 (STLURBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 597 def : Pat<(releasing_store<atomic_store_16> 598 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 599 (STLURHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 600 def : Pat<(releasing_store<atomic_store_32> 601 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 602 (STLURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 603 def : Pat<(releasing_store<atomic_store_64> 604 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 605 (STLURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 606} 607