1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Atomic operand code-gen constructs. 10// 11//===----------------------------------------------------------------------===// 12 13//===---------------------------------- 14// Atomic fences 15//===---------------------------------- 16let AddedComplexity = 15, Size = 0 in 17def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering), 18 [(atomic_fence timm:$ordering, 0)]>, Sched<[]>; 19def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>; 20def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>; 21 22//===---------------------------------- 23// Atomic loads 24//===---------------------------------- 25 26// When they're actually atomic, only one addressing mode (GPR64sp) is 27// supported, but when they're relaxed and anything can be used, all the 28// standard modes would be valid and may give efficiency gains. 29 30// A atomic load operation that actually needs acquire semantics. 31class acquiring_load<PatFrag base> 32 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 33 let IsAtomic = 1; 34 let IsAtomicOrderingAcquireOrStronger = 1; 35} 36 37// An atomic load operation that does not need either acquire or release 38// semantics. 39class relaxed_load<PatFrag base> 40 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 41 let IsAtomic = 1; 42 let IsAtomicOrderingAcquireOrStronger = 0; 43} 44 45// 8-bit loads 46def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 47def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, 48 ro_Wextend8:$offset)), 49 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>; 50def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, 51 ro_Xextend8:$offset)), 52 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>; 53def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn, 54 uimm12s1:$offset)), 55 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>; 56def : Pat<(relaxed_load<atomic_load_8> 57 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 58 (LDURBBi GPR64sp:$Rn, simm9:$offset)>; 59 60// 16-bit loads 61def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 62def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 63 ro_Wextend16:$extend)), 64 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 65def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 66 ro_Xextend16:$extend)), 67 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 68def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn, 69 uimm12s2:$offset)), 70 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>; 71def : Pat<(relaxed_load<atomic_load_16> 72 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 73 (LDURHHi GPR64sp:$Rn, simm9:$offset)>; 74 75// 32-bit loads 76def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 77def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 78 ro_Wextend32:$extend)), 79 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 80def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 81 ro_Xextend32:$extend)), 82 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 83def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 84 uimm12s4:$offset)), 85 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; 86def : Pat<(relaxed_load<atomic_load_32> 87 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 88 (LDURWi GPR64sp:$Rn, simm9:$offset)>; 89 90// 64-bit loads 91def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 92def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 93 ro_Wextend64:$extend)), 94 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 95def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 96 ro_Xextend64:$extend)), 97 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 98def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 99 uimm12s8:$offset)), 100 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; 101def : Pat<(relaxed_load<atomic_load_64> 102 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 103 (LDURXi GPR64sp:$Rn, simm9:$offset)>; 104 105// FP 32-bit loads 106def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 107 ro_Wextend32:$extend))))), 108 (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 109def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 110 ro_Xextend32:$extend))))), 111 (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 112def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 113 uimm12s8:$offset))))), 114 (LDRSui GPR64sp:$Rn, uimm12s8:$offset)>; 115def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_32> 116 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))), 117 (LDURSi GPR64sp:$Rn, simm9:$offset)>; 118 119// FP 64-bit loads 120def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 121 ro_Wextend64:$extend))))), 122 (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 123def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 124 ro_Xextend64:$extend))))), 125 (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 126def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 127 uimm12s8:$offset))))), 128 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>; 129def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64> 130 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))), 131 (LDURDi GPR64sp:$Rn, simm9:$offset)>; 132 133//===---------------------------------- 134// Atomic stores 135//===---------------------------------- 136 137// When they're actually atomic, only one addressing mode (GPR64sp) is 138// supported, but when they're relaxed and anything can be used, all the 139// standard modes would be valid and may give efficiency gains. 140 141// A store operation that actually needs release semantics. 142class releasing_store<PatFrag base> 143 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 144 let IsAtomic = 1; 145 let IsAtomicOrderingReleaseOrStronger = 1; 146} 147 148// An atomic store operation that doesn't actually need to be atomic on AArch64. 149class relaxed_store<PatFrag base> 150 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 151 let IsAtomic = 1; 152 let IsAtomicOrderingReleaseOrStronger = 0; 153} 154 155// 8-bit stores 156def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val), 157 (STLRB GPR32:$val, GPR64sp:$ptr)>; 158def : Pat<(relaxed_store<atomic_store_8> 159 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend), 160 GPR32:$val), 161 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>; 162def : Pat<(relaxed_store<atomic_store_8> 163 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend), 164 GPR32:$val), 165 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>; 166def : Pat<(relaxed_store<atomic_store_8> 167 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val), 168 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>; 169def : Pat<(relaxed_store<atomic_store_8> 170 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 171 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 172 173// 16-bit stores 174def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val), 175 (STLRH GPR32:$val, GPR64sp:$ptr)>; 176def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 177 ro_Wextend16:$extend), 178 GPR32:$val), 179 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 180def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 181 ro_Xextend16:$extend), 182 GPR32:$val), 183 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 184def : Pat<(relaxed_store<atomic_store_16> 185 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val), 186 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>; 187def : Pat<(relaxed_store<atomic_store_16> 188 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 189 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 190 191// 32-bit stores 192def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val), 193 (STLRW GPR32:$val, GPR64sp:$ptr)>; 194def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 195 ro_Wextend32:$extend), 196 GPR32:$val), 197 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 198def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 199 ro_Xextend32:$extend), 200 GPR32:$val), 201 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 202def : Pat<(relaxed_store<atomic_store_32> 203 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val), 204 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 205def : Pat<(relaxed_store<atomic_store_32> 206 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 207 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 208 209// 64-bit stores 210def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val), 211 (STLRX GPR64:$val, GPR64sp:$ptr)>; 212def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 213 ro_Wextend16:$extend), 214 GPR64:$val), 215 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 216def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 217 ro_Xextend16:$extend), 218 GPR64:$val), 219 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 220def : Pat<(relaxed_store<atomic_store_64> 221 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val), 222 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>; 223def : Pat<(relaxed_store<atomic_store_64> 224 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 225 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 226 227// FP 32-bit stores 228def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 229 ro_Wextend32:$extend), 230 (i32 (bitconvert (f32 FPR32Op:$val)))), 231 (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 232def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 233 ro_Xextend32:$extend), 234 (i32 (bitconvert (f32 FPR32Op:$val)))), 235 (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 236def : Pat<(relaxed_store<atomic_store_32> 237 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 238 (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 239def : Pat<(relaxed_store<atomic_store_32> 240 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))), 241 (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>; 242 243// FP 64-bit stores 244def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 245 ro_Wextend64:$extend), 246 (i64 (bitconvert (f64 FPR64Op:$val)))), 247 (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 248def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 249 ro_Xextend64:$extend), 250 (i64 (bitconvert (f64 FPR64Op:$val)))), 251 (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 252def : Pat<(relaxed_store<atomic_store_64> 253 (am_indexed64 GPR64sp:$Rn, uimm12s4:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 254 (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 255def : Pat<(relaxed_store<atomic_store_64> 256 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))), 257 (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>; 258 259//===---------------------------------- 260// Low-level exclusive operations 261//===---------------------------------- 262 263// Load-exclusives. 264 265def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 266 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 267}]> { 268 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 269} 270 271def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 272 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 273}]> { 274 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 275} 276 277def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 278 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 279}]> { 280 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 281} 282 283def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 284 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 285}]> { 286 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 287} 288 289def : Pat<(ldxr_1 GPR64sp:$addr), 290 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 291def : Pat<(ldxr_2 GPR64sp:$addr), 292 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 293def : Pat<(ldxr_4 GPR64sp:$addr), 294 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 295def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>; 296 297def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff), 298 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 299def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff), 300 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 301def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff), 302 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 303 304// Load-exclusives. 305 306def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 307 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 308}]> { 309 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 310} 311 312def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 313 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 314}]> { 315 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 316} 317 318def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 319 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 320}]> { 321 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 322} 323 324def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 325 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 326}]> { 327 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 328} 329 330def : Pat<(ldaxr_1 GPR64sp:$addr), 331 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 332def : Pat<(ldaxr_2 GPR64sp:$addr), 333 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 334def : Pat<(ldaxr_4 GPR64sp:$addr), 335 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 336def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>; 337 338def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff), 339 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 340def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff), 341 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 342def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff), 343 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 344 345// Store-exclusives. 346 347def stxr_1 : PatFrag<(ops node:$val, node:$ptr), 348 (int_aarch64_stxr node:$val, node:$ptr), [{ 349 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 350}]> { 351 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 352} 353 354def stxr_2 : PatFrag<(ops node:$val, node:$ptr), 355 (int_aarch64_stxr node:$val, node:$ptr), [{ 356 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 357}]> { 358 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 359} 360 361def stxr_4 : PatFrag<(ops node:$val, node:$ptr), 362 (int_aarch64_stxr node:$val, node:$ptr), [{ 363 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 364}]> { 365 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 366} 367 368def stxr_8 : PatFrag<(ops node:$val, node:$ptr), 369 (int_aarch64_stxr node:$val, node:$ptr), [{ 370 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 371}]> { 372 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 373} 374 375 376def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr), 377 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 378def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr), 379 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 380def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr), 381 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 382def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr), 383 (STXRX GPR64:$val, GPR64sp:$addr)>; 384 385def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 386 (STXRB GPR32:$val, GPR64sp:$addr)>; 387def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 388 (STXRH GPR32:$val, GPR64sp:$addr)>; 389def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr), 390 (STXRW GPR32:$val, GPR64sp:$addr)>; 391 392def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 393 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 394def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 395 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 396def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 397 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 398 399// Store-release-exclusives. 400 401def stlxr_1 : PatFrag<(ops node:$val, node:$ptr), 402 (int_aarch64_stlxr node:$val, node:$ptr), [{ 403 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 404}]> { 405 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 406} 407 408def stlxr_2 : PatFrag<(ops node:$val, node:$ptr), 409 (int_aarch64_stlxr node:$val, node:$ptr), [{ 410 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 411}]> { 412 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 413} 414 415def stlxr_4 : PatFrag<(ops node:$val, node:$ptr), 416 (int_aarch64_stlxr node:$val, node:$ptr), [{ 417 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 418}]> { 419 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 420} 421 422def stlxr_8 : PatFrag<(ops node:$val, node:$ptr), 423 (int_aarch64_stlxr node:$val, node:$ptr), [{ 424 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 425}]> { 426 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 427} 428 429 430def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr), 431 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 432def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr), 433 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 434def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr), 435 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 436def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr), 437 (STLXRX GPR64:$val, GPR64sp:$addr)>; 438 439def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 440 (STLXRB GPR32:$val, GPR64sp:$addr)>; 441def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 442 (STLXRH GPR32:$val, GPR64sp:$addr)>; 443def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr), 444 (STLXRW GPR32:$val, GPR64sp:$addr)>; 445 446def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 447 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 448def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 449 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 450def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 451 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 452 453 454// And clear exclusive. 455 456def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; 457 458//===---------------------------------- 459// Atomic cmpxchg for -O0 460//===---------------------------------- 461 462// The fast register allocator used during -O0 inserts spills to cover any VRegs 463// live across basic block boundaries. When this happens between an LDXR and an 464// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to 465// fail. 466 467// Unfortunately, this means we have to have an alternative (expanded 468// post-regalloc) path for -O0 compilations. Fortunately this path can be 469// significantly more naive than the standard expansion: we conservatively 470// assume seq_cst, strong cmpxchg and omit clrex on failure. 471 472let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch", 473 mayLoad = 1, mayStore = 1 in { 474def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 475 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 476 Sched<[WriteAtomic]>; 477 478def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 479 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 480 Sched<[WriteAtomic]>; 481 482def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 483 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 484 Sched<[WriteAtomic]>; 485 486def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch), 487 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>, 488 Sched<[WriteAtomic]>; 489} 490 491let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch", 492 mayLoad = 1, mayStore = 1 in { 493class cmp_swap_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32common:$scratch), 494 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi, 495 GPR64:$newLo, GPR64:$newHi), []>, 496 Sched<[WriteAtomic]>; 497def CMP_SWAP_128 : cmp_swap_128; 498def CMP_SWAP_128_RELEASE : cmp_swap_128; 499def CMP_SWAP_128_ACQUIRE : cmp_swap_128; 500def CMP_SWAP_128_MONOTONIC : cmp_swap_128; 501} 502 503// v8.1 Atomic instructions: 504let Predicates = [HasLSE] in { 505 defm : LDOPregister_patterns<"LDADD", "atomic_load_add">; 506 defm : LDOPregister_patterns<"LDSET", "atomic_load_or">; 507 defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">; 508 defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">; 509 defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">; 510 defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">; 511 defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">; 512 defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">; 513 defm : LDOPregister_patterns<"SWP", "atomic_swap">; 514 defm : CASregister_patterns<"CAS", "atomic_cmp_swap">; 515 516 // These two patterns are only needed for global isel, selection dag isel 517 // converts atomic load-sub into a sub and atomic load-add, and likewise for 518 // and -> clr. 519 defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">; 520 defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">; 521} 522