1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Atomic operand code-gen constructs. 10// 11//===----------------------------------------------------------------------===// 12 13//===---------------------------------- 14// Atomic fences 15//===---------------------------------- 16let AddedComplexity = 15, Size = 0 in 17def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering), 18 [(atomic_fence timm:$ordering, 0)]>, Sched<[]>; 19def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>; 20def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>; 21 22//===---------------------------------- 23// Atomic loads 24//===---------------------------------- 25 26// When they're actually atomic, only one addressing mode (GPR64sp) is 27// supported, but when they're relaxed and anything can be used, all the 28// standard modes would be valid and may give efficiency gains. 29 30// A atomic load operation that actually needs acquire semantics. 31class acquiring_load<PatFrag base> 32 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 33 let IsAtomic = 1; 34 let IsAtomicOrderingAcquireOrStronger = 1; 35} 36 37// An atomic load operation that does not need either acquire or release 38// semantics. 39class relaxed_load<PatFrag base> 40 : PatFrag<(ops node:$ptr), (base node:$ptr)> { 41 let IsAtomic = 1; 42 let IsAtomicOrderingAcquireOrStronger = 0; 43} 44 45// 8-bit loads 46def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>; 47def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, 48 ro_Wextend8:$offset)), 49 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>; 50def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, 51 ro_Xextend8:$offset)), 52 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>; 53def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn, 54 uimm12s1:$offset)), 55 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>; 56def : Pat<(relaxed_load<atomic_load_8> 57 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)), 58 (LDURBBi GPR64sp:$Rn, simm9:$offset)>; 59 60// 16-bit loads 61def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>; 62def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 63 ro_Wextend16:$extend)), 64 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 65def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 66 ro_Xextend16:$extend)), 67 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 68def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn, 69 uimm12s2:$offset)), 70 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>; 71def : Pat<(relaxed_load<atomic_load_16> 72 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)), 73 (LDURHHi GPR64sp:$Rn, simm9:$offset)>; 74 75// 32-bit loads 76def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>; 77def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 78 ro_Wextend32:$extend)), 79 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 80def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 81 ro_Xextend32:$extend)), 82 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 83def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn, 84 uimm12s4:$offset)), 85 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>; 86def : Pat<(relaxed_load<atomic_load_32> 87 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)), 88 (LDURWi GPR64sp:$Rn, simm9:$offset)>; 89 90// 64-bit loads 91def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>; 92def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 93 ro_Wextend64:$extend)), 94 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 95def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 96 ro_Xextend64:$extend)), 97 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 98def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn, 99 uimm12s8:$offset)), 100 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>; 101def : Pat<(relaxed_load<atomic_load_64> 102 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)), 103 (LDURXi GPR64sp:$Rn, simm9:$offset)>; 104 105//===---------------------------------- 106// Atomic stores 107//===---------------------------------- 108 109// When they're actually atomic, only one addressing mode (GPR64sp) is 110// supported, but when they're relaxed and anything can be used, all the 111// standard modes would be valid and may give efficiency gains. 112 113// A store operation that actually needs release semantics. 114class releasing_store<PatFrag base> 115 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 116 let IsAtomic = 1; 117 let IsAtomicOrderingReleaseOrStronger = 1; 118} 119 120// An atomic store operation that doesn't actually need to be atomic on AArch64. 121class relaxed_store<PatFrag base> 122 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> { 123 let IsAtomic = 1; 124 let IsAtomicOrderingReleaseOrStronger = 0; 125} 126 127// 8-bit stores 128def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val), 129 (STLRB GPR32:$val, GPR64sp:$ptr)>; 130def : Pat<(relaxed_store<atomic_store_8> 131 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend), 132 GPR32:$val), 133 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>; 134def : Pat<(relaxed_store<atomic_store_8> 135 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend), 136 GPR32:$val), 137 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>; 138def : Pat<(relaxed_store<atomic_store_8> 139 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val), 140 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>; 141def : Pat<(relaxed_store<atomic_store_8> 142 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 143 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 144 145// 16-bit stores 146def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val), 147 (STLRH GPR32:$val, GPR64sp:$ptr)>; 148def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm, 149 ro_Wextend16:$extend), 150 GPR32:$val), 151 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>; 152def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm, 153 ro_Xextend16:$extend), 154 GPR32:$val), 155 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>; 156def : Pat<(relaxed_store<atomic_store_16> 157 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val), 158 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>; 159def : Pat<(relaxed_store<atomic_store_16> 160 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 161 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 162 163// 32-bit stores 164def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val), 165 (STLRW GPR32:$val, GPR64sp:$ptr)>; 166def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, 167 ro_Wextend32:$extend), 168 GPR32:$val), 169 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>; 170def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, 171 ro_Xextend32:$extend), 172 GPR32:$val), 173 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>; 174def : Pat<(relaxed_store<atomic_store_32> 175 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val), 176 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>; 177def : Pat<(relaxed_store<atomic_store_32> 178 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val), 179 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>; 180 181// 64-bit stores 182def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val), 183 (STLRX GPR64:$val, GPR64sp:$ptr)>; 184def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, 185 ro_Wextend16:$extend), 186 GPR64:$val), 187 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>; 188def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, 189 ro_Xextend16:$extend), 190 GPR64:$val), 191 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>; 192def : Pat<(relaxed_store<atomic_store_64> 193 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val), 194 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>; 195def : Pat<(relaxed_store<atomic_store_64> 196 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val), 197 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>; 198 199//===---------------------------------- 200// Low-level exclusive operations 201//===---------------------------------- 202 203// Load-exclusives. 204 205def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 206 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 207}]> { 208 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 209} 210 211def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 212 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 213}]> { 214 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 215} 216 217def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 218 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 219}]> { 220 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 221} 222 223def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{ 224 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 225}]> { 226 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 227} 228 229def : Pat<(ldxr_1 GPR64sp:$addr), 230 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 231def : Pat<(ldxr_2 GPR64sp:$addr), 232 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 233def : Pat<(ldxr_4 GPR64sp:$addr), 234 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 235def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>; 236 237def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff), 238 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>; 239def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff), 240 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>; 241def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff), 242 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>; 243 244// Load-exclusives. 245 246def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 247 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 248}]> { 249 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 250} 251 252def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 253 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 254}]> { 255 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 256} 257 258def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 259 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 260}]> { 261 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 262} 263 264def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{ 265 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 266}]> { 267 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 268} 269 270def : Pat<(ldaxr_1 GPR64sp:$addr), 271 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 272def : Pat<(ldaxr_2 GPR64sp:$addr), 273 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 274def : Pat<(ldaxr_4 GPR64sp:$addr), 275 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 276def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>; 277 278def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff), 279 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>; 280def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff), 281 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>; 282def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff), 283 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>; 284 285// Store-exclusives. 286 287def stxr_1 : PatFrag<(ops node:$val, node:$ptr), 288 (int_aarch64_stxr node:$val, node:$ptr), [{ 289 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 290}]> { 291 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 292} 293 294def stxr_2 : PatFrag<(ops node:$val, node:$ptr), 295 (int_aarch64_stxr node:$val, node:$ptr), [{ 296 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 297}]> { 298 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 299} 300 301def stxr_4 : PatFrag<(ops node:$val, node:$ptr), 302 (int_aarch64_stxr node:$val, node:$ptr), [{ 303 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 304}]> { 305 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 306} 307 308def stxr_8 : PatFrag<(ops node:$val, node:$ptr), 309 (int_aarch64_stxr node:$val, node:$ptr), [{ 310 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 311}]> { 312 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 313} 314 315 316def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr), 317 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 318def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr), 319 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 320def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr), 321 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 322def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr), 323 (STXRX GPR64:$val, GPR64sp:$addr)>; 324 325def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 326 (STXRB GPR32:$val, GPR64sp:$addr)>; 327def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 328 (STXRH GPR32:$val, GPR64sp:$addr)>; 329def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr), 330 (STXRW GPR32:$val, GPR64sp:$addr)>; 331 332def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 333 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 334def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 335 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 336def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 337 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 338 339// Store-release-exclusives. 340 341def stlxr_1 : PatFrag<(ops node:$val, node:$ptr), 342 (int_aarch64_stlxr node:$val, node:$ptr), [{ 343 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8; 344}]> { 345 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }]; 346} 347 348def stlxr_2 : PatFrag<(ops node:$val, node:$ptr), 349 (int_aarch64_stlxr node:$val, node:$ptr), [{ 350 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16; 351}]> { 352 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }]; 353} 354 355def stlxr_4 : PatFrag<(ops node:$val, node:$ptr), 356 (int_aarch64_stlxr node:$val, node:$ptr), [{ 357 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32; 358}]> { 359 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }]; 360} 361 362def stlxr_8 : PatFrag<(ops node:$val, node:$ptr), 363 (int_aarch64_stlxr node:$val, node:$ptr), [{ 364 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64; 365}]> { 366 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }]; 367} 368 369 370def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr), 371 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 372def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr), 373 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 374def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr), 375 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 376def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr), 377 (STLXRX GPR64:$val, GPR64sp:$addr)>; 378 379def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr), 380 (STLXRB GPR32:$val, GPR64sp:$addr)>; 381def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr), 382 (STLXRH GPR32:$val, GPR64sp:$addr)>; 383def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr), 384 (STLXRW GPR32:$val, GPR64sp:$addr)>; 385 386def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr), 387 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 388def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr), 389 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 390def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr), 391 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>; 392 393 394// And clear exclusive. 395 396def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; 397 398//===---------------------------------- 399// Atomic cmpxchg for -O0 400//===---------------------------------- 401 402// The fast register allocator used during -O0 inserts spills to cover any VRegs 403// live across basic block boundaries. When this happens between an LDXR and an 404// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to 405// fail. 406 407// Unfortunately, this means we have to have an alternative (expanded 408// post-regalloc) path for -O0 compilations. Fortunately this path can be 409// significantly more naive than the standard expansion: we conservatively 410// assume seq_cst, strong cmpxchg and omit clrex on failure. 411 412let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch", 413 mayLoad = 1, mayStore = 1 in { 414def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 415 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 416 Sched<[WriteAtomic]>; 417 418def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 419 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 420 Sched<[WriteAtomic]>; 421 422def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), 423 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, 424 Sched<[WriteAtomic]>; 425 426def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch), 427 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>, 428 Sched<[WriteAtomic]>; 429} 430 431let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch", 432 mayLoad = 1, mayStore = 1 in 433def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch), 434 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi, 435 GPR64:$newLo, GPR64:$newHi), []>, 436 Sched<[WriteAtomic]>; 437 438// v8.1 Atomic instructions: 439let Predicates = [HasLSE] in { 440 defm : LDOPregister_patterns<"LDADD", "atomic_load_add">; 441 defm : LDOPregister_patterns<"LDSET", "atomic_load_or">; 442 defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">; 443 defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">; 444 defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">; 445 defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">; 446 defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">; 447 defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">; 448 defm : LDOPregister_patterns<"SWP", "atomic_swap">; 449 defm : CASregister_patterns<"CAS", "atomic_cmp_swap">; 450 451 // These two patterns are only needed for global isel, selection dag isel 452 // converts atomic load-sub into a sub and atomic load-add, and likewise for 453 // and -> clr. 454 defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">; 455 defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">; 456} 457