1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r = "", 17 string asmstr_s = "", bits<32> atomic_op = -1> { 18 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 19 !or(0xfe00, !and(0xff, atomic_op))>, 20 Requires<[HasAtomics]>; 21} 22 23multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 24 bits<32> atomic_op = -1> { 25 defm "" : NRI<oops, iops, pattern, asmstr, 26 !or(0xfe00, !and(0xff, atomic_op))>, 27 Requires<[HasAtomics]>; 28} 29 30//===----------------------------------------------------------------------===// 31// Atomic wait / notify 32//===----------------------------------------------------------------------===// 33 34let hasSideEffects = 1 in { 35defm ATOMIC_NOTIFY : 36 ATOMIC_I<(outs I32:$dst), 37 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 38 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 39 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 40 "atomic.notify \t${off}${p2align}", 0x00>; 41let mayLoad = 1 in { 42defm ATOMIC_WAIT_I32 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 45 I64:$timeout), 46 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 47 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 48 "i32.atomic.wait \t${off}${p2align}", 0x01>; 49defm ATOMIC_WAIT_I64 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "i64.atomic.wait \t${off}${p2align}", 0x02>; 56} // mayLoad = 1 57} // hasSideEffects = 1 58 59let Predicates = [HasAtomics] in { 60// Select notifys with no constant offset. 61def NotifyPatNoOffset : 62 Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), 63 (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 64 65// Select notifys with a constant offset. 66 67// Pattern with address + immediate offset 68class NotifyPatImmOff<PatFrag operand> : 69 Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), I32:$count)), 70 (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 71def : NotifyPatImmOff<regPlusImm>; 72def : NotifyPatImmOff<or_is_add>; 73 74// Select notifys with just a constant offset. 75def NotifyPatOffsetOnly : 76 Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 77 (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 78 79def NotifyPatGlobalAddrOffOnly : 80 Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 81 I32:$count)), 82 (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 83 84// Select waits with no constant offset. 85class WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 86 Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 87 (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 88def : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 89def : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 90 91// Select waits with a constant offset. 92 93// Pattern with address + immediate offset 94class WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 95 Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 96 (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 97def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 98def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 99def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 100def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 101 102// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 103class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 104 Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 105 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 106def : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 107def : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 108 109class WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 110 Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 111 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 112def : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 113def : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 114} // Predicates = [HasAtomics] 115 116//===----------------------------------------------------------------------===// 117// Atomic fences 118//===----------------------------------------------------------------------===// 119 120// A compiler fence instruction that prevents reordering of instructions. 121let Defs = [ARGUMENTS] in { 122let isPseudo = 1, hasSideEffects = 1 in 123defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 124let hasSideEffects = 1 in 125defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 126 0x03>; 127} // Defs = [ARGUMENTS] 128 129//===----------------------------------------------------------------------===// 130// Atomic loads 131//===----------------------------------------------------------------------===// 132 133multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 134 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 135 Requires<[HasAtomics]>; 136} 137 138defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 139defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 140 141// Select loads with no constant offset. 142let Predicates = [HasAtomics] in { 143def : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 144def : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 145 146// Select loads with a constant offset. 147 148// Pattern with address + immediate offset 149def : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 150def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 151def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 152def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 153 154// Select loads with just a constant offset. 155def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 156def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 157 158def : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 159def : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 160 161} // Predicates = [HasAtomics] 162 163// Extending loads. Note that there are only zero-extending atomic loads, no 164// sign-extending loads. 165defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 166defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 167defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 168defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 169defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 170 171// Fragments for extending loads. These are different from regular loads because 172// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 173// therefore don't have the extension type field. So instead of matching that, 174// we match the patterns that the type legalizer expands them to. 175 176// We directly match zext patterns and select the zext atomic loads. 177// i32 (zext (i8 (atomic_load_8))) gets legalized to 178// i32 (and (i32 (atomic_load_8)), 255) 179// These can be selected to a single zero-extending atomic load instruction. 180def zext_aload_8_32 : 181 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 182def zext_aload_16_32 : 183 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 184// Unlike regular loads, extension to i64 is handled differently than i32. 185// i64 (zext (i8 (atomic_load_8))) gets legalized to 186// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 187def zext_aload_8_64 : 188 PatFrag<(ops node:$addr), 189 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 190def zext_aload_16_64 : 191 PatFrag<(ops node:$addr), 192 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 193def zext_aload_32_64 : 194 PatFrag<(ops node:$addr), 195 (zext (i32 (atomic_load node:$addr)))>; 196 197// We don't have single sext atomic load instructions. So for sext loads, we 198// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 199// results) and select a zext load; the next instruction will be sext_inreg 200// which is selected by itself. 201def sext_aload_8_64 : 202 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 203def sext_aload_16_64 : 204 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 205 206let Predicates = [HasAtomics] in { 207// Select zero-extending loads with no constant offset. 208def : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 209def : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 210def : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 211def : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 212def : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 213 214// Select sign-extending loads with no constant offset 215def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 216def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 217def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 218def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 219// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 220 221// Zero-extending loads with constant offset 222def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 223def : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 224def : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 225def : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 226def : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 227def : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 228def : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 229def : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 230def : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 231def : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 232 233// Sign-extending loads with constant offset 234def : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 235def : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 236def : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 237def : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 238def : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 239def : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 240def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 241def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 242// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 243 244// Extending loads with just a constant offset 245def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 246def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 247def : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 248def : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 249def : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 250def : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 251def : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 252def : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 253def : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 254 255def : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 256def : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 257def : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 258def : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 259def : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 260def : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 261def : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 262def : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 263def : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 264 265} // Predicates = [HasAtomics] 266 267//===----------------------------------------------------------------------===// 268// Atomic stores 269//===----------------------------------------------------------------------===// 270 271multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 272 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 273 Requires<[HasAtomics]>; 274} 275 276defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 277defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 278 279// We need an 'atomic' version of store patterns because store and atomic_store 280// nodes have different operand orders: 281// store: (store $val, $ptr) 282// atomic_store: (store $ptr, $val) 283 284let Predicates = [HasAtomics] in { 285 286// Select stores with no constant offset. 287class AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 288 Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 289def : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 290def : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 291 292// Select stores with a constant offset. 293 294// Pattern with address + immediate offset 295class AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 296 Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 297 (inst 0, imm:$off, I32:$addr, ty:$val)>; 298def : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 299def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 300def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 301def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 302 303// Select stores with just a constant offset. 304class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 305 Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 306def : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 307def : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 308 309class AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 310 Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 311 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 312def : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 313def : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 314 315} // Predicates = [HasAtomics] 316 317// Truncating stores. 318defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 319defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 320defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 321defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 322defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 323 324// Fragments for truncating stores. 325 326// We don't have single truncating atomic store instructions. For 32-bit 327// instructions, we just need to match bare atomic stores. On the other hand, 328// truncating stores from i64 values are once truncated to i32 first. 329class trunc_astore_64<PatFrag kind> : 330 PatFrag<(ops node:$addr, node:$val), 331 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 332def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 333def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 334def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 335 336let Predicates = [HasAtomics] in { 337 338// Truncating stores with no constant offset 339def : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 340def : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 341def : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 342def : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 343def : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 344 345// Truncating stores with a constant offset 346def : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 347def : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 348def : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 349def : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 350def : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 351def : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 352def : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 353def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 354def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 355def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 356 357// Truncating stores with just a constant offset 358def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 359def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 360def : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 361def : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 362def : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 363 364def : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 365def : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 366def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 367def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 368def : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 369 370} // Predicates = [HasAtomics] 371 372//===----------------------------------------------------------------------===// 373// Atomic binary read-modify-writes 374//===----------------------------------------------------------------------===// 375 376multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 377 int atomic_op> { 378 defm "" : 379 ATOMIC_I<(outs rc:$dst), 380 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 381 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 382 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 383 !strconcat(name, "\t${off}${p2align}"), atomic_op>; 384} 385 386defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 387defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 388defm ATOMIC_RMW8_U_ADD_I32 : 389 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 390defm ATOMIC_RMW16_U_ADD_I32 : 391 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 392defm ATOMIC_RMW8_U_ADD_I64 : 393 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 394defm ATOMIC_RMW16_U_ADD_I64 : 395 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 396defm ATOMIC_RMW32_U_ADD_I64 : 397 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 398 399defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 400defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 401defm ATOMIC_RMW8_U_SUB_I32 : 402 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 403defm ATOMIC_RMW16_U_SUB_I32 : 404 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 405defm ATOMIC_RMW8_U_SUB_I64 : 406 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 407defm ATOMIC_RMW16_U_SUB_I64 : 408 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 409defm ATOMIC_RMW32_U_SUB_I64 : 410 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 411 412defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 413defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 414defm ATOMIC_RMW8_U_AND_I32 : 415 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 416defm ATOMIC_RMW16_U_AND_I32 : 417 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 418defm ATOMIC_RMW8_U_AND_I64 : 419 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 420defm ATOMIC_RMW16_U_AND_I64 : 421 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 422defm ATOMIC_RMW32_U_AND_I64 : 423 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 424 425defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 426defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 427defm ATOMIC_RMW8_U_OR_I32 : 428 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 429defm ATOMIC_RMW16_U_OR_I32 : 430 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 431defm ATOMIC_RMW8_U_OR_I64 : 432 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 433defm ATOMIC_RMW16_U_OR_I64 : 434 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 435defm ATOMIC_RMW32_U_OR_I64 : 436 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 437 438defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 439defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 440defm ATOMIC_RMW8_U_XOR_I32 : 441 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 442defm ATOMIC_RMW16_U_XOR_I32 : 443 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 444defm ATOMIC_RMW8_U_XOR_I64 : 445 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 446defm ATOMIC_RMW16_U_XOR_I64 : 447 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 448defm ATOMIC_RMW32_U_XOR_I64 : 449 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 450 451defm ATOMIC_RMW_XCHG_I32 : 452 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 453defm ATOMIC_RMW_XCHG_I64 : 454 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 455defm ATOMIC_RMW8_U_XCHG_I32 : 456 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 457defm ATOMIC_RMW16_U_XCHG_I32 : 458 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 459defm ATOMIC_RMW8_U_XCHG_I64 : 460 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 461defm ATOMIC_RMW16_U_XCHG_I64 : 462 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 463defm ATOMIC_RMW32_U_XCHG_I64 : 464 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 465 466// Select binary RMWs with no constant offset. 467class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 468 Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 469 470// Select binary RMWs with a constant offset. 471 472// Pattern with address + immediate offset 473class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 474 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 475 (inst 0, imm:$off, I32:$addr, ty:$val)>; 476 477// Select binary RMWs with just a constant offset. 478class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 479 Pat<(ty (kind imm:$off, ty:$val)), 480 (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 481 482class BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 483 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 484 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 485 486// Patterns for various addressing modes. 487multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 488 NI inst_64> { 489 def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 490 def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 491 492 def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 493 def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 494 def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 495 def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 496 497 def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 498 def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 499 500 def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 501 def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 502} 503 504let Predicates = [HasAtomics] in { 505defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 506 ATOMIC_RMW_ADD_I64>; 507defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 508 ATOMIC_RMW_SUB_I64>; 509defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 510 ATOMIC_RMW_AND_I64>; 511defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 512 ATOMIC_RMW_OR_I64>; 513defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 514 ATOMIC_RMW_XOR_I64>; 515defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 516 ATOMIC_RMW_XCHG_I64>; 517} // Predicates = [HasAtomics] 518 519// Truncating & zero-extending binary RMW patterns. 520// These are combined patterns of truncating store patterns and zero-extending 521// load patterns above. 522class zext_bin_rmw_8_32<PatFrag kind> : 523 PatFrag<(ops node:$addr, node:$val), 524 (and (i32 (kind node:$addr, node:$val)), 255)>; 525class zext_bin_rmw_16_32<PatFrag kind> : 526 PatFrag<(ops node:$addr, node:$val), 527 (and (i32 (kind node:$addr, node:$val)), 65535)>; 528class zext_bin_rmw_8_64<PatFrag kind> : 529 PatFrag<(ops node:$addr, node:$val), 530 (and (i64 (anyext (i32 (kind node:$addr, 531 (i32 (trunc (i64 node:$val))))))), 255)>; 532class zext_bin_rmw_16_64<PatFrag kind> : 533 PatFrag<(ops node:$addr, node:$val), 534 (and (i64 (anyext (i32 (kind node:$addr, 535 (i32 (trunc (i64 node:$val))))))), 65535)>; 536class zext_bin_rmw_32_64<PatFrag kind> : 537 PatFrag<(ops node:$addr, node:$val), 538 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 539 540// Truncating & sign-extending binary RMW patterns. 541// These are combined patterns of truncating store patterns and sign-extending 542// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 543// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 544// is selected by itself. 545class sext_bin_rmw_8_32<PatFrag kind> : 546 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 547class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 548class sext_bin_rmw_8_64<PatFrag kind> : 549 PatFrag<(ops node:$addr, node:$val), 550 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 551class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 552// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 553 554// Patterns for various addressing modes for truncating-extending binary RMWs. 555multiclass BinRMWTruncExtPattern< 556 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 557 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 558 // Truncating-extending binary RMWs with no constant offset 559 def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 560 def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 561 def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 562 def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 563 def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 564 565 def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 566 def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 567 def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 568 def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 569 570 // Truncating-extending binary RMWs with a constant offset 571 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 572 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 573 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 574 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 575 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 576 def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 577 def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 578 def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 579 def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 580 def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 581 582 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 583 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 584 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 585 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 586 def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 587 def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 588 def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 589 def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 590 591 // Truncating-extending binary RMWs with just a constant offset 592 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 593 def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 594 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 595 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 596 def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 597 598 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 599 def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 600 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 601 def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 602 603 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 604 def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 605 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 606 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 607 def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 608 609 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 610 def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 611 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 612 def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 613} 614 615let Predicates = [HasAtomics] in { 616defm : BinRMWTruncExtPattern< 617 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 618 ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 619 ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 620defm : BinRMWTruncExtPattern< 621 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 622 ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 623 ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 624defm : BinRMWTruncExtPattern< 625 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 626 ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 627 ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 628defm : BinRMWTruncExtPattern< 629 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 630 ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 631 ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 632defm : BinRMWTruncExtPattern< 633 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 634 ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 635 ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 636defm : BinRMWTruncExtPattern< 637 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 638 ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 639 ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 640} // Predicates = [HasAtomics] 641 642//===----------------------------------------------------------------------===// 643// Atomic ternary read-modify-writes 644//===----------------------------------------------------------------------===// 645 646// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 647// flag}. When we use the success flag or both values, we can't make use of i64 648// truncate/extend versions of instructions for now, which is suboptimal. 649// Consider adding a pass after instruction selection that optimizes this case 650// if it is frequent. 651 652multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 653 int atomic_op> { 654 defm "" : 655 ATOMIC_I<(outs rc:$dst), 656 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 657 rc:$new_), 658 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 659 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 660 !strconcat(name, "\t${off}${p2align}"), atomic_op>; 661} 662 663defm ATOMIC_RMW_CMPXCHG_I32 : 664 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 665defm ATOMIC_RMW_CMPXCHG_I64 : 666 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 667defm ATOMIC_RMW8_U_CMPXCHG_I32 : 668 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 669defm ATOMIC_RMW16_U_CMPXCHG_I32 : 670 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 671defm ATOMIC_RMW8_U_CMPXCHG_I64 : 672 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 673defm ATOMIC_RMW16_U_CMPXCHG_I64 : 674 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 675defm ATOMIC_RMW32_U_CMPXCHG_I64 : 676 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 677 678// Select ternary RMWs with no constant offset. 679class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 680 Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 681 (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 682 683// Select ternary RMWs with a constant offset. 684 685// Pattern with address + immediate offset 686class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 687 Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 688 (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 689 690// Select ternary RMWs with just a constant offset. 691class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 692 Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 693 (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 694 695class TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 696 Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 697 (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 698 699// Patterns for various addressing modes. 700multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 701 NI inst_64> { 702 def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 703 def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 704 705 def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 706 def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 707 def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 708 def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 709 710 def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 711 def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 712 713 def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 714 def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 715} 716 717let Predicates = [HasAtomics] in 718defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 719 ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 720 721// Truncating & zero-extending ternary RMW patterns. 722// DAG legalization & optimization before instruction selection may introduce 723// additional nodes such as anyext or assertzext depending on operand types. 724class zext_ter_rmw_8_32<PatFrag kind> : 725 PatFrag<(ops node:$addr, node:$exp, node:$new), 726 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 727class zext_ter_rmw_16_32<PatFrag kind> : 728 PatFrag<(ops node:$addr, node:$exp, node:$new), 729 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 730class zext_ter_rmw_8_64<PatFrag kind> : 731 PatFrag<(ops node:$addr, node:$exp, node:$new), 732 (zext (i32 (assertzext (i32 (kind node:$addr, 733 (i32 (trunc (i64 node:$exp))), 734 (i32 (trunc (i64 node:$new))))))))>; 735class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 736class zext_ter_rmw_32_64<PatFrag kind> : 737 PatFrag<(ops node:$addr, node:$exp, node:$new), 738 (zext (i32 (kind node:$addr, 739 (i32 (trunc (i64 node:$exp))), 740 (i32 (trunc (i64 node:$new))))))>; 741 742// Truncating & sign-extending ternary RMW patterns. 743// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 744// zext RMW; the next instruction will be sext_inreg which is selected by 745// itself. 746class sext_ter_rmw_8_32<PatFrag kind> : 747 PatFrag<(ops node:$addr, node:$exp, node:$new), 748 (kind node:$addr, node:$exp, node:$new)>; 749class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 750class sext_ter_rmw_8_64<PatFrag kind> : 751 PatFrag<(ops node:$addr, node:$exp, node:$new), 752 (anyext (i32 (assertzext (i32 753 (kind node:$addr, 754 (i32 (trunc (i64 node:$exp))), 755 (i32 (trunc (i64 node:$new))))))))>; 756class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 757// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 758 759// Patterns for various addressing modes for truncating-extending ternary RMWs. 760multiclass TerRMWTruncExtPattern< 761 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 762 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 763 // Truncating-extending ternary RMWs with no constant offset 764 def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 765 def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 766 def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 767 def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 768 def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 769 770 def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 771 def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 772 def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 773 def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 774 775 // Truncating-extending ternary RMWs with a constant offset 776 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 777 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 778 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 779 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 780 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 781 def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 782 def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 783 def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 784 def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 785 def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 786 787 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 788 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 789 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 790 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 791 def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 792 def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 793 def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 794 def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 795 796 // Truncating-extending ternary RMWs with just a constant offset 797 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 798 def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 799 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 800 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 801 def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 802 803 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 804 def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 805 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 806 def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 807 808 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 809 def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 810 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 811 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 812 def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 813 814 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 815 def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 816 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 817 def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 818} 819 820let Predicates = [HasAtomics] in 821defm : TerRMWTruncExtPattern< 822 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 823 ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 824 ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 825 ATOMIC_RMW32_U_CMPXCHG_I64>; 826