1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r, 17 string asmstr_s, bits<32> atomic_op, 18 string is64 = "false"> { 19 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 20 !or(0xfe00, !and(0xff, atomic_op)), is64>, 21 Requires<[HasAtomics]>; 22} 23 24multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 25 bits<32> atomic_op = -1> { 26 defm "" : NRI<oops, iops, pattern, asmstr, 27 !or(0xfe00, !and(0xff, atomic_op))>, 28 Requires<[HasAtomics]>; 29} 30 31//===----------------------------------------------------------------------===// 32// Atomic wait / notify 33//===----------------------------------------------------------------------===// 34 35let hasSideEffects = 1 in { 36defm MEMORY_ATOMIC_NOTIFY_A32 : 37 ATOMIC_I<(outs I32:$dst), 38 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 39 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41 "memory.atomic.notify \t${off}${p2align}", 0x00, "false">; 42defm MEMORY_ATOMIC_NOTIFY_A64 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 45 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46 "memory.atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47 "memory.atomic.notify \t${off}${p2align}", 0x00, "true">; 48let mayLoad = 1 in { 49defm MEMORY_ATOMIC_WAIT32_A32 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "memory.atomic.wait32 \t${off}${p2align}", 0x01, "false">; 56defm MEMORY_ATOMIC_WAIT32_A64 : 57 ATOMIC_I<(outs I32:$dst), 58 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 59 I64:$timeout), 60 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61 "memory.atomic.wait32 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62 "memory.atomic.wait32 \t${off}${p2align}", 0x01, "true">; 63defm MEMORY_ATOMIC_WAIT64_A32 : 64 ATOMIC_I<(outs I32:$dst), 65 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 66 I64:$timeout), 67 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69 "memory.atomic.wait64 \t${off}${p2align}", 0x02, "false">; 70defm MEMORY_ATOMIC_WAIT64_A64 : 71 ATOMIC_I<(outs I32:$dst), 72 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 73 I64:$timeout), 74 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75 "memory.atomic.wait64 \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76 "memory.atomic.wait64 \t${off}${p2align}", 0x02, "true">; 77} // mayLoad = 1 78} // hasSideEffects = 1 79 80// Select notifys with no constant offset. 81def NotifyPatNoOffset_A32 : 82 Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)), 83 (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, 84 Requires<[HasAddr32, HasAtomics]>; 85def NotifyPatNoOffset_A64 : 86 Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)), 87 (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, 88 Requires<[HasAddr64, HasAtomics]>; 89 90// Select notifys with a constant offset. 91 92// Pattern with address + immediate offset 93multiclass NotifyPatImmOff<PatFrag operand, string inst> { 94 def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off), 95 I32:$count)), 96 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, 97 Requires<[HasAddr32, HasAtomics]>; 98 def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off), 99 I32:$count)), 100 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, 101 Requires<[HasAddr64, HasAtomics]>; 102} 103defm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">; 104defm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">; 105 106// Select notifys with just a constant offset. 107def NotifyPatOffsetOnly_A32 : 108 Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 109 (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, 110 Requires<[HasAddr32, HasAtomics]>; 111def NotifyPatOffsetOnly_A64 : 112 Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)), 113 (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, 114 Requires<[HasAddr64, HasAtomics]>; 115 116def NotifyPatGlobalAddrOffOnly_A32 : 117 Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 118 I32:$count)), 119 (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count) 120 >, 121 Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 122def NotifyPatGlobalAddrOffOnly_A64 : 123 Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 124 I32:$count)), 125 (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count) 126 >, 127 Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 128 129// Select waits with no constant offset. 130multiclass WaitPatNoOffset<ValueType ty, Intrinsic kind, 131 string inst> { 132 def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 133 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, 134 Requires<[HasAddr32, HasAtomics]>; 135 def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), 136 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, 137 Requires<[HasAddr64, HasAtomics]>; 138} 139defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 140 "MEMORY_ATOMIC_WAIT32">; 141defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 142 "MEMORY_ATOMIC_WAIT64">; 143defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32, 144 "MEMORY_ATOMIC_WAIT32">; 145defm : WaitPatNoOffset<i64, int_wasm_memory_atomic_wait64, 146 "MEMORY_ATOMIC_WAIT64">; 147 148// Select waits with a constant offset. 149 150// Pattern with address + immediate offset 151multiclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, 152 string inst> { 153 def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 154 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, 155 I64:$timeout)>, 156 Requires<[HasAddr32, HasAtomics]>; 157 def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), 158 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, 159 I64:$timeout)>, 160 Requires<[HasAddr64, HasAtomics]>; 161} 162defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm, 163 "MEMORY_ATOMIC_WAIT32">; 164defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, or_is_add, 165 "MEMORY_ATOMIC_WAIT32">; 166defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, regPlusImm, 167 "MEMORY_ATOMIC_WAIT64">; 168defm : WaitPatImmOff<i64, int_wasm_memory_atomic_wait64, or_is_add, 169 "MEMORY_ATOMIC_WAIT64">; 170 171// Select waits with just a constant offset. 172multiclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> { 173 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 174 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 175 I64:$timeout)>, 176 Requires<[HasAddr32, HasAtomics]>; 177 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 178 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 179 I64:$timeout)>, 180 Requires<[HasAddr64, HasAtomics]>; 181} 182defm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32, 183 "MEMORY_ATOMIC_WAIT32">; 184defm : WaitPatOffsetOnly<i64, int_wasm_memory_atomic_wait64, 185 "MEMORY_ATOMIC_WAIT64">; 186 187multiclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> { 188 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 189 I64:$timeout)), 190 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 191 I64:$timeout)>, 192 Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 193 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 194 I64:$timeout)), 195 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 196 I64:$timeout)>, 197 Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 198} 199defm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32, 200 "MEMORY_ATOMIC_WAIT32">; 201defm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64, 202 "MEMORY_ATOMIC_WAIT64">; 203 204//===----------------------------------------------------------------------===// 205// Atomic fences 206//===----------------------------------------------------------------------===// 207 208// A compiler fence instruction that prevents reordering of instructions. 209let Defs = [ARGUMENTS] in { 210let isPseudo = 1, hasSideEffects = 1 in 211defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 212let hasSideEffects = 1 in 213defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 214 0x03>; 215} // Defs = [ARGUMENTS] 216 217//===----------------------------------------------------------------------===// 218// Atomic loads 219//===----------------------------------------------------------------------===// 220 221multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 222 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 223 [HasAtomics]>; 224} 225 226defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 227defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 228 229// Select loads with no constant offset. 230defm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 231defm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 232 233// Select loads with a constant offset. 234 235// Pattern with address + immediate offset 236defm : LoadPatImmOff<i32, atomic_load_32, regPlusImm, "ATOMIC_LOAD_I32">; 237defm : LoadPatImmOff<i64, atomic_load_64, regPlusImm, "ATOMIC_LOAD_I64">; 238defm : LoadPatImmOff<i32, atomic_load_32, or_is_add, "ATOMIC_LOAD_I32">; 239defm : LoadPatImmOff<i64, atomic_load_64, or_is_add, "ATOMIC_LOAD_I64">; 240 241// Select loads with just a constant offset. 242defm : LoadPatOffsetOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 243defm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 244 245defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 246defm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 247 248 249// Extending loads. Note that there are only zero-extending atomic loads, no 250// sign-extending loads. 251defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 252defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 253defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 254defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 255defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 256 257// Fragments for extending loads. These are different from regular loads because 258// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 259// therefore don't have the extension type field. So instead of matching that, 260// we match the patterns that the type legalizer expands them to. 261 262// Unlike regular loads, extension to i64 is handled differently than i32. 263// i64 (zext (i8 (atomic_load_8))) gets legalized to 264// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 265// Extension to i32 is elided by SelectionDAG as our atomic loads are 266// zero-extending. 267def zext_aload_8_64 : 268 PatFrag<(ops node:$addr), 269 (i64 (zext (i32 (atomic_load_8 node:$addr))))>; 270def zext_aload_16_64 : 271 PatFrag<(ops node:$addr), 272 (i64 (zext (i32 (atomic_load_16 node:$addr))))>; 273def zext_aload_32_64 : 274 PatFrag<(ops node:$addr), 275 (i64 (zext (i32 (atomic_load_32 node:$addr))))>; 276 277// We don't have single sext atomic load instructions. So for sext loads, we 278// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 279// results) and select a zext load; the next instruction will be sext_inreg 280// which is selected by itself. 281def sext_aload_8_64 : 282 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 283def sext_aload_16_64 : 284 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 285 286// Select zero-extending loads with no constant offset. 287defm : LoadPatNoOffset<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 288defm : LoadPatNoOffset<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 289defm : LoadPatNoOffset<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 290 291// Select sign-extending loads with no constant offset 292defm : LoadPatNoOffset<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 293defm : LoadPatNoOffset<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 294defm : LoadPatNoOffset<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 295defm : LoadPatNoOffset<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 296// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 297 298// Zero-extending loads with constant offset 299defm : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 300defm : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 301defm : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, "ATOMIC_LOAD32_U_I64">; 302defm : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 303defm : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 304defm : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, "ATOMIC_LOAD32_U_I64">; 305 306// Sign-extending loads with constant offset 307defm : LoadPatImmOff<i32, atomic_load_8, regPlusImm, "ATOMIC_LOAD8_U_I32">; 308defm : LoadPatImmOff<i32, atomic_load_16, regPlusImm, "ATOMIC_LOAD16_U_I32">; 309defm : LoadPatImmOff<i32, atomic_load_8, or_is_add, "ATOMIC_LOAD8_U_I32">; 310defm : LoadPatImmOff<i32, atomic_load_16, or_is_add, "ATOMIC_LOAD16_U_I32">; 311defm : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 312defm : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 313defm : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 314defm : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 315// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 316 317// Extending loads with just a constant offset 318defm : LoadPatOffsetOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 319defm : LoadPatOffsetOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 320defm : LoadPatOffsetOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 321defm : LoadPatOffsetOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 322defm : LoadPatOffsetOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 323defm : LoadPatOffsetOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 324defm : LoadPatOffsetOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 325 326defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 327defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 328defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 329defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 330defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 331defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 332defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 333 334 335//===----------------------------------------------------------------------===// 336// Atomic stores 337//===----------------------------------------------------------------------===// 338 339multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 340 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 341 [HasAtomics]>; 342} 343 344defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 345defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 346 347// We need an 'atomic' version of store patterns because store and atomic_store 348// nodes have different operand orders: 349// store: (store $val, $ptr) 350// atomic_store: (store $ptr, $val) 351 352 353// Select stores with no constant offset. 354multiclass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> { 355 def : Pat<(kind I32:$addr, ty:$val), 356 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 357 Requires<[HasAddr32, HasAtomics]>; 358 def : Pat<(kind I64:$addr, ty:$val), 359 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 360 Requires<[HasAddr64, HasAtomics]>; 361} 362defm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">; 363defm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">; 364 365// Select stores with a constant offset. 366 367// Pattern with address + immediate offset 368multiclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 369 string inst> { 370 def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 371 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 372 Requires<[HasAddr32, HasAtomics]>; 373 def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), 374 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 375 Requires<[HasAddr64, HasAtomics]>; 376} 377defm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">; 378defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">; 379 380// Select stores with just a constant offset. 381multiclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 382 def : Pat<(kind imm:$off, ty:$val), 383 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 384 Requires<[HasAddr32, HasAtomics]>; 385 def : Pat<(kind imm:$off, ty:$val), 386 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 387 Requires<[HasAddr64, HasAtomics]>; 388} 389defm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 390defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 391 392multiclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 393 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 394 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 395 Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 396 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 397 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 398 Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 399} 400defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 401defm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 402 403 404// Truncating stores. 405defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 406defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 407defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 408defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 409defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 410 411// Fragments for truncating stores. 412 413// We don't have single truncating atomic store instructions. For 32-bit 414// instructions, we just need to match bare atomic stores. On the other hand, 415// truncating stores from i64 values are once truncated to i32 first. 416class trunc_astore_64<PatFrag kind> : 417 PatFrag<(ops node:$addr, node:$val), 418 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 419def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 420def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 421def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 422 423 424// Truncating stores with no constant offset 425defm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 426defm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 427defm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 428defm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 429defm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 430 431// Truncating stores with a constant offset 432defm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">; 433defm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">; 434defm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">; 435defm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, 436 "ATOMIC_STORE16_I64">; 437defm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, 438 "ATOMIC_STORE32_I64">; 439defm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">; 440defm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">; 441defm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">; 442defm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, 443 "ATOMIC_STORE16_I64">; 444defm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, 445 "ATOMIC_STORE32_I64">; 446 447// Truncating stores with just a constant offset 448defm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 449defm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 450defm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 451defm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 452defm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 453 454defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 455defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 456defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 457defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 458defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 459 460 461//===----------------------------------------------------------------------===// 462// Atomic binary read-modify-writes 463//===----------------------------------------------------------------------===// 464 465multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 466 int atomic_op> { 467 defm "_A32" : 468 ATOMIC_I<(outs rc:$dst), 469 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 470 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 471 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 472 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 473 defm "_A64" : 474 ATOMIC_I<(outs rc:$dst), 475 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 476 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 477 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 478 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 479} 480 481defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 482defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 483defm ATOMIC_RMW8_U_ADD_I32 : 484 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 485defm ATOMIC_RMW16_U_ADD_I32 : 486 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 487defm ATOMIC_RMW8_U_ADD_I64 : 488 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 489defm ATOMIC_RMW16_U_ADD_I64 : 490 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 491defm ATOMIC_RMW32_U_ADD_I64 : 492 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 493 494defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 495defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 496defm ATOMIC_RMW8_U_SUB_I32 : 497 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 498defm ATOMIC_RMW16_U_SUB_I32 : 499 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 500defm ATOMIC_RMW8_U_SUB_I64 : 501 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 502defm ATOMIC_RMW16_U_SUB_I64 : 503 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 504defm ATOMIC_RMW32_U_SUB_I64 : 505 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 506 507defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 508defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 509defm ATOMIC_RMW8_U_AND_I32 : 510 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 511defm ATOMIC_RMW16_U_AND_I32 : 512 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 513defm ATOMIC_RMW8_U_AND_I64 : 514 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 515defm ATOMIC_RMW16_U_AND_I64 : 516 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 517defm ATOMIC_RMW32_U_AND_I64 : 518 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 519 520defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 521defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 522defm ATOMIC_RMW8_U_OR_I32 : 523 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 524defm ATOMIC_RMW16_U_OR_I32 : 525 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 526defm ATOMIC_RMW8_U_OR_I64 : 527 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 528defm ATOMIC_RMW16_U_OR_I64 : 529 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 530defm ATOMIC_RMW32_U_OR_I64 : 531 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 532 533defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 534defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 535defm ATOMIC_RMW8_U_XOR_I32 : 536 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 537defm ATOMIC_RMW16_U_XOR_I32 : 538 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 539defm ATOMIC_RMW8_U_XOR_I64 : 540 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 541defm ATOMIC_RMW16_U_XOR_I64 : 542 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 543defm ATOMIC_RMW32_U_XOR_I64 : 544 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 545 546defm ATOMIC_RMW_XCHG_I32 : 547 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 548defm ATOMIC_RMW_XCHG_I64 : 549 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 550defm ATOMIC_RMW8_U_XCHG_I32 : 551 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 552defm ATOMIC_RMW16_U_XCHG_I32 : 553 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 554defm ATOMIC_RMW8_U_XCHG_I64 : 555 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 556defm ATOMIC_RMW16_U_XCHG_I64 : 557 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 558defm ATOMIC_RMW32_U_XCHG_I64 : 559 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 560 561// Select binary RMWs with no constant offset. 562multiclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 563 def : Pat<(ty (kind I32:$addr, ty:$val)), 564 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 565 Requires<[HasAddr32, HasAtomics]>; 566 def : Pat<(ty (kind I64:$addr, ty:$val)), 567 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 568 Requires<[HasAddr64, HasAtomics]>; 569} 570 571// Select binary RMWs with a constant offset. 572 573// Pattern with address + immediate offset 574multiclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 575 string inst> { 576 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 577 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 578 Requires<[HasAddr32, HasAtomics]>; 579 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), 580 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 581 Requires<[HasAddr64, HasAtomics]>; 582} 583 584// Select binary RMWs with just a constant offset. 585multiclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 586 def : Pat<(ty (kind imm:$off, ty:$val)), 587 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 588 Requires<[HasAddr32, HasAtomics]>; 589 def : Pat<(ty (kind imm:$off, ty:$val)), 590 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 591 Requires<[HasAddr64, HasAtomics]>; 592} 593 594multiclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 595 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 596 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 597 Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 598 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 599 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 600 Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 601} 602 603// Patterns for various addressing modes. 604multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 605 string inst_64> { 606 defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 607 defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 608 609 defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 610 defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 611 defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 612 defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 613 614 defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 615 defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 616 617 defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 618 defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 619} 620 621defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 622 "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 623defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 624 "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 625defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 626 "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 627defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 628 "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 629defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 630 "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 631defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 632 "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 633 634// Truncating & zero-extending binary RMW patterns. 635// These are combined patterns of truncating store patterns and zero-extending 636// load patterns above. 637class zext_bin_rmw_8_32<PatFrag kind> : 638 PatFrag<(ops node:$addr, node:$val), (i32 (kind node:$addr, node:$val))>; 639class zext_bin_rmw_16_32<PatFrag kind> : zext_bin_rmw_8_32<kind>; 640class zext_bin_rmw_8_64<PatFrag kind> : 641 PatFrag<(ops node:$addr, node:$val), 642 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 643class zext_bin_rmw_16_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 644class zext_bin_rmw_32_64<PatFrag kind> : zext_bin_rmw_8_64<kind>; 645 646// Truncating & sign-extending binary RMW patterns. 647// These are combined patterns of truncating store patterns and sign-extending 648// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 649// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 650// is selected by itself. 651class sext_bin_rmw_8_32<PatFrag kind> : 652 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 653class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 654class sext_bin_rmw_8_64<PatFrag kind> : 655 PatFrag<(ops node:$addr, node:$val), 656 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 657class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 658// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 659 660// Patterns for various addressing modes for truncating-extending binary RMWs. 661multiclass BinRMWTruncExtPattern< 662 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 663 string inst8_32, string inst16_32, string inst8_64, string inst16_64, string inst32_64> { 664 // Truncating-extending binary RMWs with no constant offset 665 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 666 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 667 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 668 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 669 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 670 671 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 672 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 673 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 674 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 675 676 // Truncating-extending binary RMWs with a constant offset 677 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 678 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, 679 inst16_32>; 680 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 681 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, 682 inst16_64>; 683 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, 684 inst32_64>; 685 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 686 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 687 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 688 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 689 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 690 691 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 692 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, 693 inst16_32>; 694 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 695 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, 696 inst16_64>; 697 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 698 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 699 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 700 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 701 702 // Truncating-extending binary RMWs with just a constant offset 703 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 704 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 705 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 706 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 707 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 708 709 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 710 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 711 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 712 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 713 714 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 715 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 716 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 717 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 718 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 719 720 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 721 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 722 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 723 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 724} 725 726defm : BinRMWTruncExtPattern< 727 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 728 "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 729 "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 730defm : BinRMWTruncExtPattern< 731 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 732 "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 733 "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 734defm : BinRMWTruncExtPattern< 735 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 736 "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 737 "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 738defm : BinRMWTruncExtPattern< 739 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 740 "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 741 "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 742defm : BinRMWTruncExtPattern< 743 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 744 "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 745 "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 746defm : BinRMWTruncExtPattern< 747 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 748 "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 749 "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 750 "ATOMIC_RMW32_U_XCHG_I64">; 751 752//===----------------------------------------------------------------------===// 753// Atomic ternary read-modify-writes 754//===----------------------------------------------------------------------===// 755 756// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 757// flag}. When we use the success flag or both values, we can't make use of i64 758// truncate/extend versions of instructions for now, which is suboptimal. 759// Consider adding a pass after instruction selection that optimizes this case 760// if it is frequent. 761 762multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 763 int atomic_op> { 764 defm "_A32" : 765 ATOMIC_I<(outs rc:$dst), 766 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 767 rc:$new_), 768 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 769 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 770 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 771 defm "_A64" : 772 ATOMIC_I<(outs rc:$dst), 773 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 774 rc:$new_), 775 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 776 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 777 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 778} 779 780defm ATOMIC_RMW_CMPXCHG_I32 : 781 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 782defm ATOMIC_RMW_CMPXCHG_I64 : 783 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 784defm ATOMIC_RMW8_U_CMPXCHG_I32 : 785 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 786defm ATOMIC_RMW16_U_CMPXCHG_I32 : 787 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 788defm ATOMIC_RMW8_U_CMPXCHG_I64 : 789 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 790defm ATOMIC_RMW16_U_CMPXCHG_I64 : 791 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 792defm ATOMIC_RMW32_U_CMPXCHG_I64 : 793 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 794 795// Select ternary RMWs with no constant offset. 796multiclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 797 def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 798 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, 799 Requires<[HasAddr32, HasAtomics]>; 800 def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), 801 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, 802 Requires<[HasAddr64, HasAtomics]>; 803} 804 805// Select ternary RMWs with a constant offset. 806 807// Pattern with address + immediate offset 808multiclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 809 string inst> { 810 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 811 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, 812 Requires<[HasAddr32, HasAtomics]>; 813 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), 814 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, 815 Requires<[HasAddr64, HasAtomics]>; 816} 817 818// Select ternary RMWs with just a constant offset. 819multiclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 820 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 821 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 822 ty:$new)>; 823 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 824 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 825 ty:$new)>; 826} 827 828multiclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 829 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 830 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 831 ty:$new)>, 832 Requires<[HasAddr32, HasAtomics, IsNotPIC]>; 833 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 834 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 835 ty:$new)>, 836 Requires<[HasAddr64, HasAtomics, IsNotPIC]>; 837} 838 839// Patterns for various addressing modes. 840multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 841 string inst_64> { 842 defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 843 defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 844 845 defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 846 defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 847 defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 848 defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 849 850 defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 851 defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 852 853 defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 854 defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 855} 856 857defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 858 "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; 859 860// Truncating & zero-extending ternary RMW patterns. 861// DAG legalization & optimization before instruction selection may introduce 862// additional nodes such as anyext or assertzext depending on operand types. 863class zext_ter_rmw_8_32<PatFrag kind> : 864 PatFrag<(ops node:$addr, node:$exp, node:$new), 865 (i32 (kind node:$addr, node:$exp, node:$new))>; 866class zext_ter_rmw_16_32<PatFrag kind> : zext_ter_rmw_8_32<kind>; 867class zext_ter_rmw_8_64<PatFrag kind> : 868 PatFrag<(ops node:$addr, node:$exp, node:$new), 869 (zext (i32 (assertzext (i32 (kind node:$addr, 870 (i32 (trunc (i64 node:$exp))), 871 (i32 (trunc (i64 node:$new))))))))>; 872class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 873class zext_ter_rmw_32_64<PatFrag kind> : 874 PatFrag<(ops node:$addr, node:$exp, node:$new), 875 (zext (i32 (kind node:$addr, 876 (i32 (trunc (i64 node:$exp))), 877 (i32 (trunc (i64 node:$new))))))>; 878 879// Truncating & sign-extending ternary RMW patterns. 880// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 881// zext RMW; the next instruction will be sext_inreg which is selected by 882// itself. 883class sext_ter_rmw_8_32<PatFrag kind> : 884 PatFrag<(ops node:$addr, node:$exp, node:$new), 885 (kind node:$addr, node:$exp, node:$new)>; 886class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 887class sext_ter_rmw_8_64<PatFrag kind> : 888 PatFrag<(ops node:$addr, node:$exp, node:$new), 889 (anyext (i32 (assertzext (i32 890 (kind node:$addr, 891 (i32 (trunc (i64 node:$exp))), 892 (i32 (trunc (i64 node:$new))))))))>; 893class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 894// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 895 896// Patterns for various addressing modes for truncating-extending ternary RMWs. 897multiclass TerRMWTruncExtPattern< 898 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 899 string inst8_32, string inst16_32, string inst8_64, string inst16_64, 900 string inst32_64> { 901 // Truncating-extending ternary RMWs with no constant offset 902 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 903 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 904 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 905 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 906 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 907 908 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 909 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 910 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 911 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 912 913 // Truncating-extending ternary RMWs with a constant offset 914 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 915 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, 916 inst16_32>; 917 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 918 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, 919 inst16_64>; 920 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, 921 inst32_64>; 922 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 923 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 924 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 925 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 926 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 927 928 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 929 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, 930 inst16_32>; 931 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 932 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, 933 inst16_64>; 934 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 935 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 936 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 937 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 938 939 // Truncating-extending ternary RMWs with just a constant offset 940 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 941 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 942 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 943 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 944 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 945 946 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 947 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 948 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 949 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 950 951 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 952 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 953 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 954 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 955 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 956 957 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 958 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 959 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 960 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 961} 962 963defm : TerRMWTruncExtPattern< 964 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 965 "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", 966 "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", 967 "ATOMIC_RMW32_U_CMPXCHG_I64">; 968